mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
130
Commits
ext-v0.1.4
...
ec62b8f0f2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec62b8f0f2 | ||
|
|
9393f0e58b | ||
|
|
f9d395ee41 | ||
|
|
8f81c15656 | ||
|
|
3cfa9eceef | ||
|
|
34f47d7387 | ||
|
|
0a184e33d1 | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
c8c5c4a7c4 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 | ||
|
|
4549ed0892 | ||
|
|
37fbaacc94 | ||
|
|
f81a1e7796 | ||
|
|
78c2719718 | ||
|
|
f443cfae74 | ||
|
|
0ce497f017 | ||
|
|
08212497f1 | ||
|
|
f98c0606a1 | ||
|
|
1371c92721 | ||
|
|
c75f97119d | ||
|
|
ac806ae8f4 | ||
|
|
89200975f3 | ||
|
|
238a98476e | ||
|
|
1da2d54ddc | ||
|
|
6e35e62199 | ||
|
|
610782698b | ||
|
|
9504072914 | ||
|
|
b4450fbb8c | ||
|
|
1268761611 | ||
|
|
37d3e2b22f | ||
|
|
566b6c17fd | ||
|
|
2ccc0aafb9 | ||
|
|
ce8bc6c870 | ||
|
|
eeab97bcdd | ||
|
|
6c610faed9 | ||
|
|
d685c75d25 | ||
|
|
73724617f9 | ||
|
|
4ba6fb45e1 | ||
|
|
93df62a356 | ||
|
|
20002e1ff2 | ||
|
|
674c53339f | ||
|
|
9d4ac90592 | ||
|
|
cde6f22ecf | ||
|
|
d24bb6aa93 | ||
|
|
dbc4845c0d | ||
|
|
d0bee1a09b | ||
|
|
91225e5fdb | ||
|
|
66684c4ce3 | ||
|
|
0f264c5df1 | ||
|
|
49378d70de | ||
|
|
d7f0f1e78c | ||
|
|
71255ae142 | ||
|
|
3f091318d0 | ||
|
|
c6a2b3ec1e | ||
|
|
d67c1a71b0 | ||
|
|
03d2adc035 | ||
|
|
8f5df59b79 | ||
|
|
000fbf5540 | ||
|
|
a52c2d6bf2 | ||
|
|
c002c394e6 | ||
|
|
4fb2e331a6 | ||
|
|
e14ae9cea0 | ||
|
|
f319672721 | ||
|
|
5e0d26ccb2 | ||
|
|
9dbc553e33 | ||
|
|
b657ab574b | ||
|
|
959bfac1ae | ||
|
|
c095128218 | ||
|
|
f73a0e9fb3 | ||
|
|
4e44fb2823 | ||
|
|
c8963e3a68 | ||
|
|
aab3b7a8e5 | ||
|
|
2dad162a87 | ||
|
|
aa63465050 | ||
|
|
063b0bbc91 | ||
|
|
0de67c537a | ||
|
|
29ff239726 | ||
|
|
31d2a1b3dc | ||
|
|
ed92e54dff | ||
|
|
cb11cecdf6 | ||
|
|
ea03f36f3a | ||
|
|
234e9101ed | ||
|
|
5e91178e03 |
@@ -0,0 +1,105 @@
|
|||||||
|
name: Release dsh plugin
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- dsh-plugin-v*
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: release-dsh-plugin-${{ github.ref }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||||
|
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
resolve:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
version: ${{ steps.version.outputs.version }}
|
||||||
|
tag: ${{ steps.version.outputs.tag }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Resolve plugin version
|
||||||
|
id: version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||||
|
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
else
|
||||||
|
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||||
|
fi
|
||||||
|
VERSION="${VERSION#v}"
|
||||||
|
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
guard:
|
||||||
|
needs: resolve
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Verify tag matches package.json name and version
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||||
|
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||||
|
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||||
|
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||||
|
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||||
|
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||||
|
|
||||||
|
publish:
|
||||||
|
needs: [resolve, guard]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||||
|
environment: npm-publish
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- uses: pnpm/action-setup@v6
|
||||||
|
|
||||||
|
- uses: actions/setup-node@v6
|
||||||
|
with:
|
||||||
|
node-version: 22
|
||||||
|
cache: pnpm
|
||||||
|
registry-url: https://registry.npmjs.org
|
||||||
|
scope: "@wxg-prc-cpg"
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Typecheck plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||||
|
|
||||||
|
- name: Test plugin
|
||||||
|
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||||
|
|
||||||
|
- name: Publish to npm
|
||||||
|
working-directory: ${{ env.PACKAGE_DIR }}
|
||||||
|
env:
|
||||||
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||||
|
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
pnpm publish --access public --no-git-checks
|
||||||
|
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||||
@@ -126,3 +126,202 @@ jobs:
|
|||||||
make_latest: false
|
make_latest: false
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
files: dist/*.zip
|
files: dist/*.zip
|
||||||
|
|
||||||
|
publish-cws:
|
||||||
|
needs: [resolve, build-extension]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: chrome-web-store
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Download build artifact
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: extension-zip
|
||||||
|
path: dist
|
||||||
|
|
||||||
|
- name: Validate Chrome Web Store configuration
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PUBLISHER_ID: ${{ vars.CHROME_WEBSTORE_PUBLISHER_ID }}
|
||||||
|
EXTENSION_ID: ${{ vars.CHROME_WEBSTORE_EXTENSION_ID }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "$PUBLISHER_ID" ]; then
|
||||||
|
echo "::error::Repository variable CHROME_WEBSTORE_PUBLISHER_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$EXTENSION_ID" ]; then
|
||||||
|
echo "::error::Repository variable CHROME_WEBSTORE_EXTENSION_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||||
|
if [ -z "$ARCHIVE" ]; then
|
||||||
|
echo "::error::No extension zip found in the downloaded artifact"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Authenticate with service account
|
||||||
|
id: cws-auth
|
||||||
|
env:
|
||||||
|
SERVICE_ACCOUNT_JSON: ${{ secrets.CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
node --input-type=module <<'NODE'
|
||||||
|
import { sign } from "node:crypto";
|
||||||
|
import { appendFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const rawCredentials = process.env.SERVICE_ACCOUNT_JSON;
|
||||||
|
if (!rawCredentials) {
|
||||||
|
throw new Error(
|
||||||
|
"Repository secret CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON is not configured",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let credentials;
|
||||||
|
try {
|
||||||
|
credentials = JSON.parse(rawCredentials);
|
||||||
|
} catch {
|
||||||
|
throw new Error(
|
||||||
|
"Repository secret CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON is not valid JSON",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!credentials.client_email || !credentials.private_key) {
|
||||||
|
throw new Error(
|
||||||
|
"Service account JSON must contain client_email and private_key",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const encode = (value) =>
|
||||||
|
Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||||
|
const issuedAt = Math.floor(Date.now() / 1000);
|
||||||
|
const header = {
|
||||||
|
alg: "RS256",
|
||||||
|
typ: "JWT",
|
||||||
|
kid: credentials.private_key_id,
|
||||||
|
};
|
||||||
|
const claim = {
|
||||||
|
iss: credentials.client_email,
|
||||||
|
scope: "https://www.googleapis.com/auth/chromewebstore",
|
||||||
|
aud: "https://oauth2.googleapis.com/token",
|
||||||
|
iat: issuedAt,
|
||||||
|
exp: issuedAt + 3600,
|
||||||
|
};
|
||||||
|
const unsignedJwt = `${encode(header)}.${encode(claim)}`;
|
||||||
|
const signature = sign(
|
||||||
|
"RSA-SHA256",
|
||||||
|
Buffer.from(unsignedJwt),
|
||||||
|
credentials.private_key,
|
||||||
|
).toString("base64url");
|
||||||
|
const assertion = `${unsignedJwt}.${signature}`;
|
||||||
|
|
||||||
|
const response = await fetch("https://oauth2.googleapis.com/token", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||||
|
body: new URLSearchParams({
|
||||||
|
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
||||||
|
assertion,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const responseText = await response.text();
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(
|
||||||
|
`Service account token exchange failed (${response.status}): ${responseText}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const tokenResponse = JSON.parse(responseText);
|
||||||
|
if (!tokenResponse.access_token) {
|
||||||
|
throw new Error("Service account token response has no access_token");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`::add-mask::${tokenResponse.access_token}`);
|
||||||
|
appendFileSync(
|
||||||
|
process.env.GITHUB_OUTPUT,
|
||||||
|
`access_token=${tokenResponse.access_token}\n`,
|
||||||
|
);
|
||||||
|
NODE
|
||||||
|
|
||||||
|
- name: Upload and publish Chrome Web Store extension
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
ACCESS_TOKEN: ${{ steps.cws-auth.outputs.access_token }}
|
||||||
|
PUBLISHER_ID: ${{ vars.CHROME_WEBSTORE_PUBLISHER_ID }}
|
||||||
|
EXTENSION_ID: ${{ vars.CHROME_WEBSTORE_EXTENSION_ID }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
ITEM_NAME="publishers/${PUBLISHER_ID}/items/${EXTENSION_ID}"
|
||||||
|
API_ROOT="https://chromewebstore.googleapis.com"
|
||||||
|
UPLOAD_RESPONSE="$RUNNER_TEMP/cws-upload-response.json"
|
||||||
|
STATUS_RESPONSE="$RUNNER_TEMP/cws-status-response.json"
|
||||||
|
PUBLISH_RESPONSE="$RUNNER_TEMP/cws-publish-response.json"
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$UPLOAD_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||||
|
--header "Content-Type: application/zip" \
|
||||||
|
--data-binary "@$ARCHIVE" \
|
||||||
|
"$API_ROOT/upload/v2/$ITEM_NAME:upload")"
|
||||||
|
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||||
|
echo "::error::Chrome Web Store upload failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
UPLOAD_STATE="$(jq -r '.uploadState // empty' "$UPLOAD_RESPONSE")"
|
||||||
|
echo "Chrome Web Store upload state: $UPLOAD_STATE"
|
||||||
|
|
||||||
|
if [ "$UPLOAD_STATE" = "IN_PROGRESS" ] || [ "$UPLOAD_STATE" = "UPLOAD_IN_PROGRESS" ]; then
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
sleep 5
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$STATUS_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||||
|
"$API_ROOT/v2/$ITEM_NAME:fetchStatus")"
|
||||||
|
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||||
|
echo "::error::Chrome Web Store status check failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
UPLOAD_STATE="$(jq -r '.lastAsyncUploadState // empty' "$STATUS_RESPONSE")"
|
||||||
|
echo "Upload status check $attempt/30: $UPLOAD_STATE"
|
||||||
|
if [ "$UPLOAD_STATE" = "SUCCEEDED" ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$UPLOAD_STATE" != "IN_PROGRESS" ] && [ "$UPLOAD_STATE" != "UPLOAD_IN_PROGRESS" ]; then
|
||||||
|
echo "::error::Chrome Web Store upload did not succeed: $UPLOAD_STATE"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$UPLOAD_STATE" != "SUCCEEDED" ]; then
|
||||||
|
echo "::error::Chrome Web Store upload did not complete: $UPLOAD_STATE"
|
||||||
|
jq . "$UPLOAD_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$PUBLISH_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||||
|
--header "Content-Type: application/json" \
|
||||||
|
--data '{"publishType":"DEFAULT_PUBLISH"}' \
|
||||||
|
"$API_ROOT/v2/$ITEM_NAME:publish")"
|
||||||
|
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||||
|
echo "::error::Chrome Web Store publish failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Chrome Web Store submission created successfully:"
|
||||||
|
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||||
|
|||||||
@@ -3,3 +3,6 @@
|
|||||||
**/.output/**
|
**/.output/**
|
||||||
**/target/**
|
**/target/**
|
||||||
**/node_modules/**
|
**/node_modules/**
|
||||||
|
# Generated client CSS (tailwind compile output); built by
|
||||||
|
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||||
|
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||||
|
|||||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
|||||||
## 3. Open the extension install page
|
## 3. Open the extension install page
|
||||||
|
|
||||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||||
has not installed or enabled the browser extension yet. Open the
|
has not installed or enabled the browser extension yet. Open the install page
|
||||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
matching the user's browser —
|
||||||
in the user's browser, then tell the user:
|
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
for Chrome and other Chromium browsers,
|
||||||
|
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
for Microsoft Edge — then tell the user:
|
||||||
|
|
||||||
> Install **BrowserSkill** from the
|
> Install **BrowserSkill** from the
|
||||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
> open the popup, and wait until it turns green. Reply when done.
|
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||||
|
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||||
|
|
||||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||||
once more. All `ok`/`na` → tell the user it's ready.
|
once more. All `ok`/`na` → tell the user it's ready.
|
||||||
|
|||||||
Generated
+3
-2
@@ -134,9 +134,10 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk"
|
name = "bsk"
|
||||||
version = "0.1.8"
|
version = "0.1.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64",
|
||||||
"bsk-protocol",
|
"bsk-protocol",
|
||||||
"clap",
|
"clap",
|
||||||
"console",
|
"console",
|
||||||
@@ -168,7 +169,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk-protocol"
|
name = "bsk-protocol"
|
||||||
version = "0.1.8"
|
version = "0.1.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"semver",
|
"semver",
|
||||||
|
|||||||
+3
-2
@@ -7,7 +7,7 @@ members = [
|
|||||||
default-members = ["crates/bsk-cli"]
|
default-members = ["crates/bsk-cli"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.1.8"
|
version = "0.1.10"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@@ -36,13 +36,14 @@ flate2 = "1.1.9"
|
|||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
||||||
sha2 = "0.11.0"
|
sha2 = "0.11.0"
|
||||||
|
base64 = "0.22"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
nix = { version = "0.29", features = ["signal", "process"] }
|
nix = { version = "0.29", features = ["signal", "process"] }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
uuid = { version = "1", features = ["v4", "serde"] }
|
uuid = { version = "1", features = ["v4", "serde"] }
|
||||||
|
|
||||||
# Workspace crates (version required for `cargo publish` of dependents)
|
# Workspace crates (version required for `cargo publish` of dependents)
|
||||||
bsk-protocol = { version = "0.1.7", path = "crates/bsk-protocol" }
|
bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" }
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
strip = "symbols"
|
strip = "symbols"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||||
browser.
|
browser.
|
||||||
|
|
||||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||||
|
|
||||||
#### 1. Install the `bsk` CLI
|
#### 1. Install the `bsk` CLI
|
||||||
|
|
||||||
@@ -91,7 +92,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. Install the browser extension
|
#### 2. Install the browser extension
|
||||||
|
|
||||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
Install BrowserSkill from your browser's store:
|
||||||
|
|
||||||
|
| Browser | Store listing |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||||
|
|
||||||
#### 3. Install the skill
|
#### 3. Install the skill
|
||||||
|
|
||||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
|||||||
|
|
||||||
Other shell-capable agent harnesses are supported too. Copy
|
Other shell-capable agent harnesses are supported too. Copy
|
||||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||||
`browser-skill/SKILL.md` to install the skill manually.
|
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||||
|
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness plugin
|
||||||
|
|
||||||
|
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||||
|
BrowserSkill ships a first-class dsh plugin on npm as
|
||||||
|
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||||
|
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||||
|
overlay of each Agent Window.
|
||||||
|
|
||||||
|
Add it to a dsh profile, then start that profile:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||||
|
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||||
|
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||||
|
configuration, and the observation overlay.
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
|
|
||||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||||
@@ -166,7 +195,9 @@ flowchart TB
|
|||||||
|
|
||||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||||
browser task; the local daemon routes that request to the extension; the
|
browser task; the local daemon routes that request to the extension; the
|
||||||
extension runs it in an Agent Window.
|
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||||
|
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||||
|
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||||
|
|
||||||
## For Developers
|
## For Developers
|
||||||
|
|
||||||
@@ -176,6 +207,7 @@ The repository is a Cargo + pnpm workspace:
|
|||||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||||
- `apps/extension` — browser extension
|
- `apps/extension` — browser extension
|
||||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+32
-7
@@ -12,7 +12,7 @@
|
|||||||
<a href="README.md">English</a> · 中文
|
<a href="README.md">English</a> · 中文
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||||
|
|
||||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||||
|
|
||||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
|
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||||
|
|
||||||
#### 1. 安装 `bsk` CLI
|
#### 1. 安装 `bsk` CLI
|
||||||
|
|
||||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
|
||||||
|
```powershell
|
||||||
|
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||||
|
```
|
||||||
|
|
||||||
验证二进制:
|
验证二进制:
|
||||||
|
|
||||||
@@ -75,7 +79,14 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. 安装浏览器扩展
|
#### 2. 安装浏览器扩展
|
||||||
|
|
||||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
在对应浏览器的商店安装 BrowserSkill:
|
||||||
|
|
||||||
|
| 浏览器 | 商店页面 |
|
||||||
|
| --- | --- |
|
||||||
|
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||||
|
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||||
|
|
||||||
|
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||||
|
|
||||||
#### 3. 安装 skill
|
#### 3. 安装 skill
|
||||||
|
|
||||||
@@ -102,7 +113,7 @@ bsk install-skill
|
|||||||
|
|
||||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||||
|
|
||||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -112,6 +123,19 @@ bsk install-skill
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## DeepSeek Harness 插件
|
||||||
|
|
||||||
|
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||||
|
|
||||||
|
把它装进某个 dsh profile,然后启动该 profile:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||||
|
dsh --profile web
|
||||||
|
```
|
||||||
|
|
||||||
|
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||||
|
|
||||||
## 工作原理
|
## 工作原理
|
||||||
|
|
||||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||||
@@ -143,7 +167,7 @@ flowchart TB
|
|||||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||||
```
|
```
|
||||||
|
|
||||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||||
|
|
||||||
## 面向开发者
|
## 面向开发者
|
||||||
|
|
||||||
@@ -153,6 +177,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
|||||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||||
- `apps/extension` — 浏览器扩展
|
- `apps/extension` — 浏览器扩展
|
||||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||||
|
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# BrowserSkill — Privacy Policy
|
# BrowserSkill — Privacy Policy
|
||||||
|
|
||||||
**Last updated:** May 25, 2026
|
**Last updated:** August 20, 2026
|
||||||
|
|
||||||
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
|
|||||||
|
|
||||||
## 2. Single Purpose
|
## 2. Single Purpose
|
||||||
|
|
||||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||||
|
|
||||||
## 3. Data the Extension Accesses
|
## 3. Data the Extension Accesses
|
||||||
|
|
||||||
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
|
|||||||
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
||||||
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
||||||
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
||||||
|
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
|
||||||
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
||||||
|
|
||||||
## 4. Data the Extension Does **Not** Collect
|
## 4. Data the Extension Does **Not** Collect
|
||||||
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
|
|||||||
|
|
||||||
- Send any data to remote servers, the Extension's authors, or any third party.
|
- Send any data to remote servers, the Extension's authors, or any third party.
|
||||||
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
||||||
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
|
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
|
||||||
- Use webcam, microphone, geolocation, or any device sensor.
|
- Use webcam, microphone, geolocation, or any device sensor.
|
||||||
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
||||||
- Track users across websites or across sessions.
|
- Track users across websites or across sessions.
|
||||||
@@ -48,7 +49,9 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
|||||||
- **`tabs`** — Inspect, create, and close tabs in the Agent Window; query tab metadata.
|
- **`tabs`** — Inspect, create, and close tabs in the Agent Window; query tab metadata.
|
||||||
- **`windows`** — Create and manage the dedicated Agent Window that isolates agent activity from the user's normal browsing.
|
- **`windows`** — Create and manage the dedicated Agent Window that isolates agent activity from the user's normal browsing.
|
||||||
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
||||||
|
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
||||||
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
||||||
|
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
|
||||||
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
||||||
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
||||||
|
|
||||||
@@ -60,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
|
|||||||
|
|
||||||
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
||||||
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
||||||
|
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
|
||||||
|
|
||||||
## 8. User Control
|
## 8. User Control
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||||
|
<path d="M6.30589 17.9712C6.21652 18.4603 6.63298 18.8722 7.12749 18.7838C7.88158 18.6491 8.86635 18.4075 9.25454 18.0236C9.49544 17.7853 9.61164 17.4432 9.57757 17.0726C9.5435 16.702 9.36195 16.3331 9.07287 16.0472C8.78378 15.7613 8.41084 15.5818 8.03608 15.5481C7.66133 15.5144 7.31545 15.6293 7.07455 15.8676C6.6865 16.2513 6.4422 17.2253 6.30589 17.9712Z" fill="#E94500"/>
|
||||||
|
<path d="M7.77425 8.64281C10.7374 4.12636 15.4876 3.7066 17.9364 4.13499C18.3335 4.20461 18.6474 4.48261 18.7821 4.84105C20.6715 9.87191 18.4167 13.7056 16.5008 15.5334C16.1262 15.8909 15.9395 16.3745 16.0282 16.8215L16.12 17.2834C16.24 17.887 15.871 18.4819 15.2723 18.6496L12.7811 19.3469C12.1057 19.536 11.4124 19.1112 11.285 18.4289L11.036 17.0959C10.9844 16.82 10.8353 16.5712 10.6151 16.3948L8.12777 14.4035C7.90665 14.2266 7.62818 14.1345 7.34359 14.1428L5.72835 14.1897C4.9877 14.2112 4.40832 13.5657 4.51644 12.8401L4.89632 10.2912C4.98194 9.71734 5.47627 9.2895 6.06234 9.28148L6.69124 9.27171C7.11716 9.26594 7.52243 9.02649 7.77425 8.64281ZM14.4959 7.91429C13.8284 7.91436 13.2869 8.45568 13.2869 9.12327C13.2871 9.79078 13.8284 10.3322 14.4959 10.3323C15.1635 10.3323 15.7048 9.79082 15.7049 9.12327C15.7049 8.45564 15.1636 7.91429 14.4959 7.91429Z" fill="#E94500"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 1.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@browser-skill/extension",
|
"name": "@browser-skill/extension",
|
||||||
"version": "0.1.4",
|
"version": "0.1.6",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@browser-skill/i18n": "workspace:*",
|
"@browser-skill/i18n": "workspace:*",
|
||||||
"@browser-skill/ui": "workspace:*",
|
"@browser-skill/ui": "workspace:*",
|
||||||
|
"@browser-skill/vom": "workspace:*",
|
||||||
"@remixicon/react": "^4.9.0",
|
"@remixicon/react": "^4.9.0",
|
||||||
"react": "^19.0.0",
|
"react": "^19.0.0",
|
||||||
"react-dom": "^19.0.0"
|
"react-dom": "^19.0.0"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||||
|
|
||||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||||
const listeners = new Set<(...args: TArgs) => void>();
|
const listeners = new Set<(...args: TArgs) => void>();
|
||||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeApi() {
|
function fakeApi() {
|
||||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||||
const api: CdpDebuggerApi = {
|
const api: CdpDebuggerApi = {
|
||||||
attach: vi.fn(async () => {}),
|
attach: vi.fn(async () => {}),
|
||||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("ChromiumCdp", () => {
|
describe("ChromiumCdp", () => {
|
||||||
|
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (
|
||||||
|
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||||
|
method: string,
|
||||||
|
params?: { frameId?: string },
|
||||||
|
) => {
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||||
|
}
|
||||||
|
if (method !== "Page.getFrameTree") return {};
|
||||||
|
if (target.sessionId === "right-session") {
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (target.sessionId === "nested-session") {
|
||||||
|
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frameTree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
await cdp.ensureAttached(4);
|
||||||
|
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||||
|
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||||
|
sessionId: "nested-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(4);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||||
|
"right-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||||
|
"nested-session",
|
||||||
|
);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||||
|
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "right-session" },
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: "nested" },
|
||||||
|
);
|
||||||
|
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 4, sessionId: "nested-session" },
|
||||||
|
"Target.setAutoAttach",
|
||||||
|
expect.objectContaining({ flatten: true }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||||
|
const { api, onEvent } = fakeApi();
|
||||||
|
const depth = 6;
|
||||||
|
const scheduled = new Set<string>();
|
||||||
|
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||||
|
frame: {
|
||||||
|
id: index === 0 ? "main" : `frame-${index}`,
|
||||||
|
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||||
|
},
|
||||||
|
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||||
|
});
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||||
|
if (method === "Target.setAutoAttach") {
|
||||||
|
const parentIndex = target.sessionId
|
||||||
|
? Number(target.sessionId.replace("session-", ""))
|
||||||
|
: 0;
|
||||||
|
const nextIndex = parentIndex + 1;
|
||||||
|
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||||
|
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||||
|
scheduled.add(key);
|
||||||
|
setTimeout(() => {
|
||||||
|
onEvent.fire(
|
||||||
|
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||||
|
"Target.attachedToTarget",
|
||||||
|
{
|
||||||
|
sessionId: `session-${nextIndex}`,
|
||||||
|
targetInfo: { type: "iframe" },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}, 5);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "Page.getFrameTree") {
|
||||||
|
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||||
|
const index = Number(target.sessionId.replace("session-", ""));
|
||||||
|
return { frameTree: nestedTree(index) };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getFrameOwner") {
|
||||||
|
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|
||||||
|
const graph = await cdp.getFrameGraph(4);
|
||||||
|
|
||||||
|
expect(graph.frames).toHaveLength(depth + 1);
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
expect(
|
||||||
|
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||||
|
).toBe(`session-${index}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
let releaseAttach!: () => void;
|
let releaseAttach!: () => void;
|
||||||
@@ -67,6 +196,30 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.isAttached(5)).toBe(false);
|
expect(cdp.isAttached(5)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rolls back the raw attach when a domain enable fails so the tab is not left stuck", async () => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (_target, method: string) => {
|
||||||
|
if (method === "Page.enable") throw new Error("Page.enable rejected");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
|
||||||
|
// Raw attach succeeds, but Page.enable fails. Without the rollback
|
||||||
|
// the debugger would stay attached while `attachedTabs` omits the id,
|
||||||
|
// leaving the tab stuck on "Another debugger is already attached" for
|
||||||
|
// every later call until the extension is reloaded.
|
||||||
|
await expect(cdp.ensureAttached(42)).rejects.toThrow(/Page\.enable rejected/);
|
||||||
|
expect(cdp.isAttached(42)).toBe(false);
|
||||||
|
expect(api.detach).toHaveBeenCalledWith({ tabId: 42 });
|
||||||
|
|
||||||
|
// The rollback detached, so a fresh attach can succeed afterwards.
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockResolvedValue({});
|
||||||
|
await expect(cdp.ensureAttached(42)).resolves.toBeUndefined();
|
||||||
|
expect(cdp.isAttached(42)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("send() rejects on chrome.runtime.lastError-style failures", async () => {
|
it("send() rejects on chrome.runtime.lastError-style failures", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
@@ -132,11 +285,33 @@ describe("ChromiumCdp", () => {
|
|||||||
expect(cdp.isAttached(12)).toBe(true);
|
expect(cdp.isAttached(12)).toBe(true);
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Runtime.enable", {});
|
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Runtime.enable", {});
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Log.enable", {});
|
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Log.enable", {});
|
||||||
await cdp.ensureConsoleCapture(12);
|
});
|
||||||
const enableCalls = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
|
||||||
|
it("retries console capture after a domain enable fails during attach", async () => {
|
||||||
|
const { api } = fakeApi();
|
||||||
|
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||||
|
async (_target, method: string) => {
|
||||||
|
if (method === "Log.enable") throw new Error("restricted");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = new ChromiumCdp(api);
|
||||||
|
// Attach is best-effort: Log.enable fails but attach still succeeds,
|
||||||
|
// leaving the tab unmarked so capture can be retried.
|
||||||
|
await cdp.ensureAttached(12);
|
||||||
|
const afterAttach = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||||
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||||
);
|
);
|
||||||
expect(enableCalls).toHaveLength(2);
|
expect(afterAttach).toHaveLength(2);
|
||||||
|
|
||||||
|
// Before the fix the "attempted" flag was set before success, so this
|
||||||
|
// was a no-op and the tab silently returned no console output forever.
|
||||||
|
// Now it retries both domains.
|
||||||
|
await cdp.ensureConsoleCapture(12);
|
||||||
|
const afterRetry = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||||
|
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||||
|
);
|
||||||
|
expect(afterRetry).toHaveLength(4);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("retries and surfaces Network.enable failures for explicit capture", async () => {
|
it("retries and surfaces Network.enable failures for explicit capture", async () => {
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||||
|
|
||||||
|
describe("buildFrameGraph", () => {
|
||||||
|
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||||
|
const graph = buildFrameGraph([
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main", url: "https://app.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||||
|
{
|
||||||
|
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||||
|
childFrames: [
|
||||||
|
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "right", url: "https://right.test" },
|
||||||
|
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(4);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "right-session" },
|
||||||
|
});
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||||
|
parentFrameId: "right",
|
||||||
|
target: { tabId: 4, sessionId: "nested-session" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates each target boundary to its same-process descendants", () => {
|
||||||
|
const sources: CdpFrameTreeSource[] = [
|
||||||
|
{
|
||||||
|
target: { tabId: 4 },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-a", parentId: "main" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-a" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-a" },
|
||||||
|
childFrames: [
|
||||||
|
{
|
||||||
|
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||||
|
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "session-c" },
|
||||||
|
tree: {
|
||||||
|
frame: { id: "oopif-c" },
|
||||||
|
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
|
||||||
|
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||||
|
["main", undefined],
|
||||||
|
["oopif-a", "session-a"],
|
||||||
|
["same-process-b", "session-a"],
|
||||||
|
["oopif-c", "session-c"],
|
||||||
|
["same-process-d", "session-c"],
|
||||||
|
]);
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||||
|
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||||
|
"same-process-b",
|
||||||
|
);
|
||||||
|
|
||||||
|
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||||
|
expect(reordered).toEqual(graph);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||||
|
const depth = 5_000;
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||||
|
let parent = root;
|
||||||
|
for (let index = 1; index <= depth; index += 1) {
|
||||||
|
const child: CdpFrameTreeNode = {
|
||||||
|
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||||
|
};
|
||||||
|
parent.childFrames = [child];
|
||||||
|
parent = child;
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||||
|
|
||||||
|
expect(graph?.frames).toHaveLength(depth + 1);
|
||||||
|
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||||
|
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||||
|
root.childFrames = [root];
|
||||||
|
|
||||||
|
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -31,6 +31,15 @@ import type {
|
|||||||
NetworkEntryKind,
|
NetworkEntryKind,
|
||||||
NetworkResult,
|
NetworkResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
buildFrameGraph,
|
||||||
|
type CdpFrameGraph,
|
||||||
|
type CdpFrameTreeNode,
|
||||||
|
type CdpFrameTreeSource,
|
||||||
|
type CdpTarget,
|
||||||
|
} from "./frame-graph";
|
||||||
|
|
||||||
|
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||||
@@ -38,21 +47,15 @@ import type {
|
|||||||
* fake without monkey-patching the real `chrome` global.
|
* fake without monkey-patching the real `chrome` global.
|
||||||
*/
|
*/
|
||||||
export interface CdpDebuggerApi {
|
export interface CdpDebuggerApi {
|
||||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
detach(target: CdpDebuggee): Promise<void>;
|
||||||
sendCommand(
|
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||||
target: chrome.debugger.Debuggee,
|
|
||||||
method: string,
|
|
||||||
commandParams?: object,
|
|
||||||
): Promise<unknown>;
|
|
||||||
/**
|
/**
|
||||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||||
* …). The first callback argument is the source debuggee; the second
|
* …). The first callback argument is the source debuggee; the second
|
||||||
* is the CDP method name; the third is the payload.
|
* is the CDP method name; the third is the payload.
|
||||||
*/
|
*/
|
||||||
onEvent: chrome.events.Event<
|
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
|
||||||
>;
|
|
||||||
/**
|
/**
|
||||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||||
* the tab navigated to a chrome:// URL or the user clicked
|
* the tab navigated to a chrome:// URL or the user clicked
|
||||||
@@ -85,6 +88,23 @@ export const CDP_PROTOCOL_VERSION = "1.3";
|
|||||||
/** Per-tab monotonic sequence returned by [`ChromiumCdp.dialogCursor`]. */
|
/** Per-tab monotonic sequence returned by [`ChromiumCdp.dialogCursor`]. */
|
||||||
export type DialogCursor = number;
|
export type DialogCursor = number;
|
||||||
|
|
||||||
|
/** CDP `Emulation.setDeviceMetricsOverride` payload. */
|
||||||
|
export interface DeviceMetricsOverride {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
/** `0` asks Chrome to use the display's native factor. */
|
||||||
|
deviceScaleFactor: number;
|
||||||
|
mobile: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CDP `Emulation.setUserAgentOverride` payload. */
|
||||||
|
export interface UserAgentOverride {
|
||||||
|
userAgent: string;
|
||||||
|
acceptLanguage?: string;
|
||||||
|
/** CDP `Emulation.UserAgentMetadata` (already camelCase). */
|
||||||
|
userAgentMetadata?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
const MAX_DIALOG_BUFFER = 32;
|
const MAX_DIALOG_BUFFER = 32;
|
||||||
const MAX_DIALOG_FIELD_LENGTH = 4096;
|
const MAX_DIALOG_FIELD_LENGTH = 4096;
|
||||||
const MAX_CONSOLE_BUFFER = 200;
|
const MAX_CONSOLE_BUFFER = 200;
|
||||||
@@ -93,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
|||||||
const MAX_NETWORK_BUFFER = 200;
|
const MAX_NETWORK_BUFFER = 200;
|
||||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||||
const MAX_NETWORK_REQUEST_META = 1024;
|
const MAX_NETWORK_REQUEST_META = 1024;
|
||||||
|
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||||
|
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||||
|
|
||||||
interface ParsedDialogOpening {
|
interface ParsedDialogOpening {
|
||||||
type: JavaScriptDialogType;
|
type: JavaScriptDialogType;
|
||||||
@@ -114,6 +136,27 @@ interface NetworkRequestMeta {
|
|||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FrameDiscoveryState {
|
||||||
|
sessions: Set<string>;
|
||||||
|
pending: Set<Promise<void>>;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||||
|
if (promises.length === 0) return true;
|
||||||
|
const remaining = deadline - Date.now();
|
||||||
|
if (remaining <= 0) return false;
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const timedOut = await Promise.race([
|
||||||
|
Promise.allSettled(promises).then(() => false),
|
||||||
|
new Promise<boolean>((resolve) => {
|
||||||
|
timer = setTimeout(() => resolve(true), remaining);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
return !timedOut;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||||
* tabId" cache and exposes typed `send<T>()`.
|
* tabId" cache and exposes typed `send<T>()`.
|
||||||
@@ -127,15 +170,17 @@ export class ChromiumCdp {
|
|||||||
private readonly dialogSequences = new Map<number, number>();
|
private readonly dialogSequences = new Map<number, number>();
|
||||||
private readonly consoleBuffers = new Map<number, ConsoleEntry[]>();
|
private readonly consoleBuffers = new Map<number, ConsoleEntry[]>();
|
||||||
private readonly consoleSequences = new Map<number, number>();
|
private readonly consoleSequences = new Map<number, number>();
|
||||||
private readonly consoleDomainsAttemptedTabs = new Set<number>();
|
private readonly consoleDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkBuffers = new Map<number, NetworkEntry[]>();
|
private readonly networkBuffers = new Map<number, NetworkEntry[]>();
|
||||||
private readonly networkSequences = new Map<number, number>();
|
private readonly networkSequences = new Map<number, number>();
|
||||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||||
|
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||||
private detachSubscription: { dispose(): void } | null = null;
|
private detachSubscription: { dispose(): void } | null = null;
|
||||||
private dialogSubscription: { dispose(): void } | null = null;
|
private dialogSubscription: { dispose(): void } | null = null;
|
||||||
private consoleSubscription: { dispose(): void } | null = null;
|
private consoleSubscription: { dispose(): void } | null = null;
|
||||||
private networkSubscription: { dispose(): void } | null = null;
|
private networkSubscription: { dispose(): void } | null = null;
|
||||||
|
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||||
|
|
||||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||||
this.api = api;
|
this.api = api;
|
||||||
@@ -143,6 +188,7 @@ export class ChromiumCdp {
|
|||||||
this.bindDialogHandler();
|
this.bindDialogHandler();
|
||||||
this.bindConsoleHandler();
|
this.bindConsoleHandler();
|
||||||
this.bindNetworkHandler();
|
this.bindNetworkHandler();
|
||||||
|
this.bindFrameTargetHandler();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Attach to `tabId` if we haven't already in this driver. */
|
/** Attach to `tabId` if we haven't already in this driver. */
|
||||||
@@ -155,10 +201,29 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
const attach = (async () => {
|
const attach = (async () => {
|
||||||
await this.api.attach({ tabId }, CDP_PROTOCOL_VERSION);
|
await this.api.attach({ tabId }, CDP_PROTOCOL_VERSION);
|
||||||
await this.enablePageDomain(tabId);
|
try {
|
||||||
await this.enableConsoleDomains(tabId);
|
await this.enablePageDomain(tabId);
|
||||||
await this.enableNetworkDomainBestEffort(tabId);
|
await this.enableConsoleDomains(tabId);
|
||||||
this.attachedTabs.add(tabId);
|
await this.enableNetworkDomainBestEffort(tabId);
|
||||||
|
this.attachedTabs.add(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||||
|
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
// A CDP domain enable failed after the raw attach succeeded
|
||||||
|
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||||
|
// a chrome:// URL or the Web Store). `attachedTabs` does not yet
|
||||||
|
// hold `tabId`, so without rolling back the debugger would stay
|
||||||
|
// attached and the next `ensureAttached` would re-attach and hit
|
||||||
|
// "Another debugger is already attached" forever — the tab is
|
||||||
|
// unusable until the extension is reloaded. Detach directly
|
||||||
|
// (`this.detach()` is a no-op here because `attachedTabs` lacks
|
||||||
|
// the id) so the next attempt starts from a clean slate.
|
||||||
|
await this.api.detach({ tabId }).catch((detachErr) => {
|
||||||
|
console.debug("[bsk cdp] rollback detach failed", { tabId, detachErr });
|
||||||
|
});
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
})()
|
})()
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
// Chrome surfaces "Another debugger is already attached" when
|
// Chrome surfaces "Another debugger is already attached" when
|
||||||
@@ -189,11 +254,105 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||||
|
if (!this.attachedTabs.has(target.tabId)) {
|
||||||
|
await this.ensureAttached(target.tabId);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||||
|
} catch (err) {
|
||||||
|
throw normalizeError(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||||
|
await this.ensureAttached(tabId);
|
||||||
|
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||||
|
await this.drainFrameAttachTasks(tabId);
|
||||||
|
|
||||||
|
const sources: CdpFrameTreeSource[] = [];
|
||||||
|
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
{ tabId },
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||||
|
|
||||||
|
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||||
|
const childTrees = await Promise.all(
|
||||||
|
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||||
|
const target = { tabId, sessionId };
|
||||||
|
try {
|
||||||
|
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||||
|
target,
|
||||||
|
"Page.getFrameTree",
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
for (const source of childTrees) {
|
||||||
|
if (source) sources.push(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
const graph = buildFrameGraph(sources);
|
||||||
|
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||||
|
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
await Promise.all(
|
||||||
|
graph.frames.map(async (frame) => {
|
||||||
|
if (!frame.parentFrameId) return;
|
||||||
|
const parent = frameById.get(frame.parentFrameId);
|
||||||
|
if (!parent) return;
|
||||||
|
try {
|
||||||
|
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||||
|
parent.target,
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: frame.frameId },
|
||||||
|
);
|
||||||
|
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||||
|
} catch {
|
||||||
|
// The frame may have navigated between tree capture and owner lookup.
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return graph;
|
||||||
|
}
|
||||||
|
|
||||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||||
dialogCursor(tabId: number): DialogCursor {
|
dialogCursor(tabId: number): DialogCursor {
|
||||||
return this.dialogSequences.get(tabId) ?? 0;
|
return this.dialogSequences.get(tabId) ?? 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** `Emulation.setDeviceMetricsOverride` — pin the tab's viewport metrics. */
|
||||||
|
async setDeviceMetricsOverride(tabId: number, metrics: DeviceMetricsOverride): Promise<void> {
|
||||||
|
await this.send(tabId, "Emulation.setDeviceMetricsOverride", { ...metrics });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `Emulation.clearDeviceMetricsOverride` — restore native viewport metrics. */
|
||||||
|
async clearDeviceMetricsOverride(tabId: number): Promise<void> {
|
||||||
|
await this.send(tabId, "Emulation.clearDeviceMetricsOverride", {});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `Emulation.setUserAgentOverride` — an empty `userAgent` restores the real UA. */
|
||||||
|
async setUserAgentOverride(tabId: number, override: UserAgentOverride): Promise<void> {
|
||||||
|
await this.send(tabId, "Emulation.setUserAgentOverride", { ...override });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `Emulation.setTouchEmulationEnabled`. */
|
||||||
|
async setTouchEmulationEnabled(
|
||||||
|
tabId: number,
|
||||||
|
enabled: boolean,
|
||||||
|
maxTouchPoints?: number,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.send(tabId, "Emulation.setTouchEmulationEnabled", {
|
||||||
|
enabled,
|
||||||
|
...(maxTouchPoints !== undefined ? { maxTouchPoints } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/** Dialogs observed on `tabId` with sequence strictly greater than `cursor`. */
|
/** Dialogs observed on `tabId` with sequence strictly greater than `cursor`. */
|
||||||
dialogsSince(tabId: number, cursor: DialogCursor): JavaScriptDialogInfo[] {
|
dialogsSince(tabId: number, cursor: DialogCursor): JavaScriptDialogInfo[] {
|
||||||
const buf = this.dialogBuffers.get(tabId) ?? [];
|
const buf = this.dialogBuffers.get(tabId) ?? [];
|
||||||
@@ -272,6 +431,7 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(tabId);
|
this.clearDialogState(tabId);
|
||||||
this.clearConsoleState(tabId);
|
this.clearConsoleState(tabId);
|
||||||
this.clearNetworkState(tabId);
|
this.clearNetworkState(tabId);
|
||||||
|
this.clearFrameState(tabId);
|
||||||
try {
|
try {
|
||||||
await this.api.detach({ tabId });
|
await this.api.detach({ tabId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -294,7 +454,7 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
} {
|
} {
|
||||||
this.api.onEvent.addListener(handler);
|
this.api.onEvent.addListener(handler);
|
||||||
@@ -313,11 +473,12 @@ export class ChromiumCdp {
|
|||||||
this.dialogSequences.clear();
|
this.dialogSequences.clear();
|
||||||
this.consoleBuffers.clear();
|
this.consoleBuffers.clear();
|
||||||
this.consoleSequences.clear();
|
this.consoleSequences.clear();
|
||||||
this.consoleDomainsAttemptedTabs.clear();
|
this.consoleDomainsEnabledTabs.clear();
|
||||||
this.networkBuffers.clear();
|
this.networkBuffers.clear();
|
||||||
this.networkSequences.clear();
|
this.networkSequences.clear();
|
||||||
this.networkDomainsEnabledTabs.clear();
|
this.networkDomainsEnabledTabs.clear();
|
||||||
this.networkRequestMeta.clear();
|
this.networkRequestMeta.clear();
|
||||||
|
this.frameDiscovery.clear();
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
tabs.map(async (tabId) => {
|
tabs.map(async (tabId) => {
|
||||||
try {
|
try {
|
||||||
@@ -334,15 +495,24 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async enableConsoleDomains(tabId: number): Promise<void> {
|
private async enableConsoleDomains(tabId: number): Promise<void> {
|
||||||
if (this.consoleDomainsAttemptedTabs.has(tabId)) return;
|
if (this.consoleDomainsEnabledTabs.has(tabId)) return;
|
||||||
this.consoleDomainsAttemptedTabs.add(tabId);
|
// Mark the tab only after both domains enable successfully — a
|
||||||
|
// transient failure (e.g. restricted page during attach) must leave
|
||||||
|
// the tab unmarked so a later `ensureConsoleCapture` can retry,
|
||||||
|
// instead of silently returning no console output forever. Mirrors
|
||||||
|
// `enableNetworkDomain`, which records the tab after success only.
|
||||||
|
let failed = false;
|
||||||
for (const method of ["Runtime.enable", "Log.enable"]) {
|
for (const method of ["Runtime.enable", "Log.enable"]) {
|
||||||
try {
|
try {
|
||||||
await this.api.sendCommand({ tabId }, method, {});
|
await this.api.sendCommand({ tabId }, method, {});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
failed = true;
|
||||||
console.debug("[bsk cdp] console domain enable failed", { tabId, method, err });
|
console.debug("[bsk cdp] console domain enable failed", { tabId, method, err });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (!failed) {
|
||||||
|
this.consoleDomainsEnabledTabs.add(tabId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async enableNetworkDomain(tabId: number): Promise<void> {
|
private async enableNetworkDomain(tabId: number): Promise<void> {
|
||||||
@@ -359,9 +529,104 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||||
|
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||||
|
autoAttach: true,
|
||||||
|
waitForDebuggerOnStart: false,
|
||||||
|
flatten: true,
|
||||||
|
filter: [{ type: "iframe", exclude: false }],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private bindFrameTargetHandler(): void {
|
||||||
|
if (this.frameTargetSubscription) return;
|
||||||
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
|
const tabId = source.tabId;
|
||||||
|
if (typeof tabId !== "number") return;
|
||||||
|
const raw = (params ?? {}) as Record<string, unknown>;
|
||||||
|
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||||
|
if (!sessionId) return;
|
||||||
|
|
||||||
|
if (method === "Target.detachedFromTarget") {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (method !== "Target.attachedToTarget") return;
|
||||||
|
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||||
|
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||||
|
|
||||||
|
const state = this.frameDiscoveryState(tabId);
|
||||||
|
if (state.sessions.has(sessionId)) return;
|
||||||
|
state.sessions.add(sessionId);
|
||||||
|
state.generation += 1;
|
||||||
|
|
||||||
|
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||||
|
state.pending.add(task);
|
||||||
|
void task.finally(() => {
|
||||||
|
state.pending.delete(task);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
this.api.onEvent.addListener(listener);
|
||||||
|
this.frameTargetSubscription = {
|
||||||
|
dispose: () => this.api.onEvent.removeListener(listener),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.enableFrameDiscovery(target);
|
||||||
|
} catch (err) {
|
||||||
|
const state = this.frameDiscovery.get(target.tabId);
|
||||||
|
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||||
|
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||||
|
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const state = this.frameDiscovery.get(tabId);
|
||||||
|
const generation = state?.generation ?? 0;
|
||||||
|
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||||
|
|
||||||
|
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||||
|
// its command promise settles. Wait for a short quiet window, then finish
|
||||||
|
// only if neither the state object nor its generation changed.
|
||||||
|
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||||
|
if (quietTime <= 0) break;
|
||||||
|
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||||
|
const current = this.frameDiscovery.get(tabId);
|
||||||
|
if (
|
||||||
|
current === state &&
|
||||||
|
(current?.generation ?? 0) === generation &&
|
||||||
|
!current?.pending.size
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||||
|
}
|
||||||
|
|
||||||
|
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||||
|
const existing = this.frameDiscovery.get(tabId);
|
||||||
|
if (existing) return existing;
|
||||||
|
const created: FrameDiscoveryState = {
|
||||||
|
sessions: new Set(),
|
||||||
|
pending: new Set(),
|
||||||
|
generation: 0,
|
||||||
|
};
|
||||||
|
this.frameDiscovery.set(tabId, created);
|
||||||
|
return created;
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearFrameState(tabId: number): void {
|
||||||
|
this.frameDiscovery.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
private bindDialogHandler(): void {
|
private bindDialogHandler(): void {
|
||||||
if (this.dialogSubscription) return;
|
if (this.dialogSubscription) return;
|
||||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||||
if (method !== "Page.javascriptDialogOpening") return;
|
if (method !== "Page.javascriptDialogOpening") return;
|
||||||
const tabId = source.tabId;
|
const tabId = source.tabId;
|
||||||
if (typeof tabId !== "number") return;
|
if (typeof tabId !== "number") return;
|
||||||
@@ -445,7 +710,7 @@ export class ChromiumCdp {
|
|||||||
private clearConsoleState(tabId: number): void {
|
private clearConsoleState(tabId: number): void {
|
||||||
this.consoleBuffers.delete(tabId);
|
this.consoleBuffers.delete(tabId);
|
||||||
this.consoleSequences.delete(tabId);
|
this.consoleSequences.delete(tabId);
|
||||||
this.consoleDomainsAttemptedTabs.delete(tabId);
|
this.consoleDomainsEnabledTabs.delete(tabId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private bindNetworkHandler(): void {
|
private bindNetworkHandler(): void {
|
||||||
@@ -545,6 +810,7 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(source.tabId);
|
this.clearDialogState(source.tabId);
|
||||||
this.clearConsoleState(source.tabId);
|
this.clearConsoleState(source.tabId);
|
||||||
this.clearNetworkState(source.tabId);
|
this.clearNetworkState(source.tabId);
|
||||||
|
this.clearFrameState(source.tabId);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
this.api.onDetach.addListener(listener);
|
this.api.onDetach.addListener(listener);
|
||||||
@@ -563,6 +829,8 @@ export class ChromiumCdp {
|
|||||||
this.consoleSubscription = null;
|
this.consoleSubscription = null;
|
||||||
this.networkSubscription?.dispose();
|
this.networkSubscription?.dispose();
|
||||||
this.networkSubscription = null;
|
this.networkSubscription = null;
|
||||||
|
this.frameTargetSubscription?.dispose();
|
||||||
|
this.frameTargetSubscription = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Detach tabs only when no other live session has claimed them. */
|
/** Detach tabs only when no other live session has claimed them. */
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
export interface CdpTarget {
|
||||||
|
tabId: number;
|
||||||
|
sessionId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrame {
|
||||||
|
frameId: string;
|
||||||
|
parentFrameId?: string;
|
||||||
|
ownerBackendNodeId?: number;
|
||||||
|
url?: string;
|
||||||
|
target: CdpTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameGraph {
|
||||||
|
rootFrameId: string;
|
||||||
|
frames: CdpFrame[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeNode {
|
||||||
|
frame: {
|
||||||
|
id: string;
|
||||||
|
parentId?: string;
|
||||||
|
url?: string;
|
||||||
|
};
|
||||||
|
childFrames?: CdpFrameTreeNode[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CdpFrameTreeSource {
|
||||||
|
target: CdpTarget;
|
||||||
|
tree: CdpFrameTreeNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cdpTargetKey(target: CdpTarget): string {
|
||||||
|
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeFrame(
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
node: CdpFrameTreeNode,
|
||||||
|
target: CdpTarget,
|
||||||
|
): void {
|
||||||
|
const existing = frames.get(node.frame.id);
|
||||||
|
if (!existing) order.push(node.frame.id);
|
||||||
|
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||||
|
const url = node.frame.url || existing?.url;
|
||||||
|
frames.set(node.frame.id, {
|
||||||
|
frameId: node.frame.id,
|
||||||
|
...(parentFrameId ? { parentFrameId } : {}),
|
||||||
|
...(url ? { url } : {}),
|
||||||
|
target,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkFrameTree(
|
||||||
|
source: CdpFrameTreeSource,
|
||||||
|
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||||
|
frames: Map<string, CdpFrame>,
|
||||||
|
order: string[],
|
||||||
|
): void {
|
||||||
|
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||||
|
{ node: source.tree, inheritedTarget: source.target },
|
||||||
|
];
|
||||||
|
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||||
|
|
||||||
|
while (stack.length > 0) {
|
||||||
|
const current = stack.pop();
|
||||||
|
if (!current || expanded.has(current.node)) continue;
|
||||||
|
expanded.add(current.node);
|
||||||
|
|
||||||
|
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||||
|
mergeFrame(frames, order, current.node, target);
|
||||||
|
|
||||||
|
const children = current.node.childFrames ?? [];
|
||||||
|
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||||
|
stack.push({ node: children[index], inheritedTarget: target });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||||
|
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||||
|
if (!rootSource) return null;
|
||||||
|
|
||||||
|
const frames = new Map<string, CdpFrame>();
|
||||||
|
const order: string[] = [];
|
||||||
|
const targetByRootFrameId = new Map(
|
||||||
|
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||||
|
);
|
||||||
|
// The top-level source is authoritative if malformed input reports the same
|
||||||
|
// frame as both a root target and a child target.
|
||||||
|
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||||
|
|
||||||
|
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||||
|
for (const source of sources) {
|
||||||
|
if (source === rootSource) continue;
|
||||||
|
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
rootFrameId: rootSource.tree.frame.id,
|
||||||
|
frames: order.flatMap((frameId) => {
|
||||||
|
const frame = frames.get(frameId);
|
||||||
|
return frame ? [frame] : [];
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
import { RiStopCircleLine } from "@remixicon/react";
|
import { RiStopCircleLine } from "@remixicon/react";
|
||||||
import { useEffect, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import logoUrl from "../../assets/logo.png";
|
import logoUrl from "../../assets/logo.png";
|
||||||
|
|
||||||
export interface ControlOverlayProps {
|
export interface ControlOverlayProps {
|
||||||
@@ -18,6 +18,7 @@ export function ControlOverlay({
|
|||||||
}: ControlOverlayProps) {
|
}: ControlOverlayProps) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const [show, setShow] = useState(false);
|
const [show, setShow] = useState(false);
|
||||||
|
const blockerRef = useRef<HTMLDivElement>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (visible) {
|
if (visible) {
|
||||||
@@ -27,6 +28,36 @@ export function ControlOverlay({
|
|||||||
setShow(false);
|
setShow(false);
|
||||||
}, [visible]);
|
}, [visible]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const blocker = blockerRef.current;
|
||||||
|
if (!blocker) return;
|
||||||
|
const stopScroll = (event: WheelEvent | TouchEvent) => {
|
||||||
|
if (automationBypass) return;
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
};
|
||||||
|
blocker.addEventListener("wheel", stopScroll, { passive: false });
|
||||||
|
blocker.addEventListener("touchmove", stopScroll, { passive: false });
|
||||||
|
return () => {
|
||||||
|
blocker.removeEventListener("wheel", stopScroll);
|
||||||
|
blocker.removeEventListener("touchmove", stopScroll);
|
||||||
|
};
|
||||||
|
}, [automationBypass]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!visible || automationBypass) return;
|
||||||
|
const stopScroll = (event: WheelEvent | TouchEvent) => {
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
};
|
||||||
|
window.addEventListener("wheel", stopScroll, { capture: true, passive: false });
|
||||||
|
window.addEventListener("touchmove", stopScroll, { capture: true, passive: false });
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener("wheel", stopScroll, { capture: true });
|
||||||
|
window.removeEventListener("touchmove", stopScroll, { capture: true });
|
||||||
|
};
|
||||||
|
}, [visible, automationBypass]);
|
||||||
|
|
||||||
if (!visible) return null;
|
if (!visible) return null;
|
||||||
|
|
||||||
const pointerEvents = automationBypass ? "none" : "auto";
|
const pointerEvents = automationBypass ? "none" : "auto";
|
||||||
@@ -58,6 +89,7 @@ export function ControlOverlay({
|
|||||||
/>
|
/>
|
||||||
|
|
||||||
<div
|
<div
|
||||||
|
ref={blockerRef}
|
||||||
data-slot="control-overlay-blocker"
|
data-slot="control-overlay-blocker"
|
||||||
onPointerDown={(event) => {
|
onPointerDown={(event) => {
|
||||||
if (automationBypass) return;
|
if (automationBypass) return;
|
||||||
@@ -88,8 +120,6 @@ export function ControlOverlay({
|
|||||||
left: "50%",
|
left: "50%",
|
||||||
transform: "translateX(-50%)",
|
transform: "translateX(-50%)",
|
||||||
zIndex: 2147483647,
|
zIndex: 2147483647,
|
||||||
// Always receive clicks so Interrupt works even while the page
|
|
||||||
// blocker is in automation-bypass mode (CDP pass-through).
|
|
||||||
pointerEvents: "auto",
|
pointerEvents: "auto",
|
||||||
display: "flex",
|
display: "flex",
|
||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
|
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||||
import logoUrl from "../../assets/logo.png";
|
import logoUrl from "../../assets/logo.png";
|
||||||
|
|
||||||
export interface HelpRequestData {
|
export interface HelpRequestData {
|
||||||
@@ -15,8 +16,14 @@ export interface HelpRequestData {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
/** Custom overlay title; falls back to i18n when omitted. */
|
/** Custom overlay title; falls back to i18n when omitted. */
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||||
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight. */
|
/** CSS selectors to scroll to + flash-highlight. */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
|
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||||
|
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||||
onContinue: (note: string) => void;
|
onContinue: (note: string) => void;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
}
|
}
|
||||||
@@ -46,6 +53,8 @@ const VIEWPORT_MARGIN = 16;
|
|||||||
const PANEL_WIDTH = 420;
|
const PANEL_WIDTH = 420;
|
||||||
const FALLBACK_PANEL_H = 180;
|
const FALLBACK_PANEL_H = 180;
|
||||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||||
|
const EMPTY_SELECTORS: string[] = [];
|
||||||
|
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||||
|
|
||||||
function clamp(value: number, min: number, max: number): number {
|
function clamp(value: number, min: number, max: number): number {
|
||||||
return Math.max(min, Math.min(max, value));
|
return Math.max(min, Math.min(max, value));
|
||||||
@@ -174,6 +183,10 @@ function measure(selectors: string[]): Box[] {
|
|||||||
|
|
||||||
export function HelpRequestOverlay({ request }: Props) {
|
export function HelpRequestOverlay({ request }: Props) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
|
const isCompact = request?.displayMode === "compact";
|
||||||
|
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||||
|
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||||
|
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||||
const [note, setNote] = useState("");
|
const [note, setNote] = useState("");
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||||
@@ -196,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
// Reset the note and collapse state whenever a new request appears.
|
// Reset the note and collapse state whenever a new request appears.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setNote("");
|
setNote("");
|
||||||
|
setLiveRects(null);
|
||||||
setCollapsed(false);
|
setCollapsed(false);
|
||||||
placementRef.current = null;
|
placementRef.current = null;
|
||||||
setDragPos(null);
|
setDragPos(null);
|
||||||
@@ -204,11 +218,38 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
dragStartRef.current = null;
|
dragStartRef.current = null;
|
||||||
}, [request?.id]);
|
}, [request?.id]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!request || isCompact || !request.refreshRects) return;
|
||||||
|
let disposed = false;
|
||||||
|
let resolving = false;
|
||||||
|
const refresh = async () => {
|
||||||
|
if (resolving) return;
|
||||||
|
resolving = true;
|
||||||
|
try {
|
||||||
|
const rects = await request.refreshRects?.();
|
||||||
|
if (!disposed && rects) setLiveRects(rects);
|
||||||
|
} finally {
|
||||||
|
resolving = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onViewportChange = () => void refresh();
|
||||||
|
window.addEventListener("scroll", onViewportChange, true);
|
||||||
|
window.addEventListener("resize", onViewportChange);
|
||||||
|
const interval = window.setInterval(onViewportChange, 500);
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
window.removeEventListener("scroll", onViewportChange, true);
|
||||||
|
window.removeEventListener("resize", onViewportChange);
|
||||||
|
window.clearInterval(interval);
|
||||||
|
};
|
||||||
|
}, [request, isCompact]);
|
||||||
|
|
||||||
// Scroll the first matched target into view once per distinct request id.
|
// Scroll the first matched target into view once per distinct request id.
|
||||||
useLayoutEffect(() => {
|
useLayoutEffect(() => {
|
||||||
if (!request) return;
|
if (!request) return;
|
||||||
|
if (isCompact) return;
|
||||||
if (lastScrolledIdRef.current === request.id) return;
|
if (lastScrolledIdRef.current === request.id) return;
|
||||||
for (const sel of request.selectors) {
|
for (const sel of effectiveSelectors) {
|
||||||
const el = querySelectorSafe(sel);
|
const el = querySelectorSafe(sel);
|
||||||
if (el) {
|
if (el) {
|
||||||
lastScrolledIdRef.current = request.id;
|
lastScrolledIdRef.current = request.id;
|
||||||
@@ -216,16 +257,16 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [request]);
|
}, [request, isCompact, effectiveSelectors]);
|
||||||
|
|
||||||
// Keep highlight boxes aligned with the page as it scrolls / resizes.
|
// Keep highlight boxes aligned with the page as it scrolls / resizes.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!request) {
|
if (!request) {
|
||||||
setBoxes([]);
|
setBoxes((prev) => (prev.length === 0 ? prev : []));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const update = () => {
|
const update = () => {
|
||||||
const nextBoxes = measure(request.selectors);
|
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||||
};
|
};
|
||||||
update();
|
update();
|
||||||
@@ -237,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
window.removeEventListener("resize", update);
|
window.removeEventListener("resize", update);
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [request]);
|
}, [request, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
const onHeaderPointerDown = useCallback(
|
const onHeaderPointerDown = useCallback(
|
||||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
@@ -294,8 +335,12 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
setPanelPos(null);
|
setPanelPos(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (isCompact) {
|
||||||
|
setPanelPos(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (userMovedRef.current) return;
|
if (userMovedRef.current) return;
|
||||||
const measured = boxes.length > 0 ? boxes : measure(request.selectors);
|
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||||
const anchor = unionBox(measured);
|
const anchor = unionBox(measured);
|
||||||
const banner = bannerRef.current;
|
const banner = bannerRef.current;
|
||||||
if (!anchor || !banner) {
|
if (!anchor || !banner) {
|
||||||
@@ -319,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||||
}, [request, boxes, collapsed]);
|
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
@@ -342,6 +387,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
z-index: 2147483647;
|
z-index: 2147483647;
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
|
justify-content: flex-start;
|
||||||
|
min-height: 0;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
max-width: calc(100vw - ${VIEWPORT_MARGIN * 2}px);
|
||||||
@@ -359,16 +406,25 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-banner[data-collapsed="true"] {
|
.bsk-help-banner[data-collapsed="true"] {
|
||||||
gap: 0;
|
gap: 0;
|
||||||
width: var(--bsk-help-banner-width);
|
width: var(--bsk-help-banner-width);
|
||||||
padding: 10px 12px;
|
padding: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] {
|
||||||
|
width: min(360px, calc(100vw - ${VIEWPORT_MARGIN * 2}px));
|
||||||
|
padding: 12px 14px;
|
||||||
|
gap: 0;
|
||||||
|
border-radius: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-drag-strip {
|
.bsk-help-drag-strip {
|
||||||
flex-shrink: 0;
|
position: absolute;
|
||||||
|
top: 6px;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
height: 12px;
|
height: 12px;
|
||||||
margin: -8px 0 -10px;
|
|
||||||
cursor: grab;
|
cursor: grab;
|
||||||
user-select: none;
|
user-select: none;
|
||||||
touch-action: none;
|
touch-action: none;
|
||||||
@@ -395,15 +451,20 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-drag-strip {
|
||||||
margin: -6px 0 -8px;
|
top: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header {
|
.bsk-help-header {
|
||||||
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header {
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
.bsk-help-title {
|
.bsk-help-title {
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
@@ -414,31 +475,34 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
color: #111;
|
color: #111;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.bsk-help-compact-title {
|
||||||
|
flex: 1 1 auto;
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 500;
|
||||||
|
color: #374151;
|
||||||
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-title {
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions {
|
.bsk-help-header-actions {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
display: flex;
|
display: none;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
margin-left: auto;
|
margin-left: auto;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
max-width: 0;
|
|
||||||
opacity: 0;
|
|
||||||
transform: translateX(8px);
|
|
||||||
pointer-events: none;
|
|
||||||
transition:
|
|
||||||
max-width var(--bsk-help-duration) var(--bsk-help-ease),
|
|
||||||
opacity 220ms var(--bsk-help-ease),
|
|
||||||
transform var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-header-actions {
|
||||||
|
display: flex;
|
||||||
max-width: 280px;
|
max-width: 280px;
|
||||||
opacity: 1;
|
opacity: 1;
|
||||||
transform: translateX(0);
|
|
||||||
pointer-events: auto;
|
pointer-events: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -471,15 +535,13 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-body {
|
.bsk-help-body {
|
||||||
display: grid;
|
display: block;
|
||||||
grid-template-rows: 1fr;
|
flex-shrink: 0;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
transition: grid-template-rows var(--bsk-help-duration) var(--bsk-help-ease);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
.bsk-help-banner[data-collapsed="true"] .bsk-help-body {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
grid-template-rows: 0fr;
|
|
||||||
width: 0;
|
width: 0;
|
||||||
height: 0;
|
height: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -528,6 +590,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
.bsk-help-footer-actions {
|
.bsk-help-footer-actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
|
flex-wrap: wrap;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,7 +602,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
color: #4b5563;
|
color: #4b5563;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-btn-continue {
|
.bsk-help-btn-continue {
|
||||||
@@ -550,16 +614,30 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
flex-shrink: 0;
|
min-width: 0;
|
||||||
|
white-space: normal;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-cancel,
|
.bsk-help-header-actions .bsk-help-btn-cancel,
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 12px;
|
padding: 6px 12px;
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-header-actions .bsk-help-btn-continue {
|
.bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
padding: 6px 14px;
|
padding: 6px 14px;
|
||||||
|
max-width: 180px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions {
|
||||||
|
display: flex;
|
||||||
|
max-width: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-cancel,
|
||||||
|
.bsk-help-banner[data-display-mode="compact"] .bsk-help-header-actions .bsk-help-btn-continue {
|
||||||
|
padding: 6px 10px;
|
||||||
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
.bsk-help-footer-actions .bsk-help-btn-cancel {
|
||||||
@@ -584,29 +662,31 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
`}</style>
|
`}</style>
|
||||||
|
|
||||||
{boxes.map((b, i) => (
|
{!isCompact &&
|
||||||
<div
|
boxes.map((b, i) => (
|
||||||
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
<div
|
||||||
key={i}
|
// biome-ignore lint/suspicious/noArrayIndexKey: boxes are positional
|
||||||
data-slot="help-highlight"
|
key={i}
|
||||||
style={{
|
data-slot="help-highlight"
|
||||||
position: "fixed",
|
style={{
|
||||||
top: b.top,
|
position: "fixed",
|
||||||
left: b.left,
|
top: b.top,
|
||||||
width: b.width,
|
left: b.left,
|
||||||
height: b.height,
|
width: b.width,
|
||||||
borderRadius: 6,
|
height: b.height,
|
||||||
zIndex: 2147483646,
|
borderRadius: 6,
|
||||||
pointerEvents: "none",
|
zIndex: 2147483646,
|
||||||
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
pointerEvents: "none",
|
||||||
}}
|
animation: "bsk-help-flash 1.2s ease-in-out infinite",
|
||||||
/>
|
}}
|
||||||
))}
|
/>
|
||||||
|
))}
|
||||||
|
|
||||||
<div
|
<div
|
||||||
ref={bannerRef}
|
ref={bannerRef}
|
||||||
data-slot="help-request-banner"
|
data-slot="help-request-banner"
|
||||||
data-collapsed={collapsed ? "true" : "false"}
|
data-collapsed={collapsed ? "true" : "false"}
|
||||||
|
data-display-mode={isCompact ? "compact" : "full"}
|
||||||
data-anchored={panelPos ? "true" : "false"}
|
data-anchored={panelPos ? "true" : "false"}
|
||||||
data-dragging={dragging ? "true" : "false"}
|
data-dragging={dragging ? "true" : "false"}
|
||||||
data-placement={panelPos?.placement}
|
data-placement={panelPos?.placement}
|
||||||
@@ -629,15 +709,17 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
: undefined
|
: undefined
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<div
|
{!isCompact && (
|
||||||
data-slot="help-drag-handle"
|
<div
|
||||||
className="bsk-help-drag-strip"
|
data-slot="help-drag-handle"
|
||||||
role="img"
|
className="bsk-help-drag-strip"
|
||||||
aria-label={t("helpRequest.dragHandle")}
|
role="img"
|
||||||
onPointerDown={onHeaderPointerDown}
|
aria-label={t("helpRequest.dragHandle")}
|
||||||
>
|
onPointerDown={onHeaderPointerDown}
|
||||||
<span className="bsk-help-drag-pill" aria-hidden />
|
>
|
||||||
</div>
|
<span className="bsk-help-drag-pill" aria-hidden />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="bsk-help-header">
|
<div className="bsk-help-header">
|
||||||
<img
|
<img
|
||||||
@@ -645,77 +727,87 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
alt="browser-skill"
|
alt="browser-skill"
|
||||||
style={{ width: 22, height: 22, borderRadius: 4 }}
|
style={{ width: 22, height: 22, borderRadius: 4 }}
|
||||||
/>
|
/>
|
||||||
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
{isCompact ? (
|
||||||
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
<span className="bsk-help-compact-title">{t("helpRequest.compactStatus")}</span>
|
||||||
|
) : (
|
||||||
|
<span className="bsk-help-title">{request.title ?? t("helpRequest.title")}</span>
|
||||||
|
)}
|
||||||
|
{!isCompact && (
|
||||||
|
<div className="bsk-help-header-actions" aria-hidden={!collapsed}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? "help-cancel-button" : undefined}
|
||||||
|
className="bsk-help-btn-cancel"
|
||||||
|
tabIndex={collapsed ? 0 : -1}
|
||||||
|
onClick={() => request.onCancel()}
|
||||||
|
>
|
||||||
|
{t("helpRequest.cancel")}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? "help-continue-button" : undefined}
|
||||||
|
className="bsk-help-btn-continue"
|
||||||
|
tabIndex={collapsed ? 0 : -1}
|
||||||
|
onClick={() => request.onContinue(note)}
|
||||||
|
>
|
||||||
|
{t("helpRequest.continue")}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{!isCompact && (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
data-slot={collapsed ? "help-cancel-button" : undefined}
|
data-slot="help-collapse-toggle"
|
||||||
className="bsk-help-btn-cancel"
|
className="bsk-help-collapse-toggle"
|
||||||
tabIndex={collapsed ? 0 : -1}
|
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
||||||
onClick={() => request.onCancel()}
|
aria-expanded={!collapsed}
|
||||||
|
onClick={() => setCollapsed((c) => !c)}
|
||||||
>
|
>
|
||||||
{t("helpRequest.cancel")}
|
<span className="bsk-help-collapse-icon">
|
||||||
|
<RiArrowDownSLine size={20} aria-hidden />
|
||||||
|
</span>
|
||||||
</button>
|
</button>
|
||||||
<button
|
)}
|
||||||
type="button"
|
|
||||||
data-slot={collapsed ? "help-continue-button" : undefined}
|
|
||||||
className="bsk-help-btn-continue"
|
|
||||||
tabIndex={collapsed ? 0 : -1}
|
|
||||||
onClick={() => request.onContinue(note)}
|
|
||||||
>
|
|
||||||
{t("helpRequest.continue")}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
data-slot="help-collapse-toggle"
|
|
||||||
className="bsk-help-collapse-toggle"
|
|
||||||
aria-label={collapsed ? t("helpRequest.expand") : t("helpRequest.collapse")}
|
|
||||||
aria-expanded={!collapsed}
|
|
||||||
onClick={() => setCollapsed((c) => !c)}
|
|
||||||
>
|
|
||||||
<span className="bsk-help-collapse-icon">
|
|
||||||
<RiArrowDownSLine size={20} aria-hidden />
|
|
||||||
</span>
|
|
||||||
</button>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="bsk-help-body" aria-hidden={collapsed}>
|
{!isCompact && (
|
||||||
<div className="bsk-help-body-inner">
|
<div className="bsk-help-body" aria-hidden={collapsed}>
|
||||||
<div className="bsk-help-body-content">
|
<div className="bsk-help-body-inner">
|
||||||
<p className="bsk-help-prompt">{request.prompt}</p>
|
<div className="bsk-help-body-content">
|
||||||
<input
|
<p className="bsk-help-prompt">{request.prompt}</p>
|
||||||
type="text"
|
<input
|
||||||
className="bsk-help-note-input"
|
type="text"
|
||||||
aria-label={t("helpRequest.noteLabel")}
|
className="bsk-help-note-input"
|
||||||
placeholder={t("helpRequest.notePlaceholder")}
|
aria-label={t("helpRequest.noteLabel")}
|
||||||
value={note}
|
placeholder={t("helpRequest.notePlaceholder")}
|
||||||
tabIndex={collapsed ? -1 : 0}
|
value={note}
|
||||||
onChange={(e) => setNote(e.target.value)}
|
|
||||||
/>
|
|
||||||
<div className="bsk-help-footer-actions">
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
data-slot={collapsed ? undefined : "help-cancel-button"}
|
|
||||||
className="bsk-help-btn-cancel"
|
|
||||||
tabIndex={collapsed ? -1 : 0}
|
tabIndex={collapsed ? -1 : 0}
|
||||||
onClick={() => request.onCancel()}
|
onChange={(e) => setNote(e.target.value)}
|
||||||
>
|
/>
|
||||||
{t("helpRequest.cancel")}
|
<div className="bsk-help-footer-actions">
|
||||||
</button>
|
<button
|
||||||
<button
|
type="button"
|
||||||
type="button"
|
data-slot={collapsed ? undefined : "help-cancel-button"}
|
||||||
data-slot={collapsed ? undefined : "help-continue-button"}
|
className="bsk-help-btn-cancel"
|
||||||
className="bsk-help-btn-continue"
|
tabIndex={collapsed ? -1 : 0}
|
||||||
tabIndex={collapsed ? -1 : 0}
|
onClick={() => request.onCancel()}
|
||||||
onClick={() => request.onContinue(note)}
|
>
|
||||||
>
|
{t("helpRequest.cancel")}
|
||||||
{t("helpRequest.continue")}
|
</button>
|
||||||
</button>
|
<button
|
||||||
|
type="button"
|
||||||
|
data-slot={collapsed ? undefined : "help-continue-button"}
|
||||||
|
className="bsk-help-btn-continue"
|
||||||
|
tabIndex={collapsed ? -1 : 0}
|
||||||
|
onClick={() => request.onContinue(note)}
|
||||||
|
>
|
||||||
|
{t("helpRequest.continue")}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
)}
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,9 +1,21 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
import { RiStopCircleLine } from "@remixicon/react";
|
import {
|
||||||
import { useEffect, useState } from "react";
|
type PointerEvent as ReactPointerEvent,
|
||||||
|
useCallback,
|
||||||
|
useEffect,
|
||||||
|
useRef,
|
||||||
|
useState,
|
||||||
|
} from "react";
|
||||||
|
import recordIconUrl from "../../assets/record-icon.png";
|
||||||
|
|
||||||
export interface RecordRequestData {
|
export interface RecordRequestData {
|
||||||
id: string;
|
id: string;
|
||||||
|
/**
|
||||||
|
* Epoch ms when the recording began. Supplied by the background so the timer
|
||||||
|
* keeps counting the whole session across navigations; falls back to mount
|
||||||
|
* time when the background could not report it.
|
||||||
|
*/
|
||||||
|
startedAtMs?: number;
|
||||||
onFinish: () => void;
|
onFinish: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -11,9 +23,51 @@ type Props = {
|
|||||||
request: RecordRequestData | null;
|
request: RecordRequestData | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const VIEWPORT_MARGIN = 16;
|
||||||
|
const DRAG_THRESHOLD_PX = 8;
|
||||||
|
const INTRO_DURATION_MS = 10_000;
|
||||||
|
const TIMER_REFRESH_MS = 250;
|
||||||
|
|
||||||
|
function formatElapsed(seconds: number): string {
|
||||||
|
const minutes = Math.floor(seconds / 60);
|
||||||
|
const remainingSeconds = seconds % 60;
|
||||||
|
return `${String(minutes).padStart(2, "0")}:${String(remainingSeconds).padStart(2, "0")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clamp(value: number, min: number, max: number): number {
|
||||||
|
return Math.max(min, Math.min(max, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampDragPos(
|
||||||
|
top: number,
|
||||||
|
left: number,
|
||||||
|
panelW: number,
|
||||||
|
panelH: number,
|
||||||
|
): { top: number; left: number } {
|
||||||
|
const maxLeft = window.innerWidth - panelW - VIEWPORT_MARGIN;
|
||||||
|
const maxTop = window.innerHeight - panelH - VIEWPORT_MARGIN;
|
||||||
|
return {
|
||||||
|
top: clamp(top, VIEWPORT_MARGIN, Math.max(VIEWPORT_MARGIN, maxTop)),
|
||||||
|
left: clamp(left, VIEWPORT_MARGIN, Math.max(VIEWPORT_MARGIN, maxLeft)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function RecordOverlay({ request }: Props) {
|
export function RecordOverlay({ request }: Props) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const [show, setShow] = useState(false);
|
const [show, setShow] = useState(false);
|
||||||
|
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||||
|
const [dragPos, setDragPos] = useState<{ top: number; left: number } | null>(null);
|
||||||
|
const [dragging, setDragging] = useState(false);
|
||||||
|
const pillRef = useRef<HTMLDivElement>(null);
|
||||||
|
const dragStartRef = useRef<{
|
||||||
|
pointerX: number;
|
||||||
|
pointerY: number;
|
||||||
|
originLeft: number;
|
||||||
|
originTop: number;
|
||||||
|
panelW: number;
|
||||||
|
panelH: number;
|
||||||
|
armed: boolean;
|
||||||
|
} | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (request) {
|
if (request) {
|
||||||
@@ -23,8 +77,87 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
setShow(false);
|
setShow(false);
|
||||||
}, [request]);
|
}, [request]);
|
||||||
|
|
||||||
|
const requestStartedAtMs = request?.startedAtMs;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setDragPos(null);
|
||||||
|
setDragging(false);
|
||||||
|
dragStartRef.current = null;
|
||||||
|
if (!request) {
|
||||||
|
setElapsedSeconds(0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const startedAt = requestStartedAtMs ?? Date.now();
|
||||||
|
const updateElapsed = () => {
|
||||||
|
setElapsedSeconds(Math.max(0, Math.floor((Date.now() - startedAt) / 1_000)));
|
||||||
|
};
|
||||||
|
updateElapsed();
|
||||||
|
const timer = window.setInterval(updateElapsed, TIMER_REFRESH_MS);
|
||||||
|
return () => window.clearInterval(timer);
|
||||||
|
}, [request?.id, requestStartedAtMs]);
|
||||||
|
|
||||||
|
const onPillPointerDown = useCallback((e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
|
if (e.button !== 0) return;
|
||||||
|
if (e.target instanceof Element && e.target.closest("button, a, input, textarea, select")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const pill = pillRef.current;
|
||||||
|
if (!pill) return;
|
||||||
|
const rect = pill.getBoundingClientRect();
|
||||||
|
dragStartRef.current = {
|
||||||
|
pointerX: e.clientX,
|
||||||
|
pointerY: e.clientY,
|
||||||
|
originLeft: rect.left,
|
||||||
|
originTop: rect.top,
|
||||||
|
panelW: pill.offsetWidth || rect.width,
|
||||||
|
panelH: pill.offsetHeight || rect.height,
|
||||||
|
armed: false,
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const onMove = (e: PointerEvent) => {
|
||||||
|
const start = dragStartRef.current;
|
||||||
|
if (!start) return;
|
||||||
|
const dx = e.clientX - start.pointerX;
|
||||||
|
const dy = e.clientY - start.pointerY;
|
||||||
|
if (!start.armed) {
|
||||||
|
if (Math.hypot(dx, dy) < DRAG_THRESHOLD_PX) return;
|
||||||
|
start.armed = true;
|
||||||
|
setDragging(true);
|
||||||
|
setDragPos({ top: start.originTop, left: start.originLeft });
|
||||||
|
}
|
||||||
|
setDragPos(
|
||||||
|
clampDragPos(start.originTop + dy, start.originLeft + dx, start.panelW, start.panelH),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
const onUp = () => {
|
||||||
|
const start = dragStartRef.current;
|
||||||
|
if (!start) return;
|
||||||
|
if (start.armed) {
|
||||||
|
setDragging(false);
|
||||||
|
}
|
||||||
|
dragStartRef.current = null;
|
||||||
|
};
|
||||||
|
window.addEventListener("pointermove", onMove);
|
||||||
|
window.addEventListener("pointerup", onUp);
|
||||||
|
window.addEventListener("pointercancel", onUp);
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener("pointermove", onMove);
|
||||||
|
window.removeEventListener("pointerup", onUp);
|
||||||
|
window.removeEventListener("pointercancel", onUp);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
|
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||||
|
const positioned = dragPos !== null;
|
||||||
|
const defaultTransform = show
|
||||||
|
? "translateX(-50%) translateY(0)"
|
||||||
|
: "translateX(-50%) translateY(8px)";
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<style>{`
|
<style>{`
|
||||||
@@ -35,78 +168,118 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
`}</style>
|
`}</style>
|
||||||
|
|
||||||
<div
|
<div
|
||||||
|
ref={pillRef}
|
||||||
data-slot="record-overlay-pill"
|
data-slot="record-overlay-pill"
|
||||||
|
data-phase={introVisible ? "intro" : "timer"}
|
||||||
|
data-dragging={dragging ? "true" : "false"}
|
||||||
|
onPointerDown={onPillPointerDown}
|
||||||
style={{
|
style={{
|
||||||
position: "fixed",
|
position: "fixed",
|
||||||
bottom: 32,
|
...(positioned
|
||||||
left: "50%",
|
? { top: dragPos.top, left: dragPos.left, bottom: "auto" }
|
||||||
|
: { bottom: 32, left: "50%" }),
|
||||||
zIndex: 2147483647,
|
zIndex: 2147483647,
|
||||||
pointerEvents: "auto",
|
pointerEvents: "auto",
|
||||||
display: "flex",
|
display: "flex",
|
||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
gap: 10,
|
gap: 5,
|
||||||
backgroundColor: "#fff",
|
backgroundColor: "#fff",
|
||||||
borderRadius: 9999,
|
borderRadius: 9999,
|
||||||
padding: "10px 10px 10px 20px",
|
height: 50,
|
||||||
boxShadow: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
boxSizing: "border-box",
|
||||||
|
padding: "6px 8px 6px 16px",
|
||||||
|
boxShadow: dragging
|
||||||
|
? "0 12px 40px rgba(15,23,42,0.22), 0 2px 8px rgba(0,0,0,0.12)"
|
||||||
|
: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||||
opacity: show ? 1 : 0,
|
opacity: show ? 1 : 0,
|
||||||
transform: show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)",
|
transform: positioned ? "none" : defaultTransform,
|
||||||
transition: "opacity 300ms ease-out, transform 300ms ease-out",
|
transition: dragging
|
||||||
|
? "box-shadow 150ms ease-out"
|
||||||
|
: "opacity 300ms ease-out, transform 300ms ease-out, box-shadow 150ms ease-out",
|
||||||
fontFamily:
|
fontFamily:
|
||||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||||
maxWidth: "min(420px, calc(100vw - 32px))",
|
maxWidth: "min(420px, calc(100vw - 32px))",
|
||||||
|
cursor: dragging ? "grabbing" : "grab",
|
||||||
|
userSelect: "none",
|
||||||
|
touchAction: "none",
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<span
|
<div
|
||||||
data-slot="record-overlay-indicator"
|
data-slot="record-overlay-status"
|
||||||
style={{
|
|
||||||
width: 9,
|
|
||||||
height: 9,
|
|
||||||
borderRadius: "50%",
|
|
||||||
backgroundColor: "#ef4444",
|
|
||||||
flexShrink: 0,
|
|
||||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
<span
|
|
||||||
style={{
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 8,
|
||||||
|
minWidth: 0,
|
||||||
flex: 1,
|
flex: 1,
|
||||||
fontSize: 16,
|
borderRadius: 9999,
|
||||||
fontWeight: 500,
|
outline: "none",
|
||||||
color: "#333",
|
minHeight: 32,
|
||||||
whiteSpace: "nowrap",
|
paddingRight: 4,
|
||||||
userSelect: "none",
|
|
||||||
overflow: "hidden",
|
|
||||||
textOverflow: "ellipsis",
|
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{t("recordOverlay.recording")}
|
<span
|
||||||
</span>
|
data-slot="record-overlay-indicator"
|
||||||
|
style={{
|
||||||
|
width: 9,
|
||||||
|
height: 9,
|
||||||
|
borderRadius: "50%",
|
||||||
|
backgroundColor: "#ef4444",
|
||||||
|
flexShrink: 0,
|
||||||
|
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span
|
||||||
|
data-slot="record-overlay-label"
|
||||||
|
style={{
|
||||||
|
display: "block",
|
||||||
|
flex: "0 1 auto",
|
||||||
|
fontSize: 16,
|
||||||
|
fontWeight: 500,
|
||||||
|
color: "#333",
|
||||||
|
whiteSpace: "nowrap",
|
||||||
|
userSelect: "none",
|
||||||
|
overflow: "hidden",
|
||||||
|
textOverflow: "ellipsis",
|
||||||
|
width: introVisible ? "auto" : 48,
|
||||||
|
maxWidth: introVisible ? 280 : 48,
|
||||||
|
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||||
|
textAlign: introVisible ? "left" : "center",
|
||||||
|
margin: 0,
|
||||||
|
padding: 0,
|
||||||
|
minWidth: 0,
|
||||||
|
transition: "max-width 180ms ease-out",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
data-slot="record-overlay-finish"
|
data-slot="record-overlay-finish"
|
||||||
|
aria-label={t("recordOverlay.finish")}
|
||||||
onClick={request.onFinish}
|
onClick={request.onFinish}
|
||||||
style={{
|
style={{
|
||||||
pointerEvents: "auto",
|
pointerEvents: "auto",
|
||||||
display: "flex",
|
display: "block",
|
||||||
alignItems: "center",
|
width: 32,
|
||||||
gap: 6,
|
height: 32,
|
||||||
border: "none",
|
border: "none",
|
||||||
borderRadius: 9999,
|
borderRadius: 9999,
|
||||||
padding: "8px 20px 8px 16px",
|
padding: 0,
|
||||||
fontSize: 15,
|
backgroundColor: "transparent",
|
||||||
fontWeight: 600,
|
|
||||||
color: "#fff",
|
|
||||||
backgroundColor: "#f97316",
|
|
||||||
cursor: "pointer",
|
cursor: "pointer",
|
||||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
|
||||||
whiteSpace: "nowrap",
|
|
||||||
lineHeight: 1,
|
|
||||||
flexShrink: 0,
|
flexShrink: 0,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<RiStopCircleLine size={18} color="#fff" />
|
<img
|
||||||
{t("recordOverlay.finish")}
|
src={recordIconUrl}
|
||||||
|
alt=""
|
||||||
|
width={24}
|
||||||
|
height={24}
|
||||||
|
draggable={false}
|
||||||
|
style={{ display: "block", margin: "auto" }}
|
||||||
|
/>
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { cleanup, render } from "@testing-library/react";
|
import { cleanup, render } from "@testing-library/react";
|
||||||
import { afterEach, describe, expect, it } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { ControlOverlay } from "../ControlOverlay";
|
import { ControlOverlay } from "../ControlOverlay";
|
||||||
|
|
||||||
describe("ControlOverlay", () => {
|
describe("ControlOverlay", () => {
|
||||||
@@ -44,4 +44,21 @@ describe("ControlOverlay", () => {
|
|||||||
expect(blocker).toBeTruthy();
|
expect(blocker).toBeTruthy();
|
||||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("calls onInterrupt from the stop button", () => {
|
||||||
|
const onInterrupt = vi.fn();
|
||||||
|
const { container } = render(
|
||||||
|
<ControlOverlay
|
||||||
|
visible={true}
|
||||||
|
interrupting={false}
|
||||||
|
automationBypass={false}
|
||||||
|
onInterrupt={onInterrupt}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||||
|
(stopBtn as HTMLButtonElement).click();
|
||||||
|
|
||||||
|
expect(onInterrupt).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||||
import { createElement } from "react";
|
import { createElement } from "react";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||||
|
|
||||||
function renderOverlay(req: HelpRequestData) {
|
function renderOverlay(req: HelpRequestData | null) {
|
||||||
return render(
|
return render(
|
||||||
createElement(I18nextProvider, { i18n }, createElement(HelpRequestOverlay, { request: req })),
|
createElement(I18nextProvider, { i18n }, createElement(HelpRequestOverlay, { request: req })),
|
||||||
);
|
);
|
||||||
@@ -38,6 +38,75 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||||
|
);
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("30px");
|
||||||
|
expect(highlight?.style.left).toBe("40px");
|
||||||
|
expect(highlight?.style.width).toBe("120px");
|
||||||
|
expect(highlight?.style.height).toBe("50px");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||||
|
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({
|
||||||
|
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||||
|
refreshRects,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
window.dispatchEvent(new Event("scroll"));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||||
|
expect(highlight?.style.top).toBe("80px");
|
||||||
|
expect(highlight?.style.left).toBe("90px");
|
||||||
|
});
|
||||||
|
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the inactive render stable", () => {
|
||||||
|
const { container, rerender } = renderOverlay(null);
|
||||||
|
|
||||||
|
rerender(
|
||||||
|
createElement(
|
||||||
|
I18nextProvider,
|
||||||
|
{ i18n },
|
||||||
|
createElement(HelpRequestOverlay, { request: null }),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(container.querySelector("[data-slot='help-request-banner']")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders a compact status without full request controls", () => {
|
||||||
|
const el = document.createElement("div");
|
||||||
|
el.id = "login";
|
||||||
|
el.getBoundingClientRect = () =>
|
||||||
|
({ top: 10, left: 10, width: 100, height: 40, right: 110, bottom: 50 }) as DOMRect;
|
||||||
|
document.body.append(el);
|
||||||
|
|
||||||
|
const { container } = renderOverlay(
|
||||||
|
baseRequest({
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: ["#login"],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText(i18n.t("helpRequest.compactStatus", { ns: "extension" }))).toBeTruthy();
|
||||||
|
expect(screen.queryByText("Please complete the captcha")).toBeNull();
|
||||||
|
expect(screen.queryByRole("textbox")).toBeNull();
|
||||||
|
expect(screen.queryByLabelText(i18n.t("helpRequest.collapse", { ns: "extension" }))).toBeNull();
|
||||||
|
expect(container.querySelector("[data-slot='help-continue-button']")).toBeNull();
|
||||||
|
expect(container.querySelector("[data-slot='help-cancel-button']")).toBeNull();
|
||||||
|
expect(document.querySelectorAll("[data-slot='help-highlight']").length).toBe(0);
|
||||||
|
|
||||||
|
el.remove();
|
||||||
|
});
|
||||||
|
|
||||||
it("renders a custom title when provided", () => {
|
it("renders a custom title when provided", () => {
|
||||||
renderOverlay(baseRequest({ title: "Verify your identity" }));
|
renderOverlay(baseRequest({ title: "Verify your identity" }));
|
||||||
expect(screen.getByText("Verify your identity")).toBeTruthy();
|
expect(screen.getByText("Verify your identity")).toBeTruthy();
|
||||||
@@ -204,6 +273,15 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(styles).toContain("width: 0");
|
expect(styles).toContain("width: 0");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses natural body height in the expanded layout", () => {
|
||||||
|
const { container } = renderOverlay(baseRequest());
|
||||||
|
const styles = overlayStyles(container);
|
||||||
|
|
||||||
|
expect(styles).toMatch(/\.bsk-help-body\s*\{[^}]*display:\s*block;/s);
|
||||||
|
expect(styles).not.toMatch(/\.bsk-help-body\s*\{[^}]*grid-template-rows/s);
|
||||||
|
expect(styles).toMatch(/\.bsk-help-banner\s*\{[^}]*justify-content:\s*flex-start;/s);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps the same banner width when collapsed", () => {
|
it("keeps the same banner width when collapsed", () => {
|
||||||
const { container } = renderOverlay(baseRequest());
|
const { container } = renderOverlay(baseRequest());
|
||||||
const styles = overlayStyles(container);
|
const styles = overlayStyles(container);
|
||||||
|
|||||||
@@ -1,37 +1,181 @@
|
|||||||
import { cleanup, fireEvent, render } from "@testing-library/react";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
|
import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||||
|
import { createElement } from "react";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RecordOverlay } from "../RecordOverlay";
|
import { RecordOverlay } from "../RecordOverlay";
|
||||||
|
|
||||||
|
function renderOverlay(request: { id: string; startedAtMs?: number; onFinish: () => void } | null) {
|
||||||
|
return render(
|
||||||
|
createElement(I18nextProvider, { i18n }, createElement(RecordOverlay, { request })),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function advanceRecording(ms: number) {
|
||||||
|
act(() => {
|
||||||
|
vi.advanceTimersByTime(ms);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("RecordOverlay", () => {
|
describe("RecordOverlay", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
cleanup();
|
cleanup();
|
||||||
|
vi.useRealTimers();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("renders the bottom pill with orange finish CTA and no logo", () => {
|
it("renders the bottom pill with the icon-only finish button", () => {
|
||||||
const onFinish = vi.fn();
|
const onFinish = vi.fn();
|
||||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
|
const { container } = renderOverlay({ id: "rec-1", onFinish });
|
||||||
|
|
||||||
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
||||||
expect(pill).toBeTruthy();
|
expect(pill).toBeTruthy();
|
||||||
expect(pill?.querySelector("img")).toBeNull();
|
|
||||||
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
||||||
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
||||||
|
expect((pill as HTMLElement).style.gap).toBe("5px");
|
||||||
|
expect((pill as HTMLElement).style.padding).toBe("6px 8px 6px 16px");
|
||||||
|
expect((pill as HTMLElement).style.height).toBe("50px");
|
||||||
|
expect((pill as HTMLElement).style.boxSizing).toBe("border-box");
|
||||||
|
expect(container.querySelector("[data-slot='record-overlay-collapse']")).toBeNull();
|
||||||
|
|
||||||
const finish = container.querySelector("[data-slot='record-overlay-finish']");
|
const status = container.querySelector("[data-slot='record-overlay-status']") as HTMLElement;
|
||||||
|
expect(status.style.minHeight).toBe("32px");
|
||||||
|
const finish = screen.getByRole("button", {
|
||||||
|
name: i18n.t("recordOverlay.finish", { ns: "extension" }),
|
||||||
|
});
|
||||||
expect(finish).toBeTruthy();
|
expect(finish).toBeTruthy();
|
||||||
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
|
expect(finish.textContent).toBe("");
|
||||||
|
const icon = finish.querySelector("img");
|
||||||
|
expect(icon).toBeTruthy();
|
||||||
|
expect(icon?.getAttribute("src")).toContain("record-icon.png");
|
||||||
|
expect((finish as HTMLElement).style.width).toBe("32px");
|
||||||
|
expect((finish as HTMLElement).style.height).toBe("32px");
|
||||||
|
expect(icon?.getAttribute("width")).toBe("24");
|
||||||
|
expect(icon?.getAttribute("height")).toBe("24");
|
||||||
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
||||||
|
|
||||||
fireEvent.click(finish!);
|
fireEvent.click(finish);
|
||||||
expect(onFinish).toHaveBeenCalledTimes(1);
|
expect(onFinish).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
||||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
||||||
|
|
||||||
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
||||||
expect(indicator).toBeTruthy();
|
expect(indicator).toBeTruthy();
|
||||||
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("shows the recording message for ten seconds, then displays elapsed time", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
|
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||||
|
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||||
|
|
||||||
|
expect(pill.dataset.phase).toBe("intro");
|
||||||
|
expect(label.textContent).toBe(i18n.t("recordOverlay.recording", { ns: "extension" }));
|
||||||
|
|
||||||
|
advanceRecording(9999);
|
||||||
|
expect(pill.dataset.phase).toBe("intro");
|
||||||
|
|
||||||
|
advanceRecording(1);
|
||||||
|
expect(pill.dataset.phase).toBe("timer");
|
||||||
|
expect(label.textContent).toBe("00:10");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("formats elapsed time as zero-padded minutes and seconds", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
|
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||||
|
|
||||||
|
advanceRecording(125_000);
|
||||||
|
expect(label.textContent).toBe("02:05");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a fixed timer label width with tabular numerals", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
|
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||||
|
|
||||||
|
advanceRecording(10_000);
|
||||||
|
expect(label.style.width).toBe("48px");
|
||||||
|
expect(label.style.maxWidth).toBe("48px");
|
||||||
|
expect(label.style.fontVariantNumeric).toBe("tabular-nums");
|
||||||
|
expect(label.style.textAlign).toBe("center");
|
||||||
|
|
||||||
|
advanceRecording(1_000);
|
||||||
|
expect(label.textContent).toBe("00:11");
|
||||||
|
expect(label.style.width).toBe("48px");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets the intro and elapsed time for a new request", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const first = { id: "rec-1", onFinish: vi.fn() };
|
||||||
|
const second = { id: "rec-2", onFinish: vi.fn() };
|
||||||
|
const { container, rerender } = renderOverlay(first);
|
||||||
|
|
||||||
|
advanceRecording(65_000);
|
||||||
|
rerender(
|
||||||
|
createElement(I18nextProvider, { i18n }, createElement(RecordOverlay, { request: second })),
|
||||||
|
);
|
||||||
|
|
||||||
|
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||||
|
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||||
|
expect(pill.dataset.phase).toBe("intro");
|
||||||
|
expect(label.textContent).toBe(i18n.t("recordOverlay.recording", { ns: "extension" }));
|
||||||
|
|
||||||
|
advanceRecording(10_000);
|
||||||
|
expect(pill.dataset.phase).toBe("timer");
|
||||||
|
expect(label.textContent).toBe("00:10");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("continues the session timer when the overlay remounts on a new page", () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const startedAtMs = Date.now() - 65_000;
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", startedAtMs, onFinish: vi.fn() });
|
||||||
|
|
||||||
|
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||||
|
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||||
|
expect(pill.dataset.phase).toBe("timer");
|
||||||
|
expect(label.textContent).toBe("01:05");
|
||||||
|
|
||||||
|
advanceRecording(5_000);
|
||||||
|
expect(label.textContent).toBe("01:10");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not start drag when pointer down on finish", () => {
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
|
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||||
|
const finish = container.querySelector("[data-slot='record-overlay-finish']") as HTMLElement;
|
||||||
|
|
||||||
|
fireEvent.pointerDown(finish, { button: 0, clientX: 10, clientY: 10, pointerId: 1 });
|
||||||
|
expect(pill.getAttribute("data-dragging")).toBe("false");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("moves the pill when dragging the non-button area without toggling collapse", () => {
|
||||||
|
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||||
|
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||||
|
|
||||||
|
pill.getBoundingClientRect = () =>
|
||||||
|
({
|
||||||
|
top: 600,
|
||||||
|
left: 200,
|
||||||
|
width: 320,
|
||||||
|
height: 48,
|
||||||
|
right: 520,
|
||||||
|
bottom: 648,
|
||||||
|
}) as DOMRect;
|
||||||
|
Object.defineProperty(pill, "offsetWidth", { value: 320, configurable: true });
|
||||||
|
Object.defineProperty(pill, "offsetHeight", { value: 48, configurable: true });
|
||||||
|
|
||||||
|
fireEvent.pointerDown(pill, { button: 0, clientX: 300, clientY: 620, pointerId: 1 });
|
||||||
|
fireEvent(window, new PointerEvent("pointermove", { clientX: 360, clientY: 580 }));
|
||||||
|
fireEvent(window, new PointerEvent("pointerup", { pointerId: 1 }));
|
||||||
|
|
||||||
|
expect(pill.getAttribute("data-phase")).toBe("intro");
|
||||||
|
expect(pill.style.left).toBe("260px");
|
||||||
|
expect(pill.style.top).toBe("560px");
|
||||||
|
expect(pill.style.bottom).toBe("auto");
|
||||||
|
expect(pill.style.transform).toBe("none");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { CAPTURE_SUPPRESS, type CaptureSuppressAck } from "@/lib/capture-suppress-bridge";
|
||||||
|
import { CAPTURE_HIDDEN_ATTR, createCaptureSuppressController } from "../capture-suppress";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* rAF is stubbed with a manual queue so tests can drive the compositor
|
||||||
|
* frame-by-frame and assert the ack only lands after the second frame.
|
||||||
|
*/
|
||||||
|
describe("capture-suppress controller", () => {
|
||||||
|
let rafQueue: FrameRequestCallback[];
|
||||||
|
let host: HTMLElement;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
rafQueue = [];
|
||||||
|
vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => {
|
||||||
|
rafQueue.push(cb);
|
||||||
|
return rafQueue.length;
|
||||||
|
});
|
||||||
|
vi.stubGlobal("cancelAnimationFrame", vi.fn());
|
||||||
|
document.body.innerHTML = "";
|
||||||
|
host = document.createElement("div");
|
||||||
|
document.body.append(host);
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Run every queued rAF callback, draining microtasks between rounds. */
|
||||||
|
async function flushFrames(rounds = 4): Promise<void> {
|
||||||
|
for (let round = 0; round < rounds && rafQueue.length > 0; round += 1) {
|
||||||
|
const callbacks = rafQueue.splice(0);
|
||||||
|
for (const cb of callbacks) cb(0);
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("begin hides the host immediately but only acks after two frames", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
|
||||||
|
const needsAsync = controller.handleMessage(
|
||||||
|
{ type: CAPTURE_SUPPRESS, phase: "begin" },
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(true);
|
||||||
|
expect(controller.pendingCount).toBe(1);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
expect(sendResponse).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// First frame commits the style change; the ack must still be pending.
|
||||||
|
await flushFrames(1);
|
||||||
|
expect(sendResponse).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// Second frame: the compositor has produced an overlay-free frame.
|
||||||
|
await flushFrames();
|
||||||
|
const expected: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith(expected);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("end removes the attribute once the count returns to zero", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
await flushFrames();
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
const needsAsync = controller.handleMessage(
|
||||||
|
{ type: CAPTURE_SUPPRESS, phase: "end" },
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(false);
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reference-counts concurrent begins so one end does not unhide early", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
await flushFrames();
|
||||||
|
expect(controller.pendingCount).toBe(2);
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||||
|
expect(controller.pendingCount).toBe(1);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a stray end as a no-op", () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, sendResponse);
|
||||||
|
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-applies the attribute when the host is remounted mid-capture", () => {
|
||||||
|
let currentHost: HTMLElement | null = host;
|
||||||
|
const controller = createCaptureSuppressController(() => currentHost);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
|
||||||
|
// Host lost and rebuilt before `end` arrives.
|
||||||
|
currentHost = null;
|
||||||
|
const rebuilt = document.createElement("div");
|
||||||
|
controller.onHostMounted(rebuilt);
|
||||||
|
|
||||||
|
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves a remounted host visible once suppression has ended", () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const rebuilt = document.createElement("div");
|
||||||
|
|
||||||
|
controller.onHostMounted(rebuilt);
|
||||||
|
|
||||||
|
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { createHelpRequestData } from "../help-request";
|
||||||
|
|
||||||
|
describe("createHelpRequestData", () => {
|
||||||
|
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||||
|
const finish = vi.fn();
|
||||||
|
const query = vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{ finish, query },
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||||
|
{ top: 80, left: 90, width: 140, height: 60 },
|
||||||
|
]);
|
||||||
|
request.onContinue(" done ");
|
||||||
|
request.onCancel();
|
||||||
|
|
||||||
|
expect(query).toHaveBeenCalledOnce();
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||||
|
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores rectangle replies for a replaced request", async () => {
|
||||||
|
const request = createHelpRequestData(
|
||||||
|
{
|
||||||
|
requestId: "help-1",
|
||||||
|
prompt: "complete the challenge",
|
||||||
|
selectors: [],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
finish: vi.fn(),
|
||||||
|
query: vi.fn(async () => ({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: "help-2",
|
||||||
|
prompt: "new challenge",
|
||||||
|
selectors: [],
|
||||||
|
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -116,4 +116,45 @@ describe("OverlayController", () => {
|
|||||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||||
expect(controller.snapshot().controlVisible).toBe(false);
|
expect(controller.snapshot().controlVisible).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("hides control while paused and shows it again only after authoritative control mode", () => {
|
||||||
|
const controller = new OverlayController();
|
||||||
|
|
||||||
|
controller.applyAgentControlMode("sess-1", "paused");
|
||||||
|
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||||
|
expect(controller.snapshot().controlVisible).toBe(false);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||||
|
|
||||||
|
controller.applyAgentControlMode("sess-1", "control");
|
||||||
|
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||||
|
expect(controller.snapshot().controlVisible).toBe(true);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("hides the control overlay when the user hides control hints", () => {
|
||||||
|
const controller = new OverlayController();
|
||||||
|
controller.activateAgentSession("sess-1");
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
|
|
||||||
|
controller.setControlHintsHidden(true);
|
||||||
|
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||||
|
// The session still owns the tab — only the chrome is hidden.
|
||||||
|
expect(controller.snapshot().controlVisible).toBe(true);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||||
|
|
||||||
|
controller.setControlHintsHidden(false);
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the control-hints preference across session overlay resets", () => {
|
||||||
|
const controller = new OverlayController();
|
||||||
|
controller.activateAgentSession("sess-1");
|
||||||
|
controller.setControlHintsHidden(true);
|
||||||
|
|
||||||
|
controller.resetAgentOverlays("sess-1");
|
||||||
|
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||||
|
|
||||||
|
controller.activateAgentSession("sess-2");
|
||||||
|
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface";
|
||||||
|
|
||||||
|
function mockRect(
|
||||||
|
el: Element,
|
||||||
|
rect: { left: number; top: number; width: number; height: number },
|
||||||
|
): void {
|
||||||
|
vi.spyOn(el, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top,
|
||||||
|
top: rect.top,
|
||||||
|
left: rect.left,
|
||||||
|
right: rect.left + rect.width,
|
||||||
|
bottom: rect.top + rect.height,
|
||||||
|
width: rect.width,
|
||||||
|
height: rect.height,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record-hover-surface", () => {
|
||||||
|
it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<ul class="header-menu tw-flex navbar-app__authentication-menu">
|
||||||
|
<li class="tg-popover">
|
||||||
|
<div class="tg-avatar tg-avatar--img"></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="tg-popover__popper">
|
||||||
|
<div class="tg-popover__content">
|
||||||
|
<div class="tg-popover__content-body">
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/u/me">My profile</a>
|
||||||
|
</li>
|
||||||
|
<li class="header-menu__item">
|
||||||
|
<a class="header-dropdown-menu__link" href="/dashboard/groups">My groups</a>
|
||||||
|
</li>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!),
|
||||||
|
).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat elements covered by an open surface as the surface owner", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button class="avatar">image</button>
|
||||||
|
<a class="fork" href="/fork">Fork</a>
|
||||||
|
<button class="clone">Clone & Download</button>
|
||||||
|
<div class="tg-popover__content-body"></div>
|
||||||
|
`;
|
||||||
|
const avatar = document.querySelector(".avatar")!;
|
||||||
|
const fork = document.querySelector(".fork")!;
|
||||||
|
const clone = document.querySelector(".clone")!;
|
||||||
|
const surface = document.querySelector(".tg-popover__content-body")!;
|
||||||
|
mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 });
|
||||||
|
mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 });
|
||||||
|
mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 });
|
||||||
|
mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
|
||||||
|
expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false);
|
||||||
|
expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not treat fixed navigation containers as plain floating surfaces", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<header class="navbar-app">
|
||||||
|
<a href="/fork">Fork</a>
|
||||||
|
<button>image</button>
|
||||||
|
</header>
|
||||||
|
<div class="plain-floating"></div>
|
||||||
|
`;
|
||||||
|
const navbar = document.querySelector(".navbar-app")!;
|
||||||
|
const floating = document.querySelector(".plain-floating")!;
|
||||||
|
mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 });
|
||||||
|
mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 });
|
||||||
|
vi.spyOn(window, "getComputedStyle").mockImplementation((el) => {
|
||||||
|
const style = {
|
||||||
|
cursor: "",
|
||||||
|
display: "block",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
position: el === navbar ? "fixed" : "absolute",
|
||||||
|
visibility: "visible",
|
||||||
|
} as CSSStyleDeclaration;
|
||||||
|
return style;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isHoverSurfaceCandidateElement(navbar)).toBe(false);
|
||||||
|
expect(isHoverSurfaceCandidateElement(floating)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
/**
|
||||||
|
* Content-script side of the capture-suppress bridge: hides the overlay
|
||||||
|
* shadow host while the background captures a screenshot.
|
||||||
|
*
|
||||||
|
* `begin` sets `data-bsk-capture-hidden` on the host (overlay.css maps it
|
||||||
|
* to `display: none !important`, which removes the whole shadow subtree
|
||||||
|
* from compositing) and only acks after two animation frames — the first
|
||||||
|
* frame commits the style change, the second guarantees the compositor
|
||||||
|
* has produced a frame without the overlay, so the capture API cannot
|
||||||
|
* pick up a stale frame that still contains it. `end` is
|
||||||
|
* reference-counted so overlapping captures do not unhide the host early.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
CAPTURE_SUPPRESS,
|
||||||
|
type CaptureSuppressAck,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
} from "@/lib/capture-suppress-bridge";
|
||||||
|
|
||||||
|
export const CAPTURE_HIDDEN_ATTR = "data-bsk-capture-hidden";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Safety net for hidden/background tabs where rAF callbacks never fire:
|
||||||
|
* the attribute is already set synchronously, so any frame the compositor
|
||||||
|
* produces from now on excludes the overlay. Waiting longer would just
|
||||||
|
* deadlock the capture.
|
||||||
|
*/
|
||||||
|
const FRAME_FALLBACK_MS = 500;
|
||||||
|
|
||||||
|
function nextFrame(): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const raf = requestAnimationFrame(() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cancelAnimationFrame(raf);
|
||||||
|
resolve();
|
||||||
|
}, FRAME_FALLBACK_MS);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureSuppressController {
|
||||||
|
/**
|
||||||
|
* Handle one bridge message. Returns true when the ack is sent
|
||||||
|
* asynchronously and the caller must keep the message channel open.
|
||||||
|
*/
|
||||||
|
handleMessage(
|
||||||
|
message: CaptureSuppressMessage,
|
||||||
|
sendResponse: (ack: CaptureSuppressAck) => void,
|
||||||
|
): boolean;
|
||||||
|
/** Re-apply the hidden attribute when the overlay host is (re)mounted. */
|
||||||
|
onHostMounted(host: HTMLElement): void;
|
||||||
|
/** Number of in-flight `begin` phases (exposed for tests). */
|
||||||
|
readonly pendingCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createCaptureSuppressController(
|
||||||
|
getHost: () => HTMLElement | null,
|
||||||
|
): CaptureSuppressController {
|
||||||
|
let pending = 0;
|
||||||
|
|
||||||
|
const ack: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
|
||||||
|
function setHidden(host: HTMLElement | null, hidden: boolean): void {
|
||||||
|
if (!host) return;
|
||||||
|
if (hidden) host.setAttribute(CAPTURE_HIDDEN_ATTR, "");
|
||||||
|
else host.removeAttribute(CAPTURE_HIDDEN_ATTR);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
get pendingCount() {
|
||||||
|
return pending;
|
||||||
|
},
|
||||||
|
handleMessage(message, sendResponse) {
|
||||||
|
if (message.phase === "begin") {
|
||||||
|
pending += 1;
|
||||||
|
setHidden(getHost(), true);
|
||||||
|
void (async () => {
|
||||||
|
await nextFrame();
|
||||||
|
await nextFrame();
|
||||||
|
sendResponse(ack);
|
||||||
|
})();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// `end` without a matching `begin` (e.g. a retried message) must not
|
||||||
|
// drive the counter negative or clear a concurrent capture's flag.
|
||||||
|
pending = Math.max(0, pending - 1);
|
||||||
|
if (pending === 0) setHidden(getHost(), false);
|
||||||
|
sendResponse(ack);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
onHostMounted(host) {
|
||||||
|
// Host lost + rebuilt mid-capture: the fresh host must stay hidden
|
||||||
|
// until the last `end` arrives.
|
||||||
|
setHidden(host, pending > 0);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||||
|
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||||
|
|
||||||
|
export interface HelpRequestDataDeps {
|
||||||
|
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||||
|
query(): Promise<HelpQueryResponse | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build the content-side representation shared by live delivery and recovery. */
|
||||||
|
export function createHelpRequestData(
|
||||||
|
message: Omit<HelpRequestMessage, "type">,
|
||||||
|
deps: HelpRequestDataDeps,
|
||||||
|
): HelpRequestData {
|
||||||
|
return {
|
||||||
|
id: message.requestId,
|
||||||
|
prompt: message.prompt,
|
||||||
|
...(message.title ? { title: message.title } : {}),
|
||||||
|
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||||
|
selectors: message.selectors,
|
||||||
|
rects: message.rects,
|
||||||
|
refreshRects: async () => {
|
||||||
|
const response = await deps.query();
|
||||||
|
return response?.active && response.request?.requestId === message.requestId
|
||||||
|
? response.request.rects
|
||||||
|
: undefined;
|
||||||
|
},
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { OverlayMode } from "@/lib/overlay-bridge";
|
||||||
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
||||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||||
import type { RecordRequestData } from "./RecordOverlay";
|
import type { RecordRequestData } from "./RecordOverlay";
|
||||||
@@ -9,6 +10,7 @@ export interface OverlayState {
|
|||||||
controlVisible: boolean;
|
controlVisible: boolean;
|
||||||
interrupting: boolean;
|
interrupting: boolean;
|
||||||
activeSessionId: string | null;
|
activeSessionId: string | null;
|
||||||
|
controlMode: OverlayMode;
|
||||||
automationBypassCount: number;
|
automationBypassCount: number;
|
||||||
/**
|
/**
|
||||||
* After record Finish/Stop clears `activeRecord`, the session is still
|
* After record Finish/Stop clears `activeRecord`, the session is still
|
||||||
@@ -16,6 +18,12 @@ export interface OverlayState {
|
|||||||
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
||||||
*/
|
*/
|
||||||
suppressControlAfterRecord: boolean;
|
suppressControlAfterRecord: boolean;
|
||||||
|
/**
|
||||||
|
* User preference (chrome.storage, toggled from the popup): hide the
|
||||||
|
* control hints — status pill, orange glow, and the input blocker that
|
||||||
|
* comes with them. Not session state; survives overlay resets.
|
||||||
|
*/
|
||||||
|
controlHintsHidden: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
||||||
@@ -31,8 +39,10 @@ export class OverlayController {
|
|||||||
activeRecord: null,
|
activeRecord: null,
|
||||||
interrupting: false,
|
interrupting: false,
|
||||||
activeSessionId: null,
|
activeSessionId: null,
|
||||||
|
controlMode: "hidden",
|
||||||
automationBypassCount: 0,
|
automationBypassCount: 0,
|
||||||
suppressControlAfterRecord: false,
|
suppressControlAfterRecord: false,
|
||||||
|
controlHintsHidden: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
snapshot(): OverlayState {
|
snapshot(): OverlayState {
|
||||||
@@ -44,7 +54,7 @@ export class OverlayController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
isControlVisible(): boolean {
|
isControlVisible(): boolean {
|
||||||
return this.state.activeSessionId !== null;
|
return this.state.activeSessionId !== null && this.state.controlMode === "control";
|
||||||
}
|
}
|
||||||
|
|
||||||
addBorrowRequest(request: BorrowRequestData): void {
|
addBorrowRequest(request: BorrowRequestData): void {
|
||||||
@@ -61,6 +71,25 @@ export class OverlayController {
|
|||||||
activateAgentSession(sessionId: string): void {
|
activateAgentSession(sessionId: string): void {
|
||||||
this.state.activeSessionId = sessionId;
|
this.state.activeSessionId = sessionId;
|
||||||
this.state.interrupting = false;
|
this.state.interrupting = false;
|
||||||
|
this.state.controlMode = "control";
|
||||||
|
}
|
||||||
|
|
||||||
|
applyAgentControlMode(sessionId: string | null, mode: OverlayMode): void {
|
||||||
|
if (mode === "control" && sessionId) {
|
||||||
|
this.state.activeSessionId = sessionId;
|
||||||
|
this.state.interrupting = false;
|
||||||
|
this.state.controlMode = "control";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((mode === "interrupting" || mode === "paused") && sessionId) {
|
||||||
|
this.state.activeSessionId = sessionId;
|
||||||
|
this.state.interrupting = mode === "interrupting";
|
||||||
|
this.state.controlMode = mode;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.state.activeSessionId = null;
|
||||||
|
this.state.interrupting = false;
|
||||||
|
this.state.controlMode = "hidden";
|
||||||
}
|
}
|
||||||
|
|
||||||
setInterrupting(interrupting: boolean): void {
|
setInterrupting(interrupting: boolean): void {
|
||||||
@@ -107,6 +136,10 @@ export class OverlayController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setControlHintsHidden(hidden: boolean): void {
|
||||||
|
this.state.controlHintsHidden = hidden;
|
||||||
|
}
|
||||||
|
|
||||||
resetAgentOverlays(sessionId: string): HelpRequestData | null {
|
resetAgentOverlays(sessionId: string): HelpRequestData | null {
|
||||||
if (this.state.activeSessionId && this.state.activeSessionId !== sessionId) {
|
if (this.state.activeSessionId && this.state.activeSessionId !== sessionId) {
|
||||||
return null;
|
return null;
|
||||||
@@ -118,6 +151,7 @@ export class OverlayController {
|
|||||||
activeRecord: null,
|
activeRecord: null,
|
||||||
interrupting: false,
|
interrupting: false,
|
||||||
activeSessionId: null,
|
activeSessionId: null,
|
||||||
|
controlMode: "hidden",
|
||||||
automationBypassCount: 0,
|
automationBypassCount: 0,
|
||||||
suppressControlAfterRecord: false,
|
suppressControlAfterRecord: false,
|
||||||
};
|
};
|
||||||
@@ -129,6 +163,7 @@ export class OverlayController {
|
|||||||
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
||||||
return (
|
return (
|
||||||
state.controlVisible &&
|
state.controlVisible &&
|
||||||
|
!state.controlHintsHidden &&
|
||||||
!state.suppressControlAfterRecord &&
|
!state.suppressControlAfterRecord &&
|
||||||
state.activeHelp === null &&
|
state.activeHelp === null &&
|
||||||
state.activeRecord === null
|
state.activeRecord === null
|
||||||
|
|||||||
@@ -1,2 +1,23 @@
|
|||||||
@import "tailwindcss";
|
@import "tailwindcss";
|
||||||
@source "../content";
|
@source "../content";
|
||||||
|
|
||||||
|
:host([data-bsk-overlay-surface]) {
|
||||||
|
position: fixed !important;
|
||||||
|
inset: 0 !important;
|
||||||
|
display: block !important;
|
||||||
|
width: 100vw !important;
|
||||||
|
height: 100vh !important;
|
||||||
|
z-index: 2147483647 !important;
|
||||||
|
pointer-events: none !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
:host([data-bsk-overlay-surface][data-bsk-overlay-blocking]) {
|
||||||
|
pointer-events: auto !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Screenshot suppression: the host lives in the main DOM, so hiding it
|
||||||
|
removes the whole shadow subtree from compositing and captured frames
|
||||||
|
only contain page content. */
|
||||||
|
:host([data-bsk-capture-hidden]) {
|
||||||
|
display: none !important;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,4 +1,17 @@
|
|||||||
import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
|
import {
|
||||||
|
describeEventTarget,
|
||||||
|
describeTarget,
|
||||||
|
resolveClickableElement,
|
||||||
|
resolveHoverElement,
|
||||||
|
type TargetDescriptor,
|
||||||
|
} from "@/lib/describe-target";
|
||||||
|
import {
|
||||||
|
evaluateHoverTrigger,
|
||||||
|
type HoverTriggerDecision,
|
||||||
|
type HoverTriggerRect,
|
||||||
|
hasDirectHoverInteractiveSignal,
|
||||||
|
hasStrongHoverExpansionSignal,
|
||||||
|
} from "@/lib/hover-trigger-policy";
|
||||||
import {
|
import {
|
||||||
isRecordCancelMessage,
|
isRecordCancelMessage,
|
||||||
isRecordStartMessage,
|
isRecordStartMessage,
|
||||||
@@ -15,6 +28,14 @@ import {
|
|||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
|
import {
|
||||||
|
closestHoverSurfaceCandidate,
|
||||||
|
collectHoverSurfaceStates,
|
||||||
|
decideHoverSurfaceRelation,
|
||||||
|
type HoverSurfaceState,
|
||||||
|
isHoverSurfaceCandidateElement,
|
||||||
|
isLikelyHoverSurfaceOwner,
|
||||||
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
@@ -49,8 +70,29 @@ interface FillSession {
|
|||||||
lastValue: string;
|
lastValue: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HoverCandidate {
|
||||||
|
element: Element;
|
||||||
|
target: TargetDescriptor;
|
||||||
|
recordedAt: number;
|
||||||
|
score: number;
|
||||||
|
eligible: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverSurfaceNode {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
owner: HoverCandidate;
|
||||||
|
parent?: HoverSurfaceNode;
|
||||||
|
}
|
||||||
|
|
||||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||||
|
|
||||||
|
const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
|
||||||
|
const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000;
|
||||||
|
const HOVER_REPLACE_SCORE_MARGIN = 50;
|
||||||
|
const HOVER_CANDIDATE_LIMIT = 24;
|
||||||
|
const HOVER_TRIGGER_LABEL_MAX = 48;
|
||||||
|
|
||||||
function eventTarget(event: Event): EventTarget | null {
|
function eventTarget(event: Event): EventTarget | null {
|
||||||
return event.composedPath()[0] ?? event.target;
|
return event.composedPath()[0] ?? event.target;
|
||||||
}
|
}
|
||||||
@@ -122,6 +164,173 @@ function fillableValue(el: FillableElement): string {
|
|||||||
return el.textContent ?? "";
|
return el.textContent ?? "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
|
||||||
|
return {
|
||||||
|
id: el.id || undefined,
|
||||||
|
class: typeof el.className === "string" ? el.className || undefined : undefined,
|
||||||
|
"data-testid": el.getAttribute("data-testid") ?? undefined,
|
||||||
|
"data-test": el.getAttribute("data-test") ?? undefined,
|
||||||
|
"data-cy": el.getAttribute("data-cy") ?? undefined,
|
||||||
|
"aria-label": el.getAttribute("aria-label") ?? undefined,
|
||||||
|
"aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
|
||||||
|
"aria-controls": el.getAttribute("aria-controls") ?? undefined,
|
||||||
|
"aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
|
||||||
|
"aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
|
||||||
|
"aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
|
||||||
|
contenteditable: el.getAttribute("contenteditable") ?? undefined,
|
||||||
|
disabled: el.hasAttribute("disabled") ? "" : undefined,
|
||||||
|
hidden: el.hasAttribute("hidden") ? "" : undefined,
|
||||||
|
inert: el.hasAttribute("inert") ? "" : undefined,
|
||||||
|
onclick: el.getAttribute("onclick") ?? undefined,
|
||||||
|
onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
|
||||||
|
onmouseover: el.getAttribute("onmouseover") ?? undefined,
|
||||||
|
role: el.getAttribute("role") ?? undefined,
|
||||||
|
tabindex: el.getAttribute("tabindex") ?? undefined,
|
||||||
|
title: el.getAttribute("title") ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerRect(el: Element): HoverTriggerRect {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
|
||||||
|
if (!(el instanceof HTMLElement)) return {};
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
return { cursor: style.cursor, pointerEvents: style.pointerEvents };
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasHoverGraphicDescendant(el: Element): boolean {
|
||||||
|
return el.querySelector("img,svg,use,path,i") !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLabelText(value: string): string {
|
||||||
|
return value.replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string {
|
||||||
|
const normalized = normalizeLabelText(value);
|
||||||
|
if (normalized.length <= max) return normalized;
|
||||||
|
return `${normalized.slice(0, max - 1)}…`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectHoverTriggerLabelText(root: Element): string {
|
||||||
|
let text = "";
|
||||||
|
const visit = (node: Node) => {
|
||||||
|
if (node.nodeType === Node.TEXT_NODE) {
|
||||||
|
text += ` ${node.textContent ?? ""}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!(node instanceof Element)) return;
|
||||||
|
if (node !== root && isHoverSurfaceCandidateElement(node)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const child of node.childNodes) visit(child);
|
||||||
|
};
|
||||||
|
for (const child of root.childNodes) visit(child);
|
||||||
|
return normalizeLabelText(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||||
|
if (!desc.name) return desc;
|
||||||
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
|
if (!compactName || compactName === desc.name) return desc;
|
||||||
|
const compactComparable = compactName.replace(/\s+/g, "");
|
||||||
|
const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, "");
|
||||||
|
const fullTextComparable = fullText.replace(/\s+/g, "");
|
||||||
|
if (
|
||||||
|
descComparable.startsWith(compactComparable) &&
|
||||||
|
fullTextComparable.startsWith(descComparable) &&
|
||||||
|
compactName.length < desc.name.length
|
||||||
|
) {
|
||||||
|
return { ...desc, name: truncateLabel(compactName) };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||||
|
return !target.role && !target.name && target.tag === "div";
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeAvatarElement(el: Element): boolean {
|
||||||
|
const className = typeof el.className === "string" ? el.className : "";
|
||||||
|
return /\b(avatar|user-avatar)\b/i.test(className);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeHoverTarget(
|
||||||
|
el: Element,
|
||||||
|
desc: TargetDescriptor,
|
||||||
|
decision: HoverTriggerDecision,
|
||||||
|
): TargetDescriptor {
|
||||||
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
|
return { ...desc, name: "image" };
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
isWeakHoverTarget(desc) &&
|
||||||
|
(looksLikeAvatarElement(el) ||
|
||||||
|
(hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
|
||||||
|
) {
|
||||||
|
return { tag: "img", role: "img", name: "image" };
|
||||||
|
}
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||||
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
|
const style = hoverTriggerStyle(el);
|
||||||
|
return {
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
role: desc.role,
|
||||||
|
label: desc.name,
|
||||||
|
attrs: hoverTriggerAttrs(el),
|
||||||
|
rect: hoverTriggerRect(el),
|
||||||
|
cursor: style.cursor,
|
||||||
|
pointerEvents: style.pointerEvents,
|
||||||
|
hasGraphicDescendant: hasHoverGraphicDescendant(el),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null {
|
||||||
|
if (!(target instanceof Element)) return null;
|
||||||
|
const element = resolveHoverElement(target);
|
||||||
|
if (!element) return null;
|
||||||
|
const desc = describeTarget(element);
|
||||||
|
const signals = hoverTriggerSignals(element, desc);
|
||||||
|
if (!signals) return null;
|
||||||
|
const decision = evaluateHoverTrigger(signals);
|
||||||
|
const hasExpansionSignal = hasStrongHoverExpansionSignal(signals);
|
||||||
|
const hasDirectSignal = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null;
|
||||||
|
if (!decision.eligible && !desc.name && !desc.role) return null;
|
||||||
|
const normalizedTarget = normalizeHoverTarget(
|
||||||
|
element,
|
||||||
|
compactHoverTargetName(element, desc),
|
||||||
|
decision,
|
||||||
|
);
|
||||||
|
const now = Date.now();
|
||||||
|
return {
|
||||||
|
element,
|
||||||
|
target: normalizedTarget,
|
||||||
|
recordedAt: now,
|
||||||
|
score: decision.score,
|
||||||
|
eligible: decision.eligible,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldReplaceHoverCandidate(
|
||||||
|
current: HoverCandidate,
|
||||||
|
next: HoverCandidate,
|
||||||
|
now: number,
|
||||||
|
): boolean {
|
||||||
|
if (next.element === current.element) return false;
|
||||||
|
if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
|
||||||
|
if (current.element.contains(next.element)) return false;
|
||||||
|
if (next.element.contains(current.element)) return true;
|
||||||
|
return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
|
||||||
|
}
|
||||||
|
|
||||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||||
if (!session || !(target instanceof Element)) return false;
|
if (!session || !(target instanceof Element)) return false;
|
||||||
@@ -168,10 +377,17 @@ export function startRecordCapture(
|
|||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
};
|
};
|
||||||
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
let fillSession: FillSession | null = null;
|
let fillSession: FillSession | null = null;
|
||||||
let composing = false;
|
let composing = false;
|
||||||
let lastUrl = location.href;
|
let lastUrl = location.href;
|
||||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||||
|
let pendingHover: HoverCandidate | null = null;
|
||||||
|
const recentHoverCandidates: HoverCandidate[] = [];
|
||||||
|
const emittedHoverElements = new WeakSet<Element>();
|
||||||
|
let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates());
|
||||||
|
const hoverSurfaceNodes = new Map<Element, HoverSurfaceNode>();
|
||||||
let generatedControlClick: Element | null = null;
|
let generatedControlClick: Element | null = null;
|
||||||
let navigationActionPending = false;
|
let navigationActionPending = false;
|
||||||
let navigationActionVersion = 0;
|
let navigationActionVersion = 0;
|
||||||
@@ -237,6 +453,223 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const rememberHoverCandidate = (hover: HoverCandidate) => {
|
||||||
|
const duplicate = recentHoverCandidates.findIndex(
|
||||||
|
(candidate) => candidate.element === hover.element,
|
||||||
|
);
|
||||||
|
if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1);
|
||||||
|
recentHoverCandidates.push(hover);
|
||||||
|
if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) {
|
||||||
|
recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceArea = (el: Element): number => {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
return rect.width * rect.height;
|
||||||
|
};
|
||||||
|
|
||||||
|
const smallestContaining = <T>(
|
||||||
|
target: Element,
|
||||||
|
items: Iterable<T>,
|
||||||
|
elementFor: (item: T) => Element,
|
||||||
|
): T | undefined => {
|
||||||
|
let best: T | undefined;
|
||||||
|
for (const item of items) {
|
||||||
|
const element = elementFor(item);
|
||||||
|
if (!element.contains(target)) continue;
|
||||||
|
if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) {
|
||||||
|
best = item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined =>
|
||||||
|
smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element);
|
||||||
|
|
||||||
|
const surfaceStateContaining = (
|
||||||
|
target: Element,
|
||||||
|
states: HoverSurfaceState[],
|
||||||
|
): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element);
|
||||||
|
|
||||||
|
const inferOwnerForSurface = (
|
||||||
|
surface: Element,
|
||||||
|
now: number,
|
||||||
|
before?: HoverCandidate,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (before && candidate.element === before.element) continue;
|
||||||
|
if (before && candidate.recordedAt > before.recordedAt) continue;
|
||||||
|
if (surface.contains(candidate.element)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const nodeForUnownedSurface = (
|
||||||
|
surfaceState: HoverSurfaceState,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const owner = inferOwnerForSurface(surfaceState.element, now);
|
||||||
|
if (!owner) return undefined;
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: surfaceState.element,
|
||||||
|
signature: surfaceState.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(surfaceState.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => {
|
||||||
|
const closestSurface = closestHoverSurfaceCandidate(target);
|
||||||
|
if (closestSurface) {
|
||||||
|
const owned = hoverSurfaceNodes.get(closestSurface);
|
||||||
|
if (owned) return owned;
|
||||||
|
}
|
||||||
|
const ownedContaining = ownedSurfaceContaining(target);
|
||||||
|
if (ownedContaining) return ownedContaining;
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
const surfaceState = closestSurface
|
||||||
|
? currentStates.find((state) => state.element === closestSurface)
|
||||||
|
: surfaceStateContaining(target, currentStates);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
if (!surfaceState) return undefined;
|
||||||
|
return nodeForUnownedSurface(surfaceState, currentStates, now);
|
||||||
|
};
|
||||||
|
|
||||||
|
const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => {
|
||||||
|
const current = new Set(currentStates.map((state) => state.element));
|
||||||
|
for (const element of hoverSurfaceNodes.keys()) {
|
||||||
|
if (!current.has(element)) hoverSurfaceNodes.delete(element);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const parentSurfaceNodeForOwner = (
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const ownedParent = ownedSurfaceContaining(owner.element);
|
||||||
|
if (ownedParent) return ownedParent;
|
||||||
|
const parentState = surfaceStateContaining(owner.element, currentStates);
|
||||||
|
if (!parentState) return undefined;
|
||||||
|
const parentOwner = inferOwnerForSurface(parentState.element, now, owner);
|
||||||
|
if (!parentOwner) return undefined;
|
||||||
|
const parentNode: HoverSurfaceNode = {
|
||||||
|
element: parentState.element,
|
||||||
|
signature: parentState.signature,
|
||||||
|
owner: parentOwner,
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(parentState.element, parentNode);
|
||||||
|
return parentNode;
|
||||||
|
};
|
||||||
|
|
||||||
|
const createSurfaceNode = (
|
||||||
|
state: HoverSurfaceState,
|
||||||
|
owner: HoverCandidate,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
): HoverSurfaceNode | undefined => {
|
||||||
|
const parent = parentSurfaceNodeForOwner(owner, currentStates, now);
|
||||||
|
if (parent?.element === state.element) return undefined;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined;
|
||||||
|
if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined;
|
||||||
|
const node: HoverSurfaceNode = {
|
||||||
|
element: state.element,
|
||||||
|
signature: state.signature,
|
||||||
|
owner,
|
||||||
|
...(parent ? { parent } : {}),
|
||||||
|
};
|
||||||
|
hoverSurfaceNodes.set(state.element, node);
|
||||||
|
return node;
|
||||||
|
};
|
||||||
|
|
||||||
|
const bindChangedHoverSurfaces = (
|
||||||
|
previousStates: Map<Element, string>,
|
||||||
|
currentStates: HoverSurfaceState[],
|
||||||
|
now: number,
|
||||||
|
) => {
|
||||||
|
for (const state of currentStates) {
|
||||||
|
if (previousStates.get(state.element) === state.signature) continue;
|
||||||
|
const owner = inferOwnerForSurface(state.element, now);
|
||||||
|
if (!owner) continue;
|
||||||
|
createSurfaceNode(state, owner, currentStates, now);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => {
|
||||||
|
const now = Date.now();
|
||||||
|
const currentStates = collectHoverSurfaceStates();
|
||||||
|
pruneGoneHoverSurfaces(currentStates);
|
||||||
|
bindChangedHoverSurfaces(previousStates, currentStates, now);
|
||||||
|
hoverSurfaceStates = hoverSurfaceStateMap(currentStates);
|
||||||
|
};
|
||||||
|
|
||||||
|
const scheduleHoverSurfaceRefresh = (previousStates: Map<Element, string>) => {
|
||||||
|
setTimeout(() => refreshHoverSurfaces(previousStates), 0);
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitHoverStep = (hover: HoverCandidate) => {
|
||||||
|
if (emittedHoverElements.has(hover.element)) return;
|
||||||
|
emitStep({
|
||||||
|
op: "hover",
|
||||||
|
target: hover.target,
|
||||||
|
});
|
||||||
|
emittedHoverElements.add(hover.element);
|
||||||
|
};
|
||||||
|
|
||||||
|
const surfaceOwnerChain = (
|
||||||
|
surface: HoverSurfaceNode,
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate[] => {
|
||||||
|
const chain: HoverCandidate[] = [];
|
||||||
|
const selected = new WeakSet<Element>();
|
||||||
|
const surfaces: HoverSurfaceNode[] = [];
|
||||||
|
let node: HoverSurfaceNode | undefined = surface;
|
||||||
|
while (node && surfaces.length < 4) {
|
||||||
|
surfaces.unshift(node);
|
||||||
|
node = node.parent;
|
||||||
|
}
|
||||||
|
for (const owned of surfaces) {
|
||||||
|
const owner = owned.owner;
|
||||||
|
if (owner.element === actionElement || actionElement.contains(owner.element)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected.has(owner.element)) continue;
|
||||||
|
if (emittedHoverElements.has(owner.element)) continue;
|
||||||
|
if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
selected.add(owner.element);
|
||||||
|
chain.push(owner);
|
||||||
|
}
|
||||||
|
return chain;
|
||||||
|
};
|
||||||
|
|
||||||
|
const containedHoverOwnerForAction = (
|
||||||
|
actionElement: Element,
|
||||||
|
now: number,
|
||||||
|
): HoverCandidate | undefined => {
|
||||||
|
for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) {
|
||||||
|
const candidate = recentHoverCandidates[index];
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (candidate.element === actionElement) continue;
|
||||||
|
if (!candidate.element.contains(actionElement)) continue;
|
||||||
|
if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
const emitClick = (event: MouseEvent) => {
|
const emitClick = (event: MouseEvent) => {
|
||||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||||
const target = describeEventTarget(eventTarget(event));
|
const target = describeEventTarget(eventTarget(event));
|
||||||
@@ -249,6 +682,54 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => {
|
||||||
|
if (!(actionTarget instanceof Element)) return;
|
||||||
|
const actionElement = resolveClickableElement(actionTarget) ?? actionTarget;
|
||||||
|
const surface = ownedSurfaceForAction(actionElement);
|
||||||
|
const now = Date.now();
|
||||||
|
const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now);
|
||||||
|
const hoverChain = surface
|
||||||
|
? surfaceOwnerChain(surface, actionElement, now)
|
||||||
|
: containedOwner
|
||||||
|
? [containedOwner]
|
||||||
|
: [];
|
||||||
|
if (hoverChain.length === 0) {
|
||||||
|
pendingHover = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const hover of hoverChain) emitHoverStep(hover);
|
||||||
|
pendingHover = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const onMouseOver = (event: MouseEvent) => {
|
||||||
|
const target = eventTarget(event);
|
||||||
|
if (isOverlayTarget(target)) return;
|
||||||
|
const now = Date.now();
|
||||||
|
if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
|
||||||
|
if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
|
||||||
|
pendingHover = null;
|
||||||
|
}
|
||||||
|
const candidate = hoverCandidateFromEvent(target);
|
||||||
|
if (candidate) {
|
||||||
|
const previousSurfaceStates = new Map(hoverSurfaceStates);
|
||||||
|
rememberHoverCandidate(candidate);
|
||||||
|
refreshHoverSurfaces(previousSurfaceStates);
|
||||||
|
scheduleHoverSurfaceRefresh(previousSurfaceStates);
|
||||||
|
}
|
||||||
|
const hover = candidate?.eligible ? candidate : null;
|
||||||
|
if (pendingHover && target instanceof Element) {
|
||||||
|
const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target);
|
||||||
|
if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!hover) return;
|
||||||
|
if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pendingHover = hover;
|
||||||
|
};
|
||||||
|
|
||||||
const onClick = (event: MouseEvent) => {
|
const onClick = (event: MouseEvent) => {
|
||||||
if (event.button !== 0) return;
|
if (event.button !== 0) return;
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
@@ -278,6 +759,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
const fillable = fillableFromTarget(target);
|
const fillable = fillableFromTarget(target);
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
|
emitHoverCandidateBeforeAction(fillable);
|
||||||
ensureFillSession(fillable);
|
ensureFillSession(fillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -285,6 +767,7 @@ export function startRecordCapture(
|
|||||||
if (target instanceof Element) {
|
if (target instanceof Element) {
|
||||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||||
if (nearbyFillable) {
|
if (nearbyFillable) {
|
||||||
|
emitHoverCandidateBeforeAction(nearbyFillable);
|
||||||
ensureFillSession(nearbyFillable);
|
ensureFillSession(nearbyFillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -304,6 +787,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
if (target instanceof Element && target.closest("select")) return;
|
if (target instanceof Element && target.closest("select")) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitClick(event);
|
emitClick(event);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -345,6 +829,7 @@ export function startRecordCapture(
|
|||||||
commitFillSession();
|
commitFillSession();
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (target instanceof HTMLSelectElement) {
|
if (target instanceof HTMLSelectElement) {
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||||
(opt.label || opt.textContent || opt.value).trim(),
|
(opt.label || opt.textContent || opt.value).trim(),
|
||||||
@@ -398,6 +883,7 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
|
emitHoverCandidateBeforeAction(target);
|
||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
@@ -408,6 +894,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
|
|
||||||
document.addEventListener("click", onClick, true);
|
document.addEventListener("click", onClick, true);
|
||||||
|
document.addEventListener("mouseover", onMouseOver, true);
|
||||||
document.addEventListener("focusin", onFocusIn, true);
|
document.addEventListener("focusin", onFocusIn, true);
|
||||||
document.addEventListener("focusout", onFocusOut, true);
|
document.addEventListener("focusout", onFocusOut, true);
|
||||||
document.addEventListener("input", onInput, true);
|
document.addEventListener("input", onInput, true);
|
||||||
@@ -438,6 +925,7 @@ export function startRecordCapture(
|
|||||||
dispose() {
|
dispose() {
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
document.removeEventListener("click", onClick, true);
|
document.removeEventListener("click", onClick, true);
|
||||||
|
document.removeEventListener("mouseover", onMouseOver, true);
|
||||||
document.removeEventListener("focusin", onFocusIn, true);
|
document.removeEventListener("focusin", onFocusIn, true);
|
||||||
document.removeEventListener("focusout", onFocusOut, true);
|
document.removeEventListener("focusout", onFocusOut, true);
|
||||||
document.removeEventListener("input", onInput, true);
|
document.removeEventListener("input", onInput, true);
|
||||||
@@ -468,7 +956,7 @@ export function handleRecordContentMessage(
|
|||||||
capture: RecordCaptureController | null;
|
capture: RecordCaptureController | null;
|
||||||
setActiveRequestId(id: string | null): void;
|
setActiveRequestId(id: string | null): void;
|
||||||
setCapture(capture: RecordCaptureController | null): void;
|
setCapture(capture: RecordCaptureController | null): void;
|
||||||
onStart(requestId: string): void;
|
onStart(requestId: string, startedAtMs?: number): void;
|
||||||
onStop(): void;
|
onStop(): void;
|
||||||
},
|
},
|
||||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||||
@@ -485,7 +973,7 @@ export function handleRecordContentMessage(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
state.setActiveRequestId(message.requestId);
|
state.setActiveRequestId(message.requestId);
|
||||||
state.onStart(message.requestId);
|
state.onStart(message.requestId, message.startedAtMs);
|
||||||
sendResponse?.({ ok: true });
|
sendResponse?.({ ok: true });
|
||||||
return sendResponse !== undefined;
|
return sendResponse !== undefined;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
export interface HoverSurfaceDecision {
|
||||||
|
related: boolean;
|
||||||
|
reason?: string;
|
||||||
|
surface?: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceContext {
|
||||||
|
triggerElement: Element;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverSurfaceState {
|
||||||
|
element: Element;
|
||||||
|
signature: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_SURFACE_SELECTOR = [
|
||||||
|
'[role="menu"]',
|
||||||
|
'[role="menubar"]',
|
||||||
|
'[role="listbox"]',
|
||||||
|
'[role="dialog"]',
|
||||||
|
'[role="tooltip"]',
|
||||||
|
'[aria-modal="true"]',
|
||||||
|
"[data-popper-placement]",
|
||||||
|
"[data-floating-ui-placement]",
|
||||||
|
"[data-headlessui-state]",
|
||||||
|
"[data-radix-popper-content-wrapper]",
|
||||||
|
].join(",");
|
||||||
|
|
||||||
|
const HOVER_SURFACE_WORD_RE =
|
||||||
|
/(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_ITEM_RE =
|
||||||
|
/(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i;
|
||||||
|
const HOVER_SURFACE_MAX_AXIS_GAP = 96;
|
||||||
|
const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16;
|
||||||
|
const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5;
|
||||||
|
|
||||||
|
function elementTokenText(el: Element): string {
|
||||||
|
return [
|
||||||
|
el.id,
|
||||||
|
typeof el.className === "string" ? el.className : "",
|
||||||
|
el.getAttribute("data-testid"),
|
||||||
|
el.getAttribute("data-test"),
|
||||||
|
el.getAttribute("data-cy"),
|
||||||
|
el.getAttribute("aria-label"),
|
||||||
|
el.getAttribute("role"),
|
||||||
|
]
|
||||||
|
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||||
|
.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRecognizedHoverSurfaceElement(el: Element): boolean {
|
||||||
|
if (el.matches(HOVER_SURFACE_SELECTOR)) return true;
|
||||||
|
const tokenText = elementTokenText(el);
|
||||||
|
if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isVisibleSurfaceElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return rectFor(el) !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestRecognizedHoverSurface(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isRecognizedHoverSurfaceElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPositionedFloatingElement(el: Element): boolean {
|
||||||
|
if (!(el instanceof HTMLElement)) return false;
|
||||||
|
const style = getComputedStyle(el);
|
||||||
|
if (style.position !== "absolute") return false;
|
||||||
|
if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const rect = rectFor(el);
|
||||||
|
if (!rect) return false;
|
||||||
|
const viewportArea = window.innerWidth * window.innerHeight;
|
||||||
|
const area = rect.width * rect.height;
|
||||||
|
return (
|
||||||
|
area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHoverSurfaceCandidateElement(el: Element): boolean {
|
||||||
|
return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closestHoverSurfaceCandidate(element: Element): Element | null {
|
||||||
|
let node: Element | null = element;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isHoverSurfaceCandidateElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function closestFloatingSurface(clickElement: Element): Element | null {
|
||||||
|
let node: Element | null = clickElement;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 10) {
|
||||||
|
if (isPositionedFloatingElement(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hoverSurfaceSignature(el: Element): string {
|
||||||
|
const rect = rectFor(el);
|
||||||
|
return [
|
||||||
|
rect
|
||||||
|
? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}`
|
||||||
|
: "",
|
||||||
|
(el.textContent ?? "").trim().slice(0, 512),
|
||||||
|
].join("|");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectHoverSurfaceStates(): HoverSurfaceState[] {
|
||||||
|
const states: HoverSurfaceState[] = [];
|
||||||
|
for (const el of document.querySelectorAll("*")) {
|
||||||
|
if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) {
|
||||||
|
states.push({ element: el, signature: hoverSurfaceSignature(el) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return states;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectFor(el: Element): DOMRect | null {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 || rect.height <= 0) return null;
|
||||||
|
return rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function surfaceRelationMetrics(surface: Element, trigger: Element) {
|
||||||
|
const surfaceRect = rectFor(surface);
|
||||||
|
const triggerRect = rectFor(trigger);
|
||||||
|
if (!surfaceRect || !triggerRect) return null;
|
||||||
|
const horizontalOverlap =
|
||||||
|
Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left);
|
||||||
|
const verticalOverlap =
|
||||||
|
Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top);
|
||||||
|
const horizontalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.left - surfaceRect.right,
|
||||||
|
surfaceRect.left - triggerRect.right,
|
||||||
|
);
|
||||||
|
const verticalGap = Math.max(
|
||||||
|
0,
|
||||||
|
triggerRect.top - surfaceRect.bottom,
|
||||||
|
surfaceRect.top - triggerRect.bottom,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNearTrigger(surface: Element, trigger: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics;
|
||||||
|
if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true;
|
||||||
|
return (
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean {
|
||||||
|
const metrics = surfaceRelationMetrics(surface, trigger);
|
||||||
|
if (!metrics) return true;
|
||||||
|
const {
|
||||||
|
surfaceRect,
|
||||||
|
triggerRect,
|
||||||
|
horizontalOverlap,
|
||||||
|
verticalOverlap,
|
||||||
|
horizontalGap,
|
||||||
|
verticalGap,
|
||||||
|
} = metrics;
|
||||||
|
const opensVertically =
|
||||||
|
horizontalOverlap > 0 &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8);
|
||||||
|
const opensSideways =
|
||||||
|
verticalOverlap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP &&
|
||||||
|
(triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8);
|
||||||
|
const opensFromCorner =
|
||||||
|
horizontalGap > 0 &&
|
||||||
|
verticalGap > 0 &&
|
||||||
|
horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP &&
|
||||||
|
verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP;
|
||||||
|
return opensVertically || opensSideways || opensFromCorner;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decideHoverSurfaceRelation(
|
||||||
|
context: HoverSurfaceContext,
|
||||||
|
clickElement: Element,
|
||||||
|
): HoverSurfaceDecision {
|
||||||
|
const clickSurface = closestRecognizedHoverSurface(clickElement);
|
||||||
|
if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) {
|
||||||
|
return { related: true, reason: "click-inside-hover-surface", surface: clickSurface };
|
||||||
|
}
|
||||||
|
|
||||||
|
const floatingSurface = closestFloatingSurface(clickElement);
|
||||||
|
if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) {
|
||||||
|
return {
|
||||||
|
related: true,
|
||||||
|
reason: "click-inside-near-floating-surface",
|
||||||
|
surface: floatingSurface,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) {
|
||||||
|
return { related: true, reason: "click-inside-trigger-contained-surface" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { related: false, reason: "click-outside-hover-surface" };
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import { ConnectionController } from "@/lib/connection-controller";
|
import { ConnectionController } from "@/lib/connection-controller";
|
||||||
|
import { startHeartbeat } from "@/lib/heartbeat";
|
||||||
import {
|
import {
|
||||||
getConnectionEnabled,
|
getConnectionEnabled,
|
||||||
setConnectionEnabled as persistConnectionEnabled,
|
setConnectionEnabled as persistConnectionEnabled,
|
||||||
@@ -8,15 +9,20 @@ import {
|
|||||||
} from "@/lib/instance-id";
|
} from "@/lib/instance-id";
|
||||||
import { startKeepalive } from "@/lib/keepalive";
|
import { startKeepalive } from "@/lib/keepalive";
|
||||||
import {
|
import {
|
||||||
|
OVERLAY_AGENT_STATE,
|
||||||
OVERLAY_AUTOMATION_BYPASS,
|
OVERLAY_AUTOMATION_BYPASS,
|
||||||
OVERLAY_MSG_INTERRUPT,
|
OVERLAY_MSG_INTERRUPT,
|
||||||
|
OVERLAY_MSG_READY,
|
||||||
OVERLAY_MSG_WHO_AM_I,
|
OVERLAY_MSG_WHO_AM_I,
|
||||||
|
type OverlayAgentStateMessage,
|
||||||
type OverlayInterruptRequest,
|
type OverlayInterruptRequest,
|
||||||
type OverlayInterruptResponse,
|
type OverlayInterruptResponse,
|
||||||
type OverlayMessage,
|
type OverlayMessage,
|
||||||
|
type OverlayMode,
|
||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||||
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import {
|
import {
|
||||||
@@ -43,17 +49,108 @@ export default defineBackground(() => {
|
|||||||
const sessions = new SessionManager();
|
const sessions = new SessionManager();
|
||||||
const cdp = new ChromiumCdp();
|
const cdp = new ChromiumCdp();
|
||||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||||
|
let overlayGeneration = 0;
|
||||||
|
const controlModes = new Map<string, OverlayMode>();
|
||||||
|
|
||||||
|
function setControlMode(sessionId: string, mode: OverlayMode): void {
|
||||||
|
if (controlModes.get(sessionId) === mode) return;
|
||||||
|
controlModes.set(sessionId, mode);
|
||||||
|
overlayGeneration += 1;
|
||||||
|
const ctx = sessions.get(sessionId);
|
||||||
|
if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage {
|
||||||
|
const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null;
|
||||||
|
if (!ctx) {
|
||||||
|
return {
|
||||||
|
type: OVERLAY_AGENT_STATE,
|
||||||
|
sessionId: null,
|
||||||
|
mode: "hidden",
|
||||||
|
generation: overlayGeneration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
type: OVERLAY_AGENT_STATE,
|
||||||
|
sessionId: ctx.sessionId,
|
||||||
|
mode: controlModes.get(ctx.sessionId) ?? "control",
|
||||||
|
generation: overlayGeneration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pushOverlayStateToTab(
|
||||||
|
tabId: number,
|
||||||
|
state: OverlayAgentStateMessage,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, state);
|
||||||
|
} catch {
|
||||||
|
// Restricted pages and not-yet-loaded content scripts cannot receive messages.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||||
|
const state = overlayStateForWindow(windowId);
|
||||||
|
const tabs = await chrome.tabs.query({ windowId });
|
||||||
|
await Promise.all(
|
||||||
|
tabs.map((tab) =>
|
||||||
|
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushAllAgentOverlayStates(): void {
|
||||||
|
const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId));
|
||||||
|
for (const windowId of windowIds) {
|
||||||
|
void pushOverlayStateForWindow(windowId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onOverlaySessionStateChanged(): void {
|
||||||
|
void sessionsLive.syncFromManager();
|
||||||
|
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||||
|
for (const sessionId of controlModes.keys()) {
|
||||||
|
if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId);
|
||||||
|
}
|
||||||
|
overlayGeneration += 1;
|
||||||
|
pushAllAgentOverlayStates();
|
||||||
|
}
|
||||||
|
|
||||||
|
function onBrowserControlResumed(sessionId: string): void {
|
||||||
|
const ctx = sessions.get(sessionId);
|
||||||
|
if (!ctx) return;
|
||||||
|
setControlMode(sessionId, "control");
|
||||||
|
}
|
||||||
|
|
||||||
|
function pushOverlayStateForAgentWindow(windowId: number): void {
|
||||||
|
if (!sessions.findByWindowId(windowId)) return;
|
||||||
|
void pushOverlayStateForWindow(windowId);
|
||||||
|
}
|
||||||
|
chrome.tabs.onActivated.addListener((activeInfo) => {
|
||||||
|
pushOverlayStateForAgentWindow(activeInfo.windowId);
|
||||||
|
});
|
||||||
|
chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => {
|
||||||
|
if (changeInfo.status !== "complete") return;
|
||||||
|
if (typeof tab.windowId !== "number") return;
|
||||||
|
pushOverlayStateForAgentWindow(tab.windowId);
|
||||||
|
});
|
||||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||||
// stale `true` does not keep waking us on every page load until the
|
// stale `true` does not keep waking us on every page load until the
|
||||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||||
void sessionsLive.refresh();
|
void sessionsLive.refresh();
|
||||||
|
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||||
|
manager: sessions,
|
||||||
|
sessionStopDeps: { cdp },
|
||||||
|
onSessionsChanged: () => {
|
||||||
|
void sessionsLive.syncFromManager();
|
||||||
|
},
|
||||||
|
});
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
cdp,
|
cdp,
|
||||||
onSessionsChanged: () => {
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
void sessionsLive.syncFromManager();
|
onBrowserControlResumed,
|
||||||
},
|
|
||||||
approveBorrow: (ctx) =>
|
approveBorrow: (ctx) =>
|
||||||
requestBorrowConfirmation(ctx.tabId, {
|
requestBorrowConfirmation(ctx.tabId, {
|
||||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||||
@@ -123,9 +220,7 @@ export default defineBackground(() => {
|
|||||||
manager: sessions,
|
manager: sessions,
|
||||||
transport,
|
transport,
|
||||||
cdp,
|
cdp,
|
||||||
onSessionsChanged: () => {
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
void sessionsLive.syncFromManager();
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||||
@@ -139,9 +234,61 @@ export default defineBackground(() => {
|
|||||||
shouldConnect: () => controller.isConnectionEnabled,
|
shouldConnect: () => controller.isConnectionEnabled,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
||||||
|
// link is live, beat every 20s so WebSocket activity keeps the service
|
||||||
|
// worker — and thus the daemon connection — alive during use, rather
|
||||||
|
// than depending on the boundary-hugging 30s keepalive alarm. The
|
||||||
|
// daemon also uses these beats to reap a silently-dead browser.
|
||||||
|
startHeartbeat({
|
||||||
|
send: (frame) => transport.send(frame),
|
||||||
|
onActiveChange: (cb) =>
|
||||||
|
controller.subscribe((snap) =>
|
||||||
|
cb(snap.state === "connected" || snap.state === "version_skew"),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
||||||
|
// across OS sleep regardless of any keepalive, and the setTimeout-based
|
||||||
|
// transport backoff dies with it. These Chrome lifecycle events revive
|
||||||
|
// the worker and let us reconnect immediately instead of waiting for
|
||||||
|
// the next 30s alarm tick. They only help when the daemon is actually
|
||||||
|
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
||||||
|
const reconnectIfNeeded = () => {
|
||||||
|
if (!controller.isConnectionEnabled) return;
|
||||||
|
if (transport.state === "connected") return;
|
||||||
|
void transport.connect().catch((err) => {
|
||||||
|
console.debug("[browser-skill] wake reconnect attempt failed", err);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
||||||
|
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
|
||||||
|
}
|
||||||
|
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
||||||
|
chrome.idle.onStateChanged.addListener((state) => {
|
||||||
|
// "active" fires when the user returns from idle/locked — the most
|
||||||
|
// reliable "machine just woke" signal we get.
|
||||||
|
if (state === "active") reconnectIfNeeded();
|
||||||
|
});
|
||||||
|
// Treat >60s of no input as idle so the active transition is timely.
|
||||||
|
chrome.idle.setDetectionInterval?.(60);
|
||||||
|
}
|
||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const connectionEnabled = await getConnectionEnabled();
|
const connectionEnabled = await getConnectionEnabled();
|
||||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled);
|
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||||
|
beforeDisconnect: async () => {
|
||||||
|
const report = await cleanupAfterDisconnect();
|
||||||
|
if (report.failures.length > 0) {
|
||||||
|
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onDisconnected: async () => {
|
||||||
|
const report = await cleanupAfterDisconnect();
|
||||||
|
if (report.failures.length > 0) {
|
||||||
|
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
})().catch((err) => {
|
})().catch((err) => {
|
||||||
console.error("[browser-skill] controller failed to attach", err);
|
console.error("[browser-skill] controller failed to attach", err);
|
||||||
});
|
});
|
||||||
@@ -157,9 +304,19 @@ export default defineBackground(() => {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (msg.kind === OVERLAY_MSG_READY) {
|
||||||
|
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
|
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
|
||||||
const req = msg as OverlayInterruptRequest;
|
const req = msg as OverlayInterruptRequest;
|
||||||
|
const ctx = sessions.get(req.sessionId);
|
||||||
|
if (ctx) setControlMode(req.sessionId, "interrupting");
|
||||||
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
|
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
|
||||||
|
if (reply.ok && sessions.get(req.sessionId)) {
|
||||||
|
setControlMode(req.sessionId, "paused");
|
||||||
|
}
|
||||||
sendResponse(reply);
|
sendResponse(reply);
|
||||||
});
|
});
|
||||||
return true; // keep channel open
|
return true; // keep channel open
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
import React from "react";
|
import React from "react";
|
||||||
|
import { flushSync } from "react-dom";
|
||||||
import ReactDOM from "react-dom/client";
|
import ReactDOM from "react-dom/client";
|
||||||
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||||
|
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||||
|
import { createHelpRequestData } from "@/content/help-request";
|
||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||||
@@ -14,29 +17,40 @@ import {
|
|||||||
type RecordCaptureController,
|
type RecordCaptureController,
|
||||||
} from "@/content/record-capture";
|
} from "@/content/record-capture";
|
||||||
import {
|
import {
|
||||||
HELP_RESPONSE,
|
type CaptureSuppressAck,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
isCaptureSuppressMessage,
|
||||||
|
} from "@/lib/capture-suppress-bridge";
|
||||||
|
import {
|
||||||
|
HELP_ACK,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
|
type HelpAckMessage,
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
|
import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id";
|
||||||
import {
|
import {
|
||||||
isOverlayAgentOverlayResetMessage,
|
isOverlayAgentOverlayResetMessage,
|
||||||
|
isOverlayAgentStateMessage,
|
||||||
OVERLAY_AUTOMATION_BYPASS,
|
OVERLAY_AUTOMATION_BYPASS,
|
||||||
OVERLAY_MSG_WHO_AM_I,
|
OVERLAY_MSG_READY,
|
||||||
type OverlayAgentOverlayResetMessage,
|
type OverlayAgentOverlayResetMessage,
|
||||||
|
type OverlayAgentStateMessage,
|
||||||
type OverlayAutomationBypassMessage,
|
type OverlayAutomationBypassMessage,
|
||||||
type OverlayWhoAmIResponse,
|
|
||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
|
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
BorrowRequestMessage,
|
BorrowRequestMessage,
|
||||||
@@ -55,15 +69,25 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
|
||||||
null;
|
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
|
let overlayContainer: HTMLElement | null = null;
|
||||||
|
let activeAgentState: OverlayAgentStateMessage | null = null;
|
||||||
let hostLossReported = false;
|
let hostLossReported = false;
|
||||||
let remountInProgress = false;
|
let remountInProgress = false;
|
||||||
|
|
||||||
|
// Load the user's control-hints preference up front so an already-active
|
||||||
|
// Agent session does not flash the overlay before the stored value lands.
|
||||||
|
try {
|
||||||
|
overlays.setControlHintsHidden(await getControlHintsHidden());
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk overlay] control-hints preference read failed", err);
|
||||||
|
}
|
||||||
|
|
||||||
|
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
||||||
|
|
||||||
const ui = await createShadowRootUi(ctx, {
|
const ui = await createShadowRootUi(ctx, {
|
||||||
name: "browser-skill-overlay",
|
name: "browser-skill-overlay",
|
||||||
position: "inline",
|
position: "inline",
|
||||||
@@ -73,56 +97,141 @@ export default defineContentScript({
|
|||||||
shadowHost.setAttribute("aria-hidden", "true");
|
shadowHost.setAttribute("aria-hidden", "true");
|
||||||
shadowHost.setAttribute("data-bsk-overlay", "");
|
shadowHost.setAttribute("data-bsk-overlay", "");
|
||||||
overlayHost = shadowHost;
|
overlayHost = shadowHost;
|
||||||
|
overlayContainer = container;
|
||||||
hostLossReported = false;
|
hostLossReported = false;
|
||||||
|
// A host rebuilt mid-capture must stay hidden until `end` arrives.
|
||||||
|
captureSuppress.onHostMounted(shadowHost);
|
||||||
const app = document.createElement("div");
|
const app = document.createElement("div");
|
||||||
app.className = "bsk-overlay-root";
|
app.className = "bsk-overlay-root";
|
||||||
container.append(app);
|
container.append(app);
|
||||||
reactRoot = ReactDOM.createRoot(app);
|
reactRoot = ReactDOM.createRoot(app);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
|
void requestOverlayState();
|
||||||
return reactRoot;
|
return reactRoot;
|
||||||
},
|
},
|
||||||
onRemove(root) {
|
onRemove(root) {
|
||||||
overlayHost = null;
|
overlayHost = null;
|
||||||
|
overlayContainer = null;
|
||||||
root?.unmount();
|
root?.unmount();
|
||||||
reactRoot = null;
|
reactRoot = null;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
function renderOverlay() {
|
function setOverlaySurfaceState(active: boolean, blocking: boolean): void {
|
||||||
|
const host = overlayHost;
|
||||||
|
const container = overlayContainer;
|
||||||
|
if (!host || !container) return;
|
||||||
|
|
||||||
|
if (active) {
|
||||||
|
host.setAttribute("data-bsk-overlay-surface", "");
|
||||||
|
if (blocking) {
|
||||||
|
host.setAttribute("data-bsk-overlay-blocking", "");
|
||||||
|
} else {
|
||||||
|
host.removeAttribute("data-bsk-overlay-blocking");
|
||||||
|
}
|
||||||
|
Object.assign(container.style, {
|
||||||
|
position: "fixed",
|
||||||
|
inset: "0",
|
||||||
|
pointerEvents: "none",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
host.removeAttribute("data-bsk-overlay-surface");
|
||||||
|
host.removeAttribute("data-bsk-overlay-blocking");
|
||||||
|
container.style.removeProperty("position");
|
||||||
|
container.style.removeProperty("inset");
|
||||||
|
container.style.removeProperty("pointer-events");
|
||||||
|
}
|
||||||
|
|
||||||
|
function setOverlayHostHiddenFromAccessibility(hidden: boolean): void {
|
||||||
|
const host = overlayHost;
|
||||||
|
if (!host) return;
|
||||||
|
|
||||||
|
if (!hidden) {
|
||||||
|
host.removeAttribute("aria-hidden");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const focusedInShadow = host.shadowRoot?.activeElement;
|
||||||
|
if (focusedInShadow instanceof HTMLElement) {
|
||||||
|
focusedInShadow.blur();
|
||||||
|
}
|
||||||
|
if (document.activeElement === host) {
|
||||||
|
host.blur();
|
||||||
|
}
|
||||||
|
host.setAttribute("aria-hidden", "true");
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderReactOverlays(): void {
|
||||||
const overlayState = overlays.snapshot();
|
const overlayState = overlays.snapshot();
|
||||||
reactRoot?.render(
|
const controlOverlayVisible = shouldShowAgentControlOverlay(overlayState);
|
||||||
React.createElement(
|
const interactiveOverlayVisible =
|
||||||
I18nextProvider,
|
overlayState.borrowRequests.length > 0 ||
|
||||||
{ i18n },
|
overlayState.activeHelp !== null ||
|
||||||
React.createElement(
|
overlayState.activeRecord !== null;
|
||||||
React.Fragment,
|
setOverlayHostHiddenFromAccessibility(!interactiveOverlayVisible);
|
||||||
null,
|
setOverlaySurfaceState(
|
||||||
React.createElement(BorrowConfirmationOverlay, {
|
controlOverlayVisible,
|
||||||
requests: overlayState.borrowRequests,
|
controlOverlayVisible && overlayState.automationBypassCount === 0,
|
||||||
}),
|
|
||||||
React.createElement(ControlOverlay, {
|
|
||||||
visible: shouldShowAgentControlOverlay(overlayState),
|
|
||||||
interrupting: overlayState.interrupting,
|
|
||||||
automationBypass: overlayState.automationBypassCount > 0,
|
|
||||||
onInterrupt: handleInterrupt,
|
|
||||||
}),
|
|
||||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
|
||||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
const root = reactRoot;
|
||||||
|
if (!root) return;
|
||||||
|
flushSync(() => {
|
||||||
|
root.render(
|
||||||
|
React.createElement(
|
||||||
|
I18nextProvider,
|
||||||
|
{ i18n },
|
||||||
|
React.createElement(
|
||||||
|
React.Fragment,
|
||||||
|
null,
|
||||||
|
React.createElement(BorrowConfirmationOverlay, {
|
||||||
|
requests: overlayState.borrowRequests,
|
||||||
|
}),
|
||||||
|
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||||
|
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||||
|
React.createElement(ControlOverlay, {
|
||||||
|
visible: controlOverlayVisible,
|
||||||
|
interrupting: overlayState.interrupting,
|
||||||
|
automationBypass: overlayState.automationBypassCount > 0,
|
||||||
|
onInterrupt: handleInterrupt,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAll(): void {
|
||||||
|
renderReactOverlays();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForRenderedOverlayUpdate(): Promise<void> {
|
||||||
|
await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
|
||||||
|
await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearCurrentAgentSession(): void {
|
||||||
|
const sessionId = overlays.snapshot().activeSessionId;
|
||||||
|
if (!sessionId) return;
|
||||||
|
resetAgentOverlayState(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyOverlayState(state: OverlayAgentStateMessage): void {
|
||||||
|
activeAgentState = state;
|
||||||
|
overlays.applyAgentControlMode(state.sessionId, state.mode);
|
||||||
|
renderAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
function resetAgentOverlayState(sessionId: string) {
|
function resetAgentOverlayState(sessionId: string) {
|
||||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
activeHelpRespond?.("cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
activeHelpRespond = null;
|
|
||||||
}
|
}
|
||||||
recordCapture?.dispose();
|
recordCapture?.dispose();
|
||||||
recordCapture = null;
|
recordCapture = null;
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
renderOverlay();
|
renderAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleInterrupt() {
|
function handleInterrupt() {
|
||||||
@@ -134,12 +243,8 @@ export default defineContentScript({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
overlays.setInterrupting(true);
|
overlays.setInterrupting(true);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
|
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
|
||||||
// Always retract the mask after the round trip resolves
|
|
||||||
// (success, failure, or timeout). Cancellation is fire-and-
|
|
||||||
// forget on the daemon side; the user must not be stuck
|
|
||||||
// behind a transient issue. The Agent Window stays open.
|
|
||||||
resetAgentOverlayState(sessionId);
|
resetAgentOverlayState(sessionId);
|
||||||
if (!reply.ok) {
|
if (!reply.ok) {
|
||||||
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
|
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
|
||||||
@@ -153,11 +258,17 @@ export default defineContentScript({
|
|||||||
| BorrowCancelMessage
|
| BorrowCancelMessage
|
||||||
| HelpRequestMessage
|
| HelpRequestMessage
|
||||||
| HelpCancelMessage
|
| HelpCancelMessage
|
||||||
|
| CaptureSuppressMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||||
) => {
|
) => {
|
||||||
|
if (isCaptureSuppressMessage(message)) {
|
||||||
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
message,
|
message,
|
||||||
@@ -170,9 +281,10 @@ export default defineContentScript({
|
|||||||
setCapture: (capture) => {
|
setCapture: (capture) => {
|
||||||
recordCapture = capture;
|
recordCapture = capture;
|
||||||
},
|
},
|
||||||
onStart: (requestId) => {
|
onStart: (requestId, startedAtMs) => {
|
||||||
overlays.setAgentRecordRequest({
|
overlays.setAgentRecordRequest({
|
||||||
id: requestId,
|
id: requestId,
|
||||||
|
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||||
onFinish: () => {
|
onFinish: () => {
|
||||||
void chrome.runtime.sendMessage({
|
void chrome.runtime.sendMessage({
|
||||||
type: RECORD_FINISH,
|
type: RECORD_FINISH,
|
||||||
@@ -180,11 +292,11 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
renderOverlay();
|
renderAll();
|
||||||
},
|
},
|
||||||
onStop: () => {
|
onStop: () => {
|
||||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
sendResponse as unknown as
|
sendResponse as unknown as
|
||||||
@@ -202,7 +314,12 @@ export default defineContentScript({
|
|||||||
) {
|
) {
|
||||||
const bypassMsg = message as OverlayAutomationBypassMessage;
|
const bypassMsg = message as OverlayAutomationBypassMessage;
|
||||||
overlays.setAutomationBypass(bypassMsg.enabled);
|
overlays.setAutomationBypass(bypassMsg.enabled);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isOverlayAgentStateMessage(message)) {
|
||||||
|
applyOverlayState(message);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -213,48 +330,28 @@ export default defineContentScript({
|
|||||||
|
|
||||||
if (message.type === "borrow-cancel") {
|
if (message.type === "borrow-cancel") {
|
||||||
overlays.removeBorrowRequest(message.requestId);
|
overlays.removeBorrowRequest(message.requestId);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isHelpCancelMessage(message)) {
|
if (isHelpCancelMessage(message)) {
|
||||||
const state = overlays.snapshot();
|
const state = overlays.snapshot();
|
||||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||||
activeHelpRespond?.("cancelled");
|
overlays.clearAgentHelpRequest(message.requestId);
|
||||||
|
renderAll();
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
let responded = false;
|
const previousHelp = mountHelpRequest(helpMsg);
|
||||||
const respond = (outcome: "continued" | "cancelled", note?: string) => {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
if (responded) return;
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
responded = true;
|
|
||||||
const reply: HelpResponseMessage = {
|
|
||||||
type: HELP_RESPONSE,
|
|
||||||
outcome,
|
|
||||||
...(note ? { note } : {}),
|
|
||||||
};
|
|
||||||
sendResponse(reply);
|
|
||||||
activeHelpRespond = null;
|
|
||||||
overlays.clearAgentHelpRequest(helpMsg.requestId);
|
|
||||||
renderOverlay();
|
|
||||||
};
|
|
||||||
const previousHelp = overlays.setAgentHelpRequest({
|
|
||||||
id: helpMsg.requestId,
|
|
||||||
prompt: helpMsg.prompt,
|
|
||||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
|
||||||
selectors: helpMsg.selectors,
|
|
||||||
onContinue: (note: string) => respond("continued", note.trim() ? note : undefined),
|
|
||||||
onCancel: () => respond("cancelled"),
|
|
||||||
});
|
|
||||||
if (previousHelp) {
|
|
||||||
activeHelpRespond?.("cancelled");
|
|
||||||
}
|
}
|
||||||
activeHelpRespond = respond;
|
renderAll();
|
||||||
renderOverlay();
|
sendResponse({ type: HELP_ACK, ok: true });
|
||||||
return true; // async sendResponse
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.type === "borrow-request") {
|
if (message.type === "borrow-request") {
|
||||||
@@ -264,7 +361,7 @@ export default defineContentScript({
|
|||||||
responded = true;
|
responded = true;
|
||||||
sendResponse({ type: "borrow-response", allowed });
|
sendResponse({ type: "borrow-response", allowed });
|
||||||
overlays.removeBorrowRequest(message.requestId);
|
overlays.removeBorrowRequest(message.requestId);
|
||||||
renderOverlay();
|
renderAll();
|
||||||
};
|
};
|
||||||
|
|
||||||
overlays.addBorrowRequest({
|
overlays.addBorrowRequest({
|
||||||
@@ -275,41 +372,79 @@ export default defineContentScript({
|
|||||||
onAllow: () => respond(true),
|
onAllow: () => respond(true),
|
||||||
onDeny: () => respond(false),
|
onDeny: () => respond(false),
|
||||||
});
|
});
|
||||||
renderOverlay();
|
renderAll();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
async function syncAgentOverlay(): Promise<void> {
|
async function sendHelpFinish(
|
||||||
if (!(await anySessionLive())) return;
|
requestId: string,
|
||||||
try {
|
outcome: "continued" | "cancelled",
|
||||||
const reply = (await chrome.runtime.sendMessage({
|
note?: string,
|
||||||
kind: OVERLAY_MSG_WHO_AM_I,
|
): Promise<void> {
|
||||||
})) as OverlayWhoAmIResponse | undefined;
|
const msg: HelpFinishMessage = {
|
||||||
if (!reply?.sessionId) return;
|
type: HELP_FINISH,
|
||||||
|
requestId,
|
||||||
|
outcome,
|
||||||
|
...(note ? { note } : {}),
|
||||||
|
};
|
||||||
|
overlays.clearAgentHelpRequest(requestId);
|
||||||
|
renderAll();
|
||||||
|
await waitForRenderedOverlayUpdate();
|
||||||
|
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||||
|
console.debug("[bsk overlay] help finish failed", err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Query / re-arm recording *before* activating the control overlay so
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||||
// record start (navigate → content load) does not flash
|
return overlays.setAgentHelpRequest(
|
||||||
// 「Agent 正在控制」before RecordOverlay mounts.
|
createHelpRequestData(helpMsg, {
|
||||||
let recordQuery: { active?: boolean; requestId?: string } | undefined;
|
finish: (requestId, outcome, note) => {
|
||||||
|
void sendHelpFinish(requestId, outcome, note);
|
||||||
|
},
|
||||||
|
query: () =>
|
||||||
|
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||||
|
HelpQueryResponse | undefined
|
||||||
|
>,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||||
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||||
try {
|
try {
|
||||||
recordQuery = (await chrome.runtime.sendMessage({
|
const helpQuery = (await chrome.runtime.sendMessage({
|
||||||
type: RECORD_QUERY,
|
type: HELP_QUERY,
|
||||||
})) as { active?: boolean; requestId?: string } | undefined;
|
})) as HelpQueryResponse | undefined;
|
||||||
|
if (helpQuery?.active && helpQuery.request) {
|
||||||
|
mountHelpRequest(helpQuery.request);
|
||||||
|
renderAll();
|
||||||
|
return;
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk overlay] record query failed", err);
|
console.debug("[bsk overlay] help query failed", err);
|
||||||
}
|
}
|
||||||
|
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function queryActiveRecord(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const recordQuery = (await chrome.runtime.sendMessage({
|
||||||
|
type: RECORD_QUERY,
|
||||||
|
})) as RecordQueryResponse | undefined;
|
||||||
if (
|
if (
|
||||||
recordQuery?.active &&
|
recordQuery?.active &&
|
||||||
typeof recordQuery.requestId === "string" &&
|
typeof recordQuery.requestId === "string" &&
|
||||||
overlays.snapshot().activeRecord === null
|
overlays.snapshot().activeRecord === null
|
||||||
) {
|
) {
|
||||||
const requestId = recordQuery.requestId;
|
const requestId = recordQuery.requestId;
|
||||||
|
const startedAtMs = recordQuery.startedAtMs;
|
||||||
overlays.setAgentRecordRequest({
|
overlays.setAgentRecordRequest({
|
||||||
id: requestId,
|
id: requestId,
|
||||||
|
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||||
onFinish: () => {
|
onFinish: () => {
|
||||||
void chrome.runtime.sendMessage({
|
void chrome.runtime.sendMessage({
|
||||||
type: RECORD_FINISH,
|
type: RECORD_FINISH,
|
||||||
@@ -317,22 +452,51 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
renderAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
overlays.activateAgentSession(reply.sessionId);
|
|
||||||
renderOverlay();
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk overlay] syncAgentOverlay failed", err);
|
console.debug("[bsk overlay] record query failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshAuxiliaryOverlayState(): Promise<void> {
|
||||||
|
await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestOverlayState(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const state = (await chrome.runtime.sendMessage({
|
||||||
|
kind: OVERLAY_MSG_READY,
|
||||||
|
})) as OverlayAgentStateMessage | undefined;
|
||||||
|
if (state && isOverlayAgentStateMessage(state)) {
|
||||||
|
applyOverlayState(state);
|
||||||
|
}
|
||||||
|
void refreshAuxiliaryOverlayState();
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk overlay] overlay.ready failed", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const onPageShow = (event: PageTransitionEvent) => {
|
const onPageShow = (event: PageTransitionEvent) => {
|
||||||
if (event.persisted) void syncAgentOverlay();
|
if (event.persisted) void requestOverlayState();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Live-apply popup toggles of the control-hints preference.
|
||||||
|
const onStorageChange = (
|
||||||
|
changes: Record<string, chrome.storage.StorageChange>,
|
||||||
|
areaName: string,
|
||||||
|
) => {
|
||||||
|
if (areaName !== "local") return;
|
||||||
|
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||||
|
if (!change) return;
|
||||||
|
overlays.setControlHintsHidden(change.newValue === true);
|
||||||
|
renderAll();
|
||||||
|
};
|
||||||
|
chrome.storage.onChanged.addListener(onStorageChange);
|
||||||
|
|
||||||
ui.mount();
|
ui.mount();
|
||||||
chrome.runtime.onMessage.addListener(onMessage);
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
void syncAgentOverlay();
|
void requestOverlayState();
|
||||||
|
|
||||||
window.addEventListener("pageshow", onPageShow);
|
window.addEventListener("pageshow", onPageShow);
|
||||||
|
|
||||||
@@ -344,7 +508,8 @@ export default defineContentScript({
|
|||||||
remountInProgress = true;
|
remountInProgress = true;
|
||||||
try {
|
try {
|
||||||
ui.mount();
|
ui.mount();
|
||||||
void syncAgentOverlay();
|
if (activeAgentState) applyOverlayState(activeAgentState);
|
||||||
|
void requestOverlayState();
|
||||||
} finally {
|
} finally {
|
||||||
remountInProgress = false;
|
remountInProgress = false;
|
||||||
}
|
}
|
||||||
@@ -359,6 +524,7 @@ export default defineContentScript({
|
|||||||
ctx.onInvalidated(() => {
|
ctx.onInvalidated(() => {
|
||||||
hostObserver.disconnect();
|
hostObserver.disconnect();
|
||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||||
window.removeEventListener("pageshow", onPageShow);
|
window.removeEventListener("pageshow", onPageShow);
|
||||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||||
recordCapture?.dispose();
|
recordCapture?.dispose();
|
||||||
@@ -367,16 +533,3 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
async function anySessionLive(): Promise<boolean> {
|
|
||||||
if (!chrome.storage?.session?.get) return true;
|
|
||||||
try {
|
|
||||||
const result = (await chrome.storage.session.get({
|
|
||||||
[SESSIONS_LIVE_FLAG_KEY]: false,
|
|
||||||
})) as Record<string, unknown> | undefined;
|
|
||||||
return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]);
|
|
||||||
} catch (err) {
|
|
||||||
console.debug("[bsk overlay] sessions-live flag read failed", err);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { SnapshotInfo } from "@/lib/connection-controller";
|
import type { SnapshotInfo } from "@/lib/connection-controller";
|
||||||
|
import { STORAGE_KEYS } from "@/lib/instance-id";
|
||||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import { App } from "./App";
|
import { App } from "./App";
|
||||||
import { useConnectionState } from "./use-connection-state";
|
import { useConnectionState } from "./use-connection-state";
|
||||||
@@ -244,3 +245,94 @@ describe("App", () => {
|
|||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("control hints toggle", () => {
|
||||||
|
function stubChromeStorage(initial: Record<string, unknown> = {}) {
|
||||||
|
const store = { ...initial };
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { lastError: undefined },
|
||||||
|
storage: {
|
||||||
|
local: {
|
||||||
|
get: (keys: string | string[], cb: (items: Record<string, unknown>) => void) => {
|
||||||
|
const items: Record<string, unknown> = {};
|
||||||
|
for (const k of Array.isArray(keys) ? keys : [keys]) {
|
||||||
|
if (k in store) items[k] = store[k];
|
||||||
|
}
|
||||||
|
cb(items);
|
||||||
|
},
|
||||||
|
set: (items: Record<string, unknown>, cb?: () => void) => {
|
||||||
|
Object.assign(store, items);
|
||||||
|
cb?.();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
onChanged: {
|
||||||
|
addListener: vi.fn(),
|
||||||
|
removeListener: vi.fn(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return store;
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the control hints toggle on when no preference is stored", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reflects the stored hidden preference", async () => {
|
||||||
|
stubChromeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
await waitFor(() => expect(toggle.getAttribute("aria-checked")).toBe("false"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists the hidden preference when the toggle is turned off", async () => {
|
||||||
|
const store = stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
fireEvent.click(toggle);
|
||||||
|
|
||||||
|
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("false");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the hint copy in an accessible info tooltip", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const info = await screen.findByRole("button", { name: "控制提示说明" });
|
||||||
|
expect(info).toBeTruthy();
|
||||||
|
const tooltip = screen.getByRole("tooltip");
|
||||||
|
expect(tooltip.textContent).toBe("Agent 控制页面时显示提示条和橙色闪光。");
|
||||||
|
// Hidden until the info button is hovered or focused.
|
||||||
|
expect(tooltip.className).toContain("opacity-0");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses the same switch component and size for both settings rows", async () => {
|
||||||
|
stubChromeStorage();
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const hintsToggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||||
|
const connectionToggle = screen.getByRole("switch", { name: "BrowserSkill 连接" });
|
||||||
|
// One shared Switch component, one size — hierarchy comes from copy and
|
||||||
|
// the info icon, not control size. Both rows default to checked, so the
|
||||||
|
// class strings must be identical.
|
||||||
|
expect(hintsToggle.className).toContain("h-5 w-9");
|
||||||
|
expect(hintsToggle.className).toBe(connectionToggle.className);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,17 +1,20 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
import { Badge, Button, cn, Input, Label } from "@browser-skill/ui";
|
import { Badge, Button, Input, Label } from "@browser-skill/ui";
|
||||||
import {
|
import {
|
||||||
RiApps2Line,
|
|
||||||
RiArrowLeftLine,
|
RiArrowLeftLine,
|
||||||
RiArrowRightSLine,
|
RiArrowRightSLine,
|
||||||
RiCheckLine,
|
RiCheckLine,
|
||||||
RiFileCopyLine,
|
RiFileCopyLine,
|
||||||
|
RiInformationLine,
|
||||||
} from "@remixicon/react";
|
} from "@remixicon/react";
|
||||||
import { type ChangeEvent, useEffect, useState } from "react";
|
import { type ChangeEvent, useEffect, useState } from "react";
|
||||||
import { PROTOCOL_VERSION } from "@/transport/handshake";
|
import { PROTOCOL_VERSION } from "@/transport/handshake";
|
||||||
|
import functionIconUrl from "../../../assets/function.svg";
|
||||||
import { ConnectionStatusIndicator } from "./connection-status-indicator";
|
import { ConnectionStatusIndicator } from "./connection-status-indicator";
|
||||||
import { POPUP_FEATURES, type PopupView } from "./features";
|
import { POPUP_FEATURES, type PopupView } from "./features";
|
||||||
|
import { Switch } from "./switch";
|
||||||
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
||||||
|
import { useControlHintsHidden } from "./use-control-hints-hidden";
|
||||||
|
|
||||||
const STATE_LABEL_KEYS = {
|
const STATE_LABEL_KEYS = {
|
||||||
disconnected: "popup.stateLabel.disconnected",
|
disconnected: "popup.stateLabel.disconnected",
|
||||||
@@ -37,6 +40,7 @@ function getLogoSrc() {
|
|||||||
export function App() {
|
export function App() {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
||||||
|
const [controlHintsHidden, setControlHintsHidden] = useControlHintsHidden();
|
||||||
const [view, setView] = useState<PopupView>("main");
|
const [view, setView] = useState<PopupView>("main");
|
||||||
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
||||||
const [purposeDraft, setPurposeDraft] = useState("");
|
const [purposeDraft, setPurposeDraft] = useState("");
|
||||||
@@ -156,12 +160,17 @@ export function App() {
|
|||||||
type="button"
|
type="button"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
size="icon"
|
size="icon"
|
||||||
className="size-7 shrink-0 rounded-md"
|
className="group size-8 shrink-0 rounded-md"
|
||||||
aria-label={t("popup.launcher.title")}
|
aria-label={t("popup.launcher.title")}
|
||||||
onClick={() => setView("features")}
|
onClick={() => setView("features")}
|
||||||
data-slot="popup-launcher"
|
data-slot="popup-launcher"
|
||||||
>
|
>
|
||||||
<RiApps2Line className="size-4" aria-hidden />
|
<img
|
||||||
|
src={functionIconUrl}
|
||||||
|
alt=""
|
||||||
|
className="size-6 opacity-35 transition-opacity group-hover:opacity-100 group-focus-visible:opacity-100"
|
||||||
|
data-slot="popup-launcher-icon"
|
||||||
|
/>
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
</header>
|
</header>
|
||||||
@@ -187,26 +196,12 @@ export function App() {
|
|||||||
>
|
>
|
||||||
{t(STATE_BADGE_KEYS[statusState])}
|
{t(STATE_BADGE_KEYS[statusState])}
|
||||||
</Badge>
|
</Badge>
|
||||||
<button
|
<Switch
|
||||||
type="button"
|
checked={snapshot.connectionEnabled}
|
||||||
role="switch"
|
onCheckedChange={setConnectionEnabled}
|
||||||
aria-checked={snapshot.connectionEnabled}
|
|
||||||
aria-label={t("popup.connectionToggleTitle")}
|
aria-label={t("popup.connectionToggleTitle")}
|
||||||
data-slot="popup-connection-toggle"
|
data-slot="popup-connection-toggle"
|
||||||
className={cn(
|
/>
|
||||||
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
|
||||||
snapshot.connectionEnabled ? "bg-primary" : "bg-muted",
|
|
||||||
)}
|
|
||||||
onClick={() => setConnectionEnabled(!snapshot.connectionEnabled)}
|
|
||||||
>
|
|
||||||
<span
|
|
||||||
className={cn(
|
|
||||||
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
|
||||||
snapshot.connectionEnabled ? "translate-x-4" : "translate-x-0.5",
|
|
||||||
)}
|
|
||||||
aria-hidden
|
|
||||||
/>
|
|
||||||
</button>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
@@ -222,6 +217,41 @@ export function App() {
|
|||||||
)}
|
)}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section
|
||||||
|
className="rounded-xl border border-border/80 bg-card/60 px-3 py-2.5"
|
||||||
|
data-slot="popup-control-hints-card"
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between gap-2">
|
||||||
|
<span className="flex min-w-0 items-center gap-1">
|
||||||
|
<span className="truncate text-sm font-medium">
|
||||||
|
{t("popup.controlHintsToggleTitle")}
|
||||||
|
</span>
|
||||||
|
<span className="group relative inline-flex shrink-0">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
aria-label={t("popup.controlHintsInfoLabel")}
|
||||||
|
data-slot="popup-control-hints-info"
|
||||||
|
className="flex size-4 items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50"
|
||||||
|
>
|
||||||
|
<RiInformationLine className="size-3.5" aria-hidden />
|
||||||
|
</button>
|
||||||
|
<span
|
||||||
|
role="tooltip"
|
||||||
|
className="pointer-events-none absolute bottom-full left-0 z-10 mb-1.5 w-56 whitespace-normal rounded-md bg-foreground/65 px-2 py-1 text-[10px] font-medium leading-snug text-background opacity-0 shadow-md backdrop-blur-sm transition-opacity group-hover:opacity-100 group-focus-within:opacity-100"
|
||||||
|
>
|
||||||
|
{t("popup.controlHintsToggleHint")}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
<Switch
|
||||||
|
checked={!controlHintsHidden}
|
||||||
|
onCheckedChange={(shown) => setControlHintsHidden(!shown)}
|
||||||
|
aria-label={t("popup.controlHintsToggleTitle")}
|
||||||
|
data-slot="popup-control-hints-toggle"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
{snapshot.lastError && (
|
{snapshot.lastError && (
|
||||||
<div
|
<div
|
||||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { Switch } from "./switch";
|
||||||
|
|
||||||
|
describe("Switch", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders switch semantics with the checked state", () => {
|
||||||
|
render(<Switch checked={true} onCheckedChange={vi.fn()} aria-label="开关" />);
|
||||||
|
|
||||||
|
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||||
|
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||||
|
expect(toggle.className).toContain("h-5 w-9");
|
||||||
|
expect(toggle.className).toContain("bg-primary");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("calls onCheckedChange with the negated state when clicked", () => {
|
||||||
|
const onCheckedChange = vi.fn();
|
||||||
|
render(<Switch checked={false} onCheckedChange={onCheckedChange} aria-label="开关" />);
|
||||||
|
|
||||||
|
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||||
|
expect(toggle.className).toContain("bg-muted");
|
||||||
|
|
||||||
|
fireEvent.click(toggle);
|
||||||
|
expect(onCheckedChange).toHaveBeenCalledWith(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { cn } from "@browser-skill/ui";
|
||||||
|
|
||||||
|
export interface SwitchProps {
|
||||||
|
checked: boolean;
|
||||||
|
onCheckedChange: (checked: boolean) => void;
|
||||||
|
"aria-label": string;
|
||||||
|
"data-slot"?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The popup's single toggle switch — every settings row uses this same
|
||||||
|
* component and size; hierarchy between primary and secondary settings is
|
||||||
|
* conveyed by copy/iconography, not by control size.
|
||||||
|
*/
|
||||||
|
export function Switch({
|
||||||
|
checked,
|
||||||
|
onCheckedChange,
|
||||||
|
"aria-label": ariaLabel,
|
||||||
|
"data-slot": dataSlot,
|
||||||
|
}: SwitchProps) {
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="switch"
|
||||||
|
aria-checked={checked}
|
||||||
|
aria-label={ariaLabel}
|
||||||
|
data-slot={dataSlot}
|
||||||
|
className={cn(
|
||||||
|
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
||||||
|
checked ? "bg-primary" : "bg-muted",
|
||||||
|
)}
|
||||||
|
onClick={() => onCheckedChange(!checked)}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
className={cn(
|
||||||
|
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
||||||
|
checked ? "translate-x-4" : "translate-x-0.5",
|
||||||
|
)}
|
||||||
|
aria-hidden
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { getControlHintsHidden, STORAGE_KEYS, setControlHintsHidden } from "@/lib/instance-id";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Popup-side view of the "hide control hints" preference in
|
||||||
|
* `chrome.storage.local`. Defaults to shown when storage is unavailable
|
||||||
|
* (e.g. unit tests) so the popup still renders; writes go straight to
|
||||||
|
* storage and content scripts pick them up via `storage.onChanged`.
|
||||||
|
*/
|
||||||
|
export function useControlHintsHidden(): [boolean, (hidden: boolean) => void] {
|
||||||
|
const [hidden, setHidden] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (typeof chrome === "undefined" || !chrome.storage?.local) return undefined;
|
||||||
|
let cancelled = false;
|
||||||
|
getControlHintsHidden()
|
||||||
|
.then((value) => {
|
||||||
|
if (!cancelled) setHidden(value);
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
console.debug("[browser-skill] control-hints preference read failed", err);
|
||||||
|
});
|
||||||
|
const onChanged = (changes: Record<string, chrome.storage.StorageChange>, areaName: string) => {
|
||||||
|
if (areaName !== "local") return;
|
||||||
|
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||||
|
if (change) setHidden(change.newValue === true);
|
||||||
|
};
|
||||||
|
chrome.storage.onChanged.addListener(onChanged);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
chrome.storage.onChanged.removeListener(onChanged);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const update = (value: boolean) => {
|
||||||
|
setHidden(value);
|
||||||
|
if (typeof chrome === "undefined" || !chrome.storage?.local) return;
|
||||||
|
setControlHintsHidden(value).catch((err) => {
|
||||||
|
console.debug("[browser-skill] control-hints preference write failed", err);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return [hidden, update];
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
CAPTURE_SUPPRESS,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
withOverlaysHiddenForCapture,
|
||||||
|
} from "../capture-suppress-bridge";
|
||||||
|
|
||||||
|
function recordingSendToTab(events: string[]) {
|
||||||
|
return vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||||
|
events.push(message.phase);
|
||||||
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("withOverlaysHiddenForCapture", () => {
|
||||||
|
it("sends begin → fn → end and returns fn's result", async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(
|
||||||
|
7,
|
||||||
|
async () => {
|
||||||
|
events.push("capture");
|
||||||
|
return "shot";
|
||||||
|
},
|
||||||
|
sendToTab,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "end" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still sends end when fn throws, then rethrows", async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
withOverlaysHiddenForCapture(
|
||||||
|
7,
|
||||||
|
async () => {
|
||||||
|
events.push("capture");
|
||||||
|
throw new Error("capture exploded");
|
||||||
|
},
|
||||||
|
sendToTab,
|
||||||
|
),
|
||||||
|
).rejects.toThrow("capture exploded");
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures directly when the tab has no content script", async () => {
|
||||||
|
const sendToTab = vi.fn(async () => {
|
||||||
|
throw new Error("Could not establish connection. Receiving end does not exist.");
|
||||||
|
});
|
||||||
|
const fn = vi.fn(async () => "shot");
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(fn).toHaveBeenCalledTimes(1);
|
||||||
|
// begin failed → no matching end may be sent.
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("swallows end failures so the capture result stands", async () => {
|
||||||
|
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||||
|
if (message.phase === "end") throw new Error("tab navigated away");
|
||||||
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { MIN_COMPATIBLE_PROTOCOL } from "../../transport/handshake";
|
import { MIN_COMPATIBLE_PROTOCOL } from "../../transport/handshake";
|
||||||
import type { ConnectionStateHandler, Transport } from "../../transport/transport";
|
import type { ConnectionStateHandler, FrameHandler, Transport } from "../../transport/transport";
|
||||||
import type { ConnectionState, HandshakeResult } from "../../transport/types";
|
import type { ConnectionState, HandshakeResult, ProtocolFrame } from "../../transport/types";
|
||||||
import { __testing__, ConnectionController } from "../connection-controller";
|
import { __testing__, ConnectionController } from "../connection-controller";
|
||||||
|
|
||||||
vi.mock("../instance-id", () => ({
|
vi.mock("../instance-id", () => ({
|
||||||
@@ -97,6 +97,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
||||||
let state = initialState;
|
let state = initialState;
|
||||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||||
|
const messageHandlers = new Set<FrameHandler>();
|
||||||
const transport = {
|
const transport = {
|
||||||
get state() {
|
get state() {
|
||||||
return state;
|
return state;
|
||||||
@@ -110,7 +111,10 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
|||||||
for (const h of stateHandlers) h("disconnected");
|
for (const h of stateHandlers) h("disconnected");
|
||||||
}),
|
}),
|
||||||
send: vi.fn(),
|
send: vi.fn(),
|
||||||
onMessage: vi.fn(() => ({ dispose: () => {} })),
|
onMessage: vi.fn((handler: FrameHandler) => {
|
||||||
|
messageHandlers.add(handler);
|
||||||
|
return { dispose: () => messageHandlers.delete(handler) };
|
||||||
|
}),
|
||||||
onConnectionStateChange: vi.fn((handler: ConnectionStateHandler) => {
|
onConnectionStateChange: vi.fn((handler: ConnectionStateHandler) => {
|
||||||
stateHandlers.add(handler);
|
stateHandlers.add(handler);
|
||||||
return { dispose: () => stateHandlers.delete(handler) };
|
return { dispose: () => stateHandlers.delete(handler) };
|
||||||
@@ -119,6 +123,9 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
|||||||
state = next;
|
state = next;
|
||||||
for (const h of stateHandlers) h(next);
|
for (const h of stateHandlers) h(next);
|
||||||
},
|
},
|
||||||
|
emitMessage(frame: ProtocolFrame) {
|
||||||
|
for (const handler of messageHandlers) handler(frame);
|
||||||
|
},
|
||||||
};
|
};
|
||||||
return transport as typeof transport & Transport;
|
return transport as typeof transport & Transport;
|
||||||
}
|
}
|
||||||
@@ -128,6 +135,10 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
it("does not connect on attach when connection is disabled", async () => {
|
it("does not connect on attach when connection is disabled", async () => {
|
||||||
const controller = new ConnectionController();
|
const controller = new ConnectionController();
|
||||||
const transport = makeMockTransport();
|
const transport = makeMockTransport();
|
||||||
@@ -152,6 +163,36 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
expect(controller.snapshot().lastError).toBeNull();
|
expect(controller.snapshot().lastError).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("runs safe session cleanup before an intentional disconnect", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
const order: string[] = [];
|
||||||
|
transport.disconnect.mockImplementation(async () => {
|
||||||
|
order.push("disconnect");
|
||||||
|
});
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||||
|
beforeDisconnect: async () => {
|
||||||
|
order.push("cleanup");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await controller.setConnectionEnabled(false);
|
||||||
|
|
||||||
|
expect(order).toEqual(["cleanup", "disconnect"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs session cleanup when the transport disconnects unexpectedly", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
const onDisconnected = vi.fn(async () => {});
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||||
|
onDisconnected,
|
||||||
|
});
|
||||||
|
|
||||||
|
transport.emitState("disconnected");
|
||||||
|
await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
|
||||||
|
});
|
||||||
|
|
||||||
it("reconnects when connection is re-enabled", async () => {
|
it("reconnects when connection is re-enabled", async () => {
|
||||||
const controller = new ConnectionController();
|
const controller = new ConnectionController();
|
||||||
const transport = makeMockTransport();
|
const transport = makeMockTransport();
|
||||||
@@ -171,4 +212,41 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
|
|
||||||
expect(controller.snapshot().state).toBe("disconnected");
|
expect(controller.snapshot().state).toBe("disconnected");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("disconnects and retries when handshake fails while the socket is still open", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||||
|
const request = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||||
|
|
||||||
|
transport.emitMessage({
|
||||||
|
id: request.id,
|
||||||
|
error: { code: "protocol_error", message: "bad handshake" },
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(transport.disconnect).toHaveBeenCalledTimes(1));
|
||||||
|
expect(controller.snapshot().state).toBe("disconnected");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
expect(transport.connect).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("binds each handshake to the connection that initiated it", async () => {
|
||||||
|
const controller = new ConnectionController();
|
||||||
|
const transport = makeMockTransport();
|
||||||
|
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||||
|
const first = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||||
|
|
||||||
|
transport.emitState("disconnected");
|
||||||
|
transport.emitState("connected");
|
||||||
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
|
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
|
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||||
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { ProtocolFrame } from "@/transport/types";
|
||||||
|
import { HEARTBEAT_EVENT, HEARTBEAT_INTERVAL_MS, startHeartbeat } from "../heartbeat";
|
||||||
|
|
||||||
|
function makeTimers() {
|
||||||
|
let nextId = 1;
|
||||||
|
const intervals = new Map<number, { cb: () => void; ms: number }>();
|
||||||
|
return {
|
||||||
|
intervals,
|
||||||
|
api: {
|
||||||
|
setInterval: (cb: () => void, ms: number) => {
|
||||||
|
const id = nextId++;
|
||||||
|
intervals.set(id, { cb, ms });
|
||||||
|
return id;
|
||||||
|
},
|
||||||
|
clearInterval: (id: number) => {
|
||||||
|
intervals.delete(id);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
tick: (id: number) => intervals.get(id)?.cb(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drives `onActiveChange` subscribers so a test can flip the link state. */
|
||||||
|
function makeActiveSource() {
|
||||||
|
const listeners = new Set<(active: boolean) => void>();
|
||||||
|
return {
|
||||||
|
subscribe: (cb: (active: boolean) => void) => {
|
||||||
|
// Mirror the controller contract: fire immediately with the current
|
||||||
|
// value (starts inactive), then on every change.
|
||||||
|
cb(false);
|
||||||
|
listeners.add(cb);
|
||||||
|
return () => listeners.delete(cb);
|
||||||
|
},
|
||||||
|
set: (active: boolean) => {
|
||||||
|
for (const cb of listeners) cb(active);
|
||||||
|
},
|
||||||
|
listenerCount: () => listeners.size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("startHeartbeat", () => {
|
||||||
|
it("does not beat while the link is inactive", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
const send = vi.fn();
|
||||||
|
startHeartbeat({ send, onActiveChange: active.subscribe, timers: timers.api });
|
||||||
|
|
||||||
|
expect(timers.intervals.size).toBe(0);
|
||||||
|
expect(send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts a 20s interval on activation and sends heartbeat events", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
const send = vi.fn();
|
||||||
|
startHeartbeat({ send, onActiveChange: active.subscribe, timers: timers.api });
|
||||||
|
|
||||||
|
active.set(true);
|
||||||
|
expect(timers.intervals.size).toBe(1);
|
||||||
|
const [id, info] = [...timers.intervals.entries()][0];
|
||||||
|
expect(info.ms).toBe(HEARTBEAT_INTERVAL_MS);
|
||||||
|
|
||||||
|
timers.tick(id);
|
||||||
|
expect(send).toHaveBeenCalledWith({ event: HEARTBEAT_EVENT, payload: {} });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops beating when the link goes inactive", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
const send = vi.fn();
|
||||||
|
startHeartbeat({ send, onActiveChange: active.subscribe, timers: timers.api });
|
||||||
|
|
||||||
|
active.set(true);
|
||||||
|
expect(timers.intervals.size).toBe(1);
|
||||||
|
active.set(false);
|
||||||
|
expect(timers.intervals.size).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not stack intervals on repeated activation", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
startHeartbeat({ send: vi.fn(), onActiveChange: active.subscribe, timers: timers.api });
|
||||||
|
|
||||||
|
active.set(true);
|
||||||
|
active.set(true);
|
||||||
|
expect(timers.intervals.size).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("swallows send errors so the interval keeps running", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
const send = vi.fn(() => {
|
||||||
|
throw new Error("socket closed");
|
||||||
|
});
|
||||||
|
const handle = startHeartbeat({ send, onActiveChange: active.subscribe, timers: timers.api });
|
||||||
|
expect(() => handle.beatForTest()).not.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("dispose clears the interval and unsubscribes", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
const handle = startHeartbeat({
|
||||||
|
send: vi.fn(),
|
||||||
|
onActiveChange: active.subscribe,
|
||||||
|
timers: timers.api,
|
||||||
|
});
|
||||||
|
active.set(true);
|
||||||
|
expect(timers.intervals.size).toBe(1);
|
||||||
|
|
||||||
|
handle.dispose();
|
||||||
|
expect(timers.intervals.size).toBe(0);
|
||||||
|
expect(active.listenerCount()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sends a well-formed protocol frame", () => {
|
||||||
|
const timers = makeTimers();
|
||||||
|
const active = makeActiveSource();
|
||||||
|
let captured: ProtocolFrame | null = null;
|
||||||
|
const handle = startHeartbeat({
|
||||||
|
send: (f) => {
|
||||||
|
captured = f;
|
||||||
|
},
|
||||||
|
onActiveChange: active.subscribe,
|
||||||
|
timers: timers.api,
|
||||||
|
});
|
||||||
|
handle.beatForTest();
|
||||||
|
expect(captured).toEqual({ event: HEARTBEAT_EVENT, payload: {} });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,10 +1,17 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
HELP_RESPONSE,
|
||||||
|
isHelpAckMessage,
|
||||||
isHelpCancelMessage,
|
isHelpCancelMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
isHelpRequestMessage,
|
isHelpRequestMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "../help-bridge";
|
} from "../help-bridge";
|
||||||
|
|
||||||
describe("help-bridge", () => {
|
describe("help-bridge", () => {
|
||||||
@@ -12,6 +19,9 @@ describe("help-bridge", () => {
|
|||||||
expect(HELP_REQUEST).toBe("bsk-help-request");
|
expect(HELP_REQUEST).toBe("bsk-help-request");
|
||||||
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
expect(HELP_RESPONSE).toBe("bsk-help-response");
|
||||||
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
expect(HELP_CANCEL).toBe("bsk-help-cancel");
|
||||||
|
expect(HELP_ACK).toBe("bsk-help-ack");
|
||||||
|
expect(HELP_QUERY).toBe("bsk-help-query");
|
||||||
|
expect(HELP_FINISH).toBe("bsk-help-finish");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("type-guards a help-request message", () => {
|
it("type-guards a help-request message", () => {
|
||||||
@@ -20,7 +30,9 @@ describe("help-bridge", () => {
|
|||||||
type: HELP_REQUEST,
|
type: HELP_REQUEST,
|
||||||
requestId: "r1",
|
requestId: "r1",
|
||||||
prompt: "log in",
|
prompt: "log in",
|
||||||
|
displayMode: "compact",
|
||||||
selectors: ["#login"],
|
selectors: ["#login"],
|
||||||
|
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||||
timeoutMs: 1000,
|
timeoutMs: 1000,
|
||||||
}),
|
}),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
@@ -55,4 +67,18 @@ describe("help-bridge", () => {
|
|||||||
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
expect(isHelpCancelMessage({ type: HELP_REQUEST, requestId: "r1" })).toBe(false);
|
||||||
expect(isHelpCancelMessage(null)).toBe(false);
|
expect(isHelpCancelMessage(null)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("type-guards lifecycle messages", () => {
|
||||||
|
expect(isHelpAckMessage({ type: HELP_ACK, ok: true })).toBe(true);
|
||||||
|
expect(isHelpQueryMessage({ type: HELP_QUERY })).toBe(true);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "continued" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpResponseMessage({ type: HELP_RESPONSE, outcome: "cancelled", note: "no" })).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(isHelpFinishMessage({ type: HELP_FINISH, requestId: "r1", outcome: "weird" })).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { evaluateHoverTrigger } from "../hover-trigger-policy";
|
||||||
|
|
||||||
|
describe("evaluateHoverTrigger", () => {
|
||||||
|
it("accepts explicit popup triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Open user navigation menu",
|
||||||
|
attrs: { "aria-haspopup": "menu", "aria-expanded": "false" },
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(
|
||||||
|
expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image buttons without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts compact topbar image links without popup attributes", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "image",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 900, y: 8, w: 32, h: 32 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
hasGraphicDescendant: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(true);
|
||||||
|
expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects topbar menu links without surface trigger evidence", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects text navigation items even when their attributes contain popup words", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "a",
|
||||||
|
role: "link",
|
||||||
|
label: "My profile",
|
||||||
|
attrs: { class: "dropdown-item profile-link" },
|
||||||
|
rect: { x: 820, y: 72, w: 96, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects labelled dropdown list items as hover triggers", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "li",
|
||||||
|
label: "文档C+D",
|
||||||
|
attrs: { class: "dropdown-item", tabindex: "0" },
|
||||||
|
rect: { x: 820, y: 48, w: 160, h: 32 },
|
||||||
|
cursor: "pointer",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ordinary controls without hover popup signals", () => {
|
||||||
|
const decision = evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
label: "Search",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 100, y: 240, w: 120, h: 36 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision.eligible).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unsafe or non-visible triggers", () => {
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
evaluateHoverTrigger({
|
||||||
|
tag: "button",
|
||||||
|
role: "button",
|
||||||
|
attrs: { hidden: "" },
|
||||||
|
rect: { x: 10, y: 10, w: 80, h: 24 },
|
||||||
|
pointerEvents: "auto",
|
||||||
|
}).eligible,
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,10 +1,12 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import {
|
import {
|
||||||
getConnectionEnabled,
|
getConnectionEnabled,
|
||||||
|
getControlHintsHidden,
|
||||||
getLabel,
|
getLabel,
|
||||||
getOrCreateInstanceId,
|
getOrCreateInstanceId,
|
||||||
STORAGE_KEYS,
|
STORAGE_KEYS,
|
||||||
setConnectionEnabled,
|
setConnectionEnabled,
|
||||||
|
setControlHintsHidden,
|
||||||
setLabel,
|
setLabel,
|
||||||
} from "../instance-id";
|
} from "../instance-id";
|
||||||
|
|
||||||
@@ -93,4 +95,26 @@ describe("instance-id", () => {
|
|||||||
expect(store[STORAGE_KEYS.CONNECTION_ENABLED]).toBe(false);
|
expect(store[STORAGE_KEYS.CONNECTION_ENABLED]).toBe(false);
|
||||||
expect(await getConnectionEnabled(backend)).toBe(false);
|
expect(await getConnectionEnabled(backend)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden returns false when storage is empty", async () => {
|
||||||
|
const { backend } = fakeStorage();
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden returns persisted boolean values", async () => {
|
||||||
|
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getControlHintsHidden treats non-boolean stored values as shown", async () => {
|
||||||
|
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: "true" });
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("setControlHintsHidden persists the value retrievable by getControlHintsHidden", async () => {
|
||||||
|
const { backend, store } = fakeStorage();
|
||||||
|
await setControlHintsHidden(true, backend);
|
||||||
|
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||||
|
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -104,6 +104,27 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps committed empty fill steps", () => {
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
target: { tag: "input", role: "textbox", name: "Search query" },
|
||||||
|
value: "",
|
||||||
|
page_url: "https://example.com/search",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"https://example.com/search",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
op: "fill",
|
||||||
|
value: "",
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
const { pages, steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
@@ -130,6 +151,30 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "span", role: "button", name: "Account" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { tag: "a", role: "link", name: "Profile" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"https://example.com/app",
|
||||||
|
);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
page: "p1",
|
||||||
|
target: { name: "Account" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
resolveTraceStartUrl(
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
/**
|
||||||
|
* Wire protocol and background-side helper for hiding the extension's
|
||||||
|
* in-page overlay while a screenshot is captured, so captured frames only
|
||||||
|
* contain the page itself.
|
||||||
|
*
|
||||||
|
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||||
|
* which sends `begin` (the content script hides the overlay host and only
|
||||||
|
* acks once the compositor has produced an overlay-free frame) and always
|
||||||
|
* follows up with `end`, even when the capture throws. Tabs without the
|
||||||
|
* content script (chrome://, the Web Store, ...) host no overlay, so a
|
||||||
|
* failed `begin` falls through to capturing directly.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const CAPTURE_SUPPRESS = "bsk/capture-suppress";
|
||||||
|
|
||||||
|
export type CaptureSuppressPhase = "begin" | "end";
|
||||||
|
|
||||||
|
export interface CaptureSuppressMessage {
|
||||||
|
type: typeof CAPTURE_SUPPRESS;
|
||||||
|
phase: CaptureSuppressPhase;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureSuppressAck {
|
||||||
|
type: typeof CAPTURE_SUPPRESS;
|
||||||
|
ok: true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isCaptureSuppressMessage(msg: unknown): msg is CaptureSuppressMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === CAPTURE_SUPPRESS && (m.phase === "begin" || m.phase === "end");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Minimal `chrome.tabs.sendMessage` surface so tests can inject a fake. */
|
||||||
|
export type CaptureSuppressSendToTab = (
|
||||||
|
tabId: number,
|
||||||
|
message: CaptureSuppressMessage,
|
||||||
|
) => Promise<unknown>;
|
||||||
|
|
||||||
|
const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, message);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run `fn` with the target tab's overlay hidden. `end` is sent from a
|
||||||
|
* `finally` so the overlay always reappears, and a missing content script
|
||||||
|
* (or any other `begin` failure) simply skips suppression — there is no
|
||||||
|
* overlay to hide in that tab.
|
||||||
|
*/
|
||||||
|
export async function withOverlaysHiddenForCapture<T>(
|
||||||
|
tabId: number,
|
||||||
|
fn: () => Promise<T>,
|
||||||
|
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||||
|
): Promise<T> {
|
||||||
|
let began = false;
|
||||||
|
try {
|
||||||
|
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||||
|
began = true;
|
||||||
|
} catch {
|
||||||
|
// No content script in the target tab → no overlay can leak into the
|
||||||
|
// capture; proceed without suppression.
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} finally {
|
||||||
|
if (began) {
|
||||||
|
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "end" }).catch((err) => {
|
||||||
|
console.debug("[bsk capture] overlay restore failed", err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ import type { ConnectionState, HandshakeResult } from "../transport/types";
|
|||||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||||
|
|
||||||
|
const HANDSHAKE_RETRY_DELAY_MS = 1_000;
|
||||||
|
|
||||||
export interface SnapshotInfo {
|
export interface SnapshotInfo {
|
||||||
state: ConnectionState;
|
state: ConnectionState;
|
||||||
instanceId: string;
|
instanceId: string;
|
||||||
@@ -21,6 +23,13 @@ export interface SnapshotInfo {
|
|||||||
|
|
||||||
type Listener = (s: SnapshotInfo) => void;
|
type Listener = (s: SnapshotInfo) => void;
|
||||||
|
|
||||||
|
export interface ConnectionLifecycleHooks {
|
||||||
|
/** Safe local teardown that must finish before an intentional disconnect. */
|
||||||
|
beforeDisconnect?: () => void | Promise<void>;
|
||||||
|
/** Best-effort teardown after an unexpected transport loss. */
|
||||||
|
onDisconnected?: () => void | Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Orchestrates Transport + handshake lifecycle for the background SW.
|
* Orchestrates Transport + handshake lifecycle for the background SW.
|
||||||
*
|
*
|
||||||
@@ -36,7 +45,12 @@ export class ConnectionController {
|
|||||||
private lastError: string | null = null;
|
private lastError: string | null = null;
|
||||||
private connectionEnabled = true;
|
private connectionEnabled = true;
|
||||||
private listeners = new Set<Listener>();
|
private listeners = new Set<Listener>();
|
||||||
private handshakeInFlight = false;
|
private lifecycleHooks: ConnectionLifecycleHooks = {};
|
||||||
|
private disconnectRecovery: Promise<void> | null = null;
|
||||||
|
private connectionGeneration = 0;
|
||||||
|
private handshakeAbort: AbortController | null = null;
|
||||||
|
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private suppressDisconnectRecovery = false;
|
||||||
|
|
||||||
get isConnectionEnabled(): boolean {
|
get isConnectionEnabled(): boolean {
|
||||||
return this.connectionEnabled;
|
return this.connectionEnabled;
|
||||||
@@ -68,20 +82,36 @@ export class ConnectionController {
|
|||||||
transport: Transport,
|
transport: Transport,
|
||||||
browser: { name: string; version: string },
|
browser: { name: string; version: string },
|
||||||
connectionEnabled = true,
|
connectionEnabled = true,
|
||||||
|
lifecycleHooks: ConnectionLifecycleHooks = {},
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
this.transport = transport;
|
this.transport = transport;
|
||||||
this.connectionEnabled = connectionEnabled;
|
this.connectionEnabled = connectionEnabled;
|
||||||
|
this.lifecycleHooks = lifecycleHooks;
|
||||||
this.instanceId = await getOrCreateInstanceId();
|
this.instanceId = await getOrCreateInstanceId();
|
||||||
this.label = await getLabel();
|
this.label = await getLabel();
|
||||||
|
|
||||||
transport.onConnectionStateChange((s) => {
|
transport.onConnectionStateChange((s) => {
|
||||||
if (!this.connectionEnabled) return;
|
if (s === "disconnected") {
|
||||||
if (s === "connected") {
|
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||||
void this.runHandshake(browser);
|
// a late resolution can never clobber the next attempt.
|
||||||
|
this.cancelHandshake();
|
||||||
|
this.handshake = null;
|
||||||
|
if (this.connectionEnabled) {
|
||||||
|
this.setState("disconnected");
|
||||||
|
if (this.suppressDisconnectRecovery) {
|
||||||
|
// Deliberate disconnect from runHandshake (rejected peer or failed
|
||||||
|
// handshake bounce); the caller owns the reconnect policy there.
|
||||||
|
this.suppressDisconnectRecovery = false;
|
||||||
|
} else {
|
||||||
|
void this.recoverFromDisconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (s === "disconnected") {
|
if (!this.connectionEnabled) return;
|
||||||
this.handshake = null;
|
if (s === "connected") {
|
||||||
|
this.startHandshake(browser);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
this.setState(s);
|
this.setState(s);
|
||||||
});
|
});
|
||||||
@@ -126,41 +156,69 @@ export class ConnectionController {
|
|||||||
this.fire();
|
this.fire();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runHandshake(browser: { name: string; version: string }): Promise<void> {
|
private startHandshake(browser: { name: string; version: string }): void {
|
||||||
|
this.cancelHandshake();
|
||||||
|
const generation = ++this.connectionGeneration;
|
||||||
|
const abort = new AbortController();
|
||||||
|
this.handshakeAbort = abort;
|
||||||
|
void this.runHandshake(browser, generation, abort.signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runHandshake(
|
||||||
|
browser: { name: string; version: string },
|
||||||
|
generation: number,
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<void> {
|
||||||
if (!this.transport) return;
|
if (!this.transport) return;
|
||||||
if (!this.connectionEnabled) return;
|
if (!this.connectionEnabled) return;
|
||||||
if (this.handshakeInFlight) return;
|
|
||||||
this.handshakeInFlight = true;
|
|
||||||
this.setState("connecting");
|
this.setState("connecting");
|
||||||
try {
|
try {
|
||||||
const outcome = await performHandshake(this.transport, {
|
const outcome = await performHandshake(
|
||||||
instanceId: this.instanceId,
|
this.transport,
|
||||||
browser,
|
{
|
||||||
label: this.label,
|
instanceId: this.instanceId,
|
||||||
});
|
browser,
|
||||||
|
label: this.label,
|
||||||
|
},
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
if (generation !== this.connectionGeneration || signal.aborted) return;
|
||||||
this.handshake = outcome.result;
|
this.handshake = outcome.result;
|
||||||
const verdict = computeConnectedState(outcome.result);
|
const verdict = computeConnectedState(outcome.result);
|
||||||
if (verdict.kind === "rejected") {
|
if (verdict.kind === "rejected") {
|
||||||
this.lastError = `version_too_old: ${verdict.reason}`;
|
this.lastError = `version_too_old: ${verdict.reason}`;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
|
// Rejected peers must not be auto-retried: mark this disconnect as
|
||||||
|
// deliberate so the listener skips unexpected-loss recovery.
|
||||||
|
this.suppressDisconnectRecovery = true;
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
this.clearHandshakeRetry();
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
this.setState(verdict.kind);
|
this.setState(verdict.kind);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||||
|
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||||
|
this.suppressDisconnectRecovery = true;
|
||||||
|
await this.transport.disconnect().catch(() => {});
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
|
this.scheduleHandshakeRetry(browser);
|
||||||
} finally {
|
} finally {
|
||||||
this.handshakeInFlight = false;
|
if (generation === this.connectionGeneration) this.handshakeAbort = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async applyDisabledState(): Promise<void> {
|
private async applyDisabledState(): Promise<void> {
|
||||||
|
this.cancelHandshake();
|
||||||
|
this.clearHandshakeRetry();
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
|
await this.lifecycleHooks.beforeDisconnect?.();
|
||||||
if (this.transport) {
|
if (this.transport) {
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
}
|
}
|
||||||
@@ -169,6 +227,60 @@ export class ConnectionController {
|
|||||||
if (was === "disconnected") this.fire();
|
if (was === "disconnected") this.fire();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private recoverFromDisconnect(): Promise<void> {
|
||||||
|
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||||
|
const recovery = (async () => {
|
||||||
|
// WSTransport schedules its reconnect timer immediately after notifying
|
||||||
|
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||||
|
// is cancelled before local session teardown begins.
|
||||||
|
await Promise.resolve();
|
||||||
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
|
// Another path already reconnected while we yielded — nothing to do.
|
||||||
|
if (this.currentState !== "disconnected") return;
|
||||||
|
await this.transport.disconnect().catch(() => {});
|
||||||
|
try {
|
||||||
|
await this.lifecycleHooks.onDisconnected?.();
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||||
|
}
|
||||||
|
if (!this.connectionEnabled) return;
|
||||||
|
try {
|
||||||
|
await this.transport.connect();
|
||||||
|
} catch (err) {
|
||||||
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
this.setState("disconnected");
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
this.disconnectRecovery = recovery.finally(() => {
|
||||||
|
this.disconnectRecovery = null;
|
||||||
|
});
|
||||||
|
return this.disconnectRecovery;
|
||||||
|
}
|
||||||
|
|
||||||
|
private cancelHandshake(): void {
|
||||||
|
this.connectionGeneration += 1;
|
||||||
|
this.handshakeAbort?.abort();
|
||||||
|
this.handshakeAbort = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||||
|
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||||
|
this.handshakeRetryTimer = setTimeout(() => {
|
||||||
|
this.handshakeRetryTimer = null;
|
||||||
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
|
void this.transport.connect().catch((err) => {
|
||||||
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
|
this.setState("disconnected");
|
||||||
|
});
|
||||||
|
}, HANDSHAKE_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearHandshakeRetry(): void {
|
||||||
|
if (!this.handshakeRetryTimer) return;
|
||||||
|
clearTimeout(this.handshakeRetryTimer);
|
||||||
|
this.handshakeRetryTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
private setState(next: ConnectionState): void {
|
private setState(next: ConnectionState): void {
|
||||||
if (this.currentState === next) return;
|
if (this.currentState === next) return;
|
||||||
this.currentState = next;
|
this.currentState = next;
|
||||||
@@ -187,6 +299,12 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAbortError(err: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verdict of the symmetric post-handshake compat check.
|
* Verdict of the symmetric post-handshake compat check.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveHoverElement(target: Element): Element | null {
|
||||||
|
const clickable = resolveClickableElement(target);
|
||||||
|
if (clickable) return clickable;
|
||||||
|
|
||||||
|
let node: Element | null = target;
|
||||||
|
let depth = 0;
|
||||||
|
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||||
|
if (looksClickable(node)) return node;
|
||||||
|
node = node.parentElement;
|
||||||
|
depth += 1;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
/**
|
||||||
|
* Application-level WebSocket heartbeat.
|
||||||
|
*
|
||||||
|
* Since Chrome 116, sending or receiving a message over a WebSocket
|
||||||
|
* resets the MV3 service-worker idle timer (see
|
||||||
|
* https://developer.chrome.com/docs/extensions/how-to/web-platform/websockets).
|
||||||
|
* By emitting a tiny `system.heartbeat` event every 20s — comfortably
|
||||||
|
* inside the 30s idle window — we keep the worker (and therefore the
|
||||||
|
* daemon link) alive for as long as the connection is up, instead of
|
||||||
|
* relying solely on the 30s keepalive alarm that sits right on the
|
||||||
|
* eviction boundary.
|
||||||
|
*
|
||||||
|
* The daemon also treats the heartbeat as a liveness signal, so a
|
||||||
|
* silently-dead browser can be reaped on its side.
|
||||||
|
*
|
||||||
|
* The heartbeat only runs while the *post-handshake* link is live: the
|
||||||
|
* daemon rejects any first frame that is not `system.handshake`, so
|
||||||
|
* beating before the handshake completes would get the connection
|
||||||
|
* kicked. Callers wire {@link HeartbeatOptions.onActiveChange} to the
|
||||||
|
* connection controller, which only reports `connected` / `version_skew`
|
||||||
|
* after the handshake has landed.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ProtocolFrame } from "@/transport/types";
|
||||||
|
|
||||||
|
export const HEARTBEAT_EVENT = "system.heartbeat";
|
||||||
|
/** 20s: the interval Chrome's own WebSocket keepalive sample recommends. */
|
||||||
|
export const HEARTBEAT_INTERVAL_MS = 20_000;
|
||||||
|
|
||||||
|
/** Subset of the timer API we use; lets tests inject fakes. */
|
||||||
|
export interface HeartbeatTimers {
|
||||||
|
setInterval(cb: () => void, ms: number): number;
|
||||||
|
clearInterval(handle: number): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HeartbeatOptions {
|
||||||
|
/** Send a frame over the live transport. */
|
||||||
|
send: (frame: ProtocolFrame) => void;
|
||||||
|
/**
|
||||||
|
* Subscribe to "is the post-handshake link live" changes. Must invoke
|
||||||
|
* the callback with the current value immediately and again on every
|
||||||
|
* transition, and return an unsubscribe function.
|
||||||
|
*/
|
||||||
|
onActiveChange: (cb: (active: boolean) => void) => () => void;
|
||||||
|
/** Override the beat interval (tests). */
|
||||||
|
intervalMs?: number;
|
||||||
|
/** Inject fake timers (tests). */
|
||||||
|
timers?: HeartbeatTimers;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HeartbeatHandle {
|
||||||
|
/** Stop beating and detach the active-state listener. */
|
||||||
|
dispose(): void;
|
||||||
|
/** Test hook: emit one heartbeat frame right now. */
|
||||||
|
beatForTest(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultTimers(): HeartbeatTimers {
|
||||||
|
return {
|
||||||
|
setInterval: (cb, ms) => setInterval(cb, ms) as unknown as number,
|
||||||
|
clearInterval: (handle) => clearInterval(handle),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a heartbeat that beats every `intervalMs` while the link is
|
||||||
|
* active and pauses entirely while it is not.
|
||||||
|
*/
|
||||||
|
export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
||||||
|
const intervalMs = options.intervalMs ?? HEARTBEAT_INTERVAL_MS;
|
||||||
|
const timers = options.timers ?? defaultTimers();
|
||||||
|
let timer: number | null = null;
|
||||||
|
|
||||||
|
const beat = () => {
|
||||||
|
try {
|
||||||
|
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
||||||
|
} catch (err) {
|
||||||
|
// The socket raced a close between the state change and this
|
||||||
|
// tick. The transport's own reconnect path recovers; swallow so
|
||||||
|
// the interval keeps running (it is cleared on the disconnect
|
||||||
|
// notification anyway).
|
||||||
|
console.debug("[bsk heartbeat] send failed", err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const stop = () => {
|
||||||
|
if (timer !== null) {
|
||||||
|
timers.clearInterval(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const start = () => {
|
||||||
|
if (timer !== null) return;
|
||||||
|
timer = timers.setInterval(beat, intervalMs);
|
||||||
|
};
|
||||||
|
|
||||||
|
const unsubscribe = options.onActiveChange((active) => {
|
||||||
|
if (active) start();
|
||||||
|
else stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
dispose: () => {
|
||||||
|
unsubscribe();
|
||||||
|
stop();
|
||||||
|
},
|
||||||
|
beatForTest: beat,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -6,12 +6,22 @@
|
|||||||
* Mirrors the borrow-confirmation message style (see
|
* Mirrors the borrow-confirmation message style (see
|
||||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||||
* the content script replies once the user clicks Continue / Cancel.
|
* the content script acks display, then later reports Done / Cancel.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const HELP_REQUEST = "bsk-help-request";
|
export const HELP_REQUEST = "bsk-help-request";
|
||||||
export const HELP_RESPONSE = "bsk-help-response";
|
export const HELP_RESPONSE = "bsk-help-response";
|
||||||
export const HELP_CANCEL = "bsk-help-cancel";
|
export const HELP_CANCEL = "bsk-help-cancel";
|
||||||
|
export const HELP_ACK = "bsk-help-ack";
|
||||||
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
|
export interface HelpHighlightRect {
|
||||||
|
top: number;
|
||||||
|
left: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
@@ -19,11 +29,20 @@ export interface HelpRequestMessage {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
/** Custom overlay title; omitted when the extension should use its default. */
|
/** Custom overlay title; omitted when the extension should use its default. */
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Full task UI on the subject tab; compact status UI on related tabs. */
|
||||||
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
|
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||||
|
rects?: HelpHighlightRect[];
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpAckMessage {
|
||||||
|
type: typeof HELP_ACK;
|
||||||
|
ok: true;
|
||||||
|
}
|
||||||
|
|
||||||
export interface HelpResponseMessage {
|
export interface HelpResponseMessage {
|
||||||
type: typeof HELP_RESPONSE;
|
type: typeof HELP_RESPONSE;
|
||||||
outcome: "continued" | "cancelled";
|
outcome: "continued" | "cancelled";
|
||||||
@@ -35,6 +54,22 @@ export interface HelpCancelMessage {
|
|||||||
requestId: string;
|
requestId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryMessage {
|
||||||
|
type: typeof HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpQueryResponse {
|
||||||
|
active: boolean;
|
||||||
|
request?: Omit<HelpRequestMessage, "type">;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HelpFinishMessage {
|
||||||
|
type: typeof HELP_FINISH;
|
||||||
|
requestId: string;
|
||||||
|
outcome: "continued" | "cancelled";
|
||||||
|
note?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||||
if (typeof msg !== "object" || msg === null) {
|
if (typeof msg !== "object" || msg === null) {
|
||||||
return false;
|
return false;
|
||||||
@@ -45,8 +80,19 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
|||||||
typeof m.requestId === "string" &&
|
typeof m.requestId === "string" &&
|
||||||
typeof m.prompt === "string" &&
|
typeof m.prompt === "string" &&
|
||||||
(m.title === undefined || typeof m.title === "string") &&
|
(m.title === undefined || typeof m.title === "string") &&
|
||||||
|
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||||
Array.isArray(m.selectors) &&
|
Array.isArray(m.selectors) &&
|
||||||
m.selectors.every((selector) => typeof selector === "string") &&
|
m.selectors.every((selector) => typeof selector === "string") &&
|
||||||
|
(m.rects === undefined ||
|
||||||
|
(Array.isArray(m.rects) &&
|
||||||
|
m.rects.every(
|
||||||
|
(rect) =>
|
||||||
|
typeof rect === "object" &&
|
||||||
|
rect !== null &&
|
||||||
|
["top", "left", "width", "height"].every((key) =>
|
||||||
|
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||||
|
),
|
||||||
|
))) &&
|
||||||
typeof m.timeoutMs === "number"
|
typeof m.timeoutMs === "number"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -58,3 +104,36 @@ export function isHelpCancelMessage(msg: unknown): msg is HelpCancelMessage {
|
|||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
return m.type === HELP_CANCEL && typeof m.requestId === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isHelpResponseMessage(msg: unknown): msg is HelpResponseMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_RESPONSE &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpAckMessage(msg: unknown): msg is HelpAckMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_ACK && m.ok === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpQueryMessage(msg: unknown): msg is HelpQueryMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === HELP_QUERY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isHelpFinishMessage(msg: unknown): msg is HelpFinishMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === HELP_FINISH &&
|
||||||
|
typeof m.requestId === "string" &&
|
||||||
|
(m.outcome === "continued" || m.outcome === "cancelled") &&
|
||||||
|
(m.note === undefined || typeof m.note === "string")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
export interface HoverTriggerRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
w: number;
|
||||||
|
h: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerSignals {
|
||||||
|
tag: string;
|
||||||
|
role?: string;
|
||||||
|
label?: string;
|
||||||
|
attrs?: Record<string, string | undefined>;
|
||||||
|
rect?: HoverTriggerRect | null;
|
||||||
|
cursor?: string;
|
||||||
|
pointerEvents?: string;
|
||||||
|
hasGraphicDescendant?: boolean;
|
||||||
|
cssHoverMatch?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverTriggerDecision {
|
||||||
|
eligible: boolean;
|
||||||
|
score: number;
|
||||||
|
reasons: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOVER_TRIGGER_MIN_SCORE = 45;
|
||||||
|
const HOVER_POPUP_SIGNAL_RE =
|
||||||
|
/(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/;
|
||||||
|
const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/;
|
||||||
|
|
||||||
|
function attr(signals: HoverTriggerSignals, name: string): string | undefined {
|
||||||
|
return signals.attrs?.[name.toLowerCase()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||||
|
if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true;
|
||||||
|
if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true;
|
||||||
|
return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const attrs = signals.attrs ?? {};
|
||||||
|
const haystack = [
|
||||||
|
attrs.id,
|
||||||
|
attrs.class,
|
||||||
|
attrs["data-testid"],
|
||||||
|
attrs["data-test"],
|
||||||
|
attrs["data-cy"],
|
||||||
|
attrs["aria-label"],
|
||||||
|
attrs["aria-haspopup"],
|
||||||
|
attrs["aria-controls"],
|
||||||
|
attrs.title,
|
||||||
|
signals.role === "button" ? signals.label : undefined,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ")
|
||||||
|
.toLowerCase();
|
||||||
|
return HOVER_POPUP_SIGNAL_RE.test(haystack);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
return (
|
||||||
|
signals.cssHoverMatch === true ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
(attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean {
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
["button", "a", "summary"].includes(tag) ||
|
||||||
|
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
|
||||||
|
attr(signals, "tabindex") !== undefined ||
|
||||||
|
signals.cursor === "pointer" ||
|
||||||
|
attr(signals, "onclick") !== undefined ||
|
||||||
|
attr(signals, "onmouseenter") !== undefined ||
|
||||||
|
attr(signals, "onmouseover") !== undefined ||
|
||||||
|
attr(signals, "aria-haspopup") !== undefined ||
|
||||||
|
attr(signals, "aria-expanded") === "false"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision {
|
||||||
|
const rect = signals.rect;
|
||||||
|
if (
|
||||||
|
!rect ||
|
||||||
|
rect.w <= 0 ||
|
||||||
|
rect.h <= 0 ||
|
||||||
|
signals.pointerEvents === "none" ||
|
||||||
|
attr(signals, "hidden") !== undefined ||
|
||||||
|
attr(signals, "inert") !== undefined ||
|
||||||
|
(attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" ||
|
||||||
|
isUnsafeHoverTrigger(signals)
|
||||||
|
) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const area = rect.w * rect.h;
|
||||||
|
if (area <= 0 || area > 160_000) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
const reasons: string[] = [];
|
||||||
|
let score = 0;
|
||||||
|
const tag = signals.tag.toLowerCase();
|
||||||
|
const role = (signals.role ?? "").toLowerCase();
|
||||||
|
const label = signals.label?.trim();
|
||||||
|
const graphic = signals.hasGraphicDescendant === true;
|
||||||
|
const directInteractive = hasDirectHoverInteractiveSignal(signals);
|
||||||
|
const popupSignal = hasHoverPopupSignal(signals);
|
||||||
|
const isTextNavigationItem =
|
||||||
|
(tag === "a" ||
|
||||||
|
tag === "li" ||
|
||||||
|
role === "link" ||
|
||||||
|
role === "option" ||
|
||||||
|
role.startsWith("menuitem") ||
|
||||||
|
HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) &&
|
||||||
|
!graphic &&
|
||||||
|
!!label &&
|
||||||
|
label.toLowerCase() !== "image";
|
||||||
|
const compactTopbarIcon =
|
||||||
|
(role === "button" || role === "link" || tag === "button" || tag === "a") &&
|
||||||
|
graphic &&
|
||||||
|
rect.y < 120 &&
|
||||||
|
rect.w <= 96 &&
|
||||||
|
rect.h <= 96 &&
|
||||||
|
(!label || label.toLowerCase() === "image");
|
||||||
|
const surfaceTriggerEvidence =
|
||||||
|
(popupSignal && !isTextNavigationItem) ||
|
||||||
|
compactTopbarIcon ||
|
||||||
|
hasStrongHoverExpansionSignal(signals);
|
||||||
|
if (!directInteractive || !surfaceTriggerEvidence) {
|
||||||
|
return { eligible: false, score: 0, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attr(signals, "aria-haspopup") !== undefined) {
|
||||||
|
score += 80;
|
||||||
|
reasons.push("aria-haspopup");
|
||||||
|
}
|
||||||
|
if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") {
|
||||||
|
score += 55;
|
||||||
|
reasons.push("collapsed");
|
||||||
|
}
|
||||||
|
if (popupSignal) {
|
||||||
|
score += 45;
|
||||||
|
reasons.push("popup-signal");
|
||||||
|
}
|
||||||
|
if (tag === "button" || role === "button") {
|
||||||
|
score += 30;
|
||||||
|
reasons.push("button");
|
||||||
|
}
|
||||||
|
if (signals.cursor === "pointer") {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("pointer");
|
||||||
|
}
|
||||||
|
if (graphic && (!label || label.toLowerCase() === "image")) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("icon-only");
|
||||||
|
}
|
||||||
|
if (rect.y < 120) {
|
||||||
|
score += 25;
|
||||||
|
reasons.push("topbar");
|
||||||
|
}
|
||||||
|
if (rect.w <= 80 && rect.h <= 80) {
|
||||||
|
score += 20;
|
||||||
|
reasons.push("compact");
|
||||||
|
}
|
||||||
|
if (signals.cssHoverMatch) {
|
||||||
|
score += 40;
|
||||||
|
reasons.push("css-hover");
|
||||||
|
}
|
||||||
|
|
||||||
|
return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons };
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
const STORAGE_KEY = "bsk_instance_id";
|
const STORAGE_KEY = "bsk_instance_id";
|
||||||
const LABEL_STORAGE_KEY = "bh_label";
|
const LABEL_STORAGE_KEY = "bh_label";
|
||||||
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
||||||
|
const CONTROL_HINTS_HIDDEN_KEY = "bsk_control_hints_hidden";
|
||||||
|
|
||||||
export interface StorageBackend {
|
export interface StorageBackend {
|
||||||
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
||||||
@@ -105,8 +106,30 @@ export async function setConnectionEnabled(
|
|||||||
await storage.set({ [CONNECTION_ENABLED_KEY]: enabled });
|
await storage.set({ [CONNECTION_ENABLED_KEY]: enabled });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User preference for the in-page control hints (status pill + orange glow
|
||||||
|
* shown while the Agent controls a tab). Defaults to shown when unset or
|
||||||
|
* non-boolean; when hidden the whole control overlay — including its input
|
||||||
|
* blocker — is skipped so the page looks and behaves normally.
|
||||||
|
*/
|
||||||
|
export async function getControlHintsHidden(
|
||||||
|
storage: StorageBackend = defaultStorage(),
|
||||||
|
): Promise<boolean> {
|
||||||
|
const items = await storage.get(CONTROL_HINTS_HIDDEN_KEY);
|
||||||
|
const raw = items[CONTROL_HINTS_HIDDEN_KEY];
|
||||||
|
return typeof raw === "boolean" ? raw : false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function setControlHintsHidden(
|
||||||
|
hidden: boolean,
|
||||||
|
storage: StorageBackend = defaultStorage(),
|
||||||
|
): Promise<void> {
|
||||||
|
await storage.set({ [CONTROL_HINTS_HIDDEN_KEY]: hidden });
|
||||||
|
}
|
||||||
|
|
||||||
export const STORAGE_KEYS = {
|
export const STORAGE_KEYS = {
|
||||||
INSTANCE_ID: STORAGE_KEY,
|
INSTANCE_ID: STORAGE_KEY,
|
||||||
LABEL: LABEL_STORAGE_KEY,
|
LABEL: LABEL_STORAGE_KEY,
|
||||||
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
||||||
|
CONTROL_HINTS_HIDDEN: CONTROL_HINTS_HIDDEN_KEY,
|
||||||
} as const;
|
} as const;
|
||||||
|
|||||||
@@ -3,9 +3,8 @@
|
|||||||
* between the content-script control overlay and the background SW.
|
* between the content-script control overlay and the background SW.
|
||||||
*
|
*
|
||||||
* Content script → background:
|
* Content script → background:
|
||||||
* - `{ kind: "overlay.who_am_i", tabId, windowId }` → background
|
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||||
* replies `{ sessionId: string | null }` so the overlay knows
|
* overlay state for the sender's window.
|
||||||
* whether to render itself.
|
|
||||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||||
* every inflight + queued tool call for that session with
|
* every inflight + queued tool call for that session with
|
||||||
@@ -14,8 +13,40 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
|
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
|
||||||
|
export const OVERLAY_MSG_READY = "overlay.ready";
|
||||||
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
|
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* WXT shadow-host element name (`createShadowRootUi({ name })`) and the marker
|
||||||
|
* attribute set on the host for the agent's own overlay UI. The VOM capture
|
||||||
|
* adapter uses these to skip the overlay host and its shadow subtree.
|
||||||
|
*/
|
||||||
|
export const OVERLAY_HOST_NAME = "browser-skill-overlay";
|
||||||
|
export const OVERLAY_HOST_MARKER_ATTR = "data-bsk-overlay";
|
||||||
|
|
||||||
|
/** CSS selector for the WXT shadow host (tag + marker attribute). */
|
||||||
|
export const OVERLAY_HOST_SELECTOR = `${OVERLAY_HOST_NAME}, [${OVERLAY_HOST_MARKER_ATTR}]`;
|
||||||
|
|
||||||
|
export function isOverlayHostElementName(tagName: string): boolean {
|
||||||
|
return tagName.toLowerCase() === OVERLAY_HOST_NAME;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOverlayHostMarkerAttribute(attrName: string): boolean {
|
||||||
|
return attrName.toLowerCase() === OVERLAY_HOST_MARKER_ATTR;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOverlayHostNode(tagName: string, attributeNames?: Iterable<string>): boolean {
|
||||||
|
if (isOverlayHostElementName(tagName)) return true;
|
||||||
|
if (!attributeNames) return false;
|
||||||
|
for (const name of attributeNames) {
|
||||||
|
if (isOverlayHostMarkerAttribute(name)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Page-world `document.querySelector(...)` for the overlay shadow host. */
|
||||||
|
export const OVERLAY_HOST_LOOKUP_EXPR = `document.querySelector(${JSON.stringify(OVERLAY_HOST_SELECTOR)})`;
|
||||||
|
|
||||||
export interface OverlayWhoAmIRequest {
|
export interface OverlayWhoAmIRequest {
|
||||||
kind: typeof OVERLAY_MSG_WHO_AM_I;
|
kind: typeof OVERLAY_MSG_WHO_AM_I;
|
||||||
tabId?: number;
|
tabId?: number;
|
||||||
@@ -26,6 +57,22 @@ export interface OverlayWhoAmIResponse {
|
|||||||
sessionId: string | null;
|
sessionId: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface OverlayReadyRequest {
|
||||||
|
kind: typeof OVERLAY_MSG_READY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type OverlayMode = "control" | "interrupting" | "paused" | "hidden";
|
||||||
|
|
||||||
|
/** Background → content: complete, authoritative control-overlay state. */
|
||||||
|
export const OVERLAY_AGENT_STATE = "bh-agent-overlay-state";
|
||||||
|
|
||||||
|
export interface OverlayAgentStateMessage {
|
||||||
|
type: typeof OVERLAY_AGENT_STATE;
|
||||||
|
sessionId: string | null;
|
||||||
|
mode: OverlayMode;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface OverlayInterruptRequest {
|
export interface OverlayInterruptRequest {
|
||||||
kind: typeof OVERLAY_MSG_INTERRUPT;
|
kind: typeof OVERLAY_MSG_INTERRUPT;
|
||||||
sessionId: string;
|
sessionId: string;
|
||||||
@@ -59,4 +106,23 @@ export function isOverlayAgentOverlayResetMessage(
|
|||||||
return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string";
|
return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
export type OverlayMessage = OverlayWhoAmIRequest | OverlayInterruptRequest;
|
export function isOverlayAgentStateMessage(message: unknown): message is OverlayAgentStateMessage {
|
||||||
|
if (!message || typeof message !== "object") return false;
|
||||||
|
const candidate = message as {
|
||||||
|
type?: unknown;
|
||||||
|
sessionId?: unknown;
|
||||||
|
mode?: unknown;
|
||||||
|
generation?: unknown;
|
||||||
|
};
|
||||||
|
return (
|
||||||
|
candidate.type === OVERLAY_AGENT_STATE &&
|
||||||
|
(typeof candidate.sessionId === "string" || candidate.sessionId === null) &&
|
||||||
|
(candidate.mode === "control" ||
|
||||||
|
candidate.mode === "interrupting" ||
|
||||||
|
candidate.mode === "paused" ||
|
||||||
|
candidate.mode === "hidden") &&
|
||||||
|
typeof candidate.generation === "number"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export type OverlayMessage = OverlayWhoAmIRequest | OverlayReadyRequest | OverlayInterruptRequest;
|
||||||
|
|||||||
@@ -25,15 +25,23 @@ export interface RecordQueryMessage {
|
|||||||
export interface RecordQueryResponse {
|
export interface RecordQueryResponse {
|
||||||
active: boolean;
|
active: boolean;
|
||||||
requestId?: string;
|
requestId?: string;
|
||||||
|
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||||
|
startedAtMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStartMessage {
|
export interface RecordStartMessage {
|
||||||
type: typeof RECORD_START;
|
type: typeof RECORD_START;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
|
/**
|
||||||
|
* Epoch ms when the whole recording began, not when this tab was armed.
|
||||||
|
* The overlay timer must span the session, so it survives navigations and
|
||||||
|
* content-script remounts instead of restarting per page.
|
||||||
|
*/
|
||||||
|
startedAtMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
|
|||||||
@@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "hover",
|
||||||
|
target: payload.target,
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
case "fill":
|
case "fill":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ export function shouldRecordPress(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -139,6 +138,13 @@ function toV2Step(
|
|||||||
},
|
},
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
effectForNavigation(step.navigated_to, urlToId),
|
||||||
);
|
);
|
||||||
|
case "hover":
|
||||||
|
return {
|
||||||
|
op: "hover",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
target: step.target,
|
||||||
|
};
|
||||||
case "fill":
|
case "fill":
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
|
|||||||
@@ -44,3 +44,53 @@ describe("chromeAgentWindowApi.ensureActiveTab", () => {
|
|||||||
expect(update).not.toHaveBeenCalled();
|
expect(update).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("chromeAgentWindowApi.create", () => {
|
||||||
|
const create = vi.fn();
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
windows: { create },
|
||||||
|
});
|
||||||
|
create.mockReset();
|
||||||
|
create.mockResolvedValue({ id: 100 });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("focuses Agent Windows by default", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME);
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: true,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("can create an Agent Window without stealing focus", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, { focused: false });
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: false,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes an optional window size through the options object", async () => {
|
||||||
|
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, {
|
||||||
|
size: { width: 1280, height: 800 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith({
|
||||||
|
type: "normal",
|
||||||
|
focused: true,
|
||||||
|
url: AGENT_WINDOW_HOME,
|
||||||
|
width: 1280,
|
||||||
|
height: 800,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { createDisconnectCleanup } from "../disconnect-cleanup";
|
||||||
|
import { SessionManager } from "../manager";
|
||||||
|
|
||||||
|
describe("disconnect session cleanup", () => {
|
||||||
|
it("stops every local session through the safe session-stop path", async () => {
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
let nextWindowId = 100;
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => nextWindowId++),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
remove,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await manager.start("aa11");
|
||||||
|
await manager.start("bb22");
|
||||||
|
const detachSession = vi.fn(async () => {});
|
||||||
|
const onSessionsChanged = vi.fn();
|
||||||
|
const cleanup = createDisconnectCleanup({
|
||||||
|
manager,
|
||||||
|
sessionStopDeps: { cdp: { detachSession } },
|
||||||
|
onSessionsChanged,
|
||||||
|
});
|
||||||
|
|
||||||
|
const report = await cleanup();
|
||||||
|
|
||||||
|
expect(report).toEqual({ stoppedSessionIds: ["aa11", "bb22"], failures: [] });
|
||||||
|
expect(manager.list()).toEqual([]);
|
||||||
|
expect(remove).toHaveBeenCalledTimes(2);
|
||||||
|
expect(detachSession).toHaveBeenCalledWith("aa11");
|
||||||
|
expect(detachSession).toHaveBeenCalledWith("bb22");
|
||||||
|
expect(onSessionsChanged).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("coalesces overlapping disconnect notifications", async () => {
|
||||||
|
let releaseRemove: () => void = () => {};
|
||||||
|
const removeGate = new Promise<void>((resolve) => {
|
||||||
|
releaseRemove = resolve;
|
||||||
|
});
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
remove: vi.fn(() => removeGate),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await manager.start("aa11");
|
||||||
|
const cleanup = createDisconnectCleanup({ manager });
|
||||||
|
|
||||||
|
const first = cleanup();
|
||||||
|
const second = cleanup();
|
||||||
|
expect(second).toBe(first);
|
||||||
|
releaseRemove();
|
||||||
|
|
||||||
|
await expect(first).resolves.toEqual({ stoppedSessionIds: ["aa11"], failures: [] });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import type { AgentWindowApi } from "../agent-window";
|
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||||
import { SessionManager } from "../manager";
|
import { SessionManager } from "../manager";
|
||||||
|
|
||||||
function fakeAgentWindow(): AgentWindowApi & {
|
function fakeAgentWindow(): AgentWindowApi & {
|
||||||
@@ -8,7 +8,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
|||||||
ensureActiveTabMock: ReturnType<typeof vi.fn>;
|
ensureActiveTabMock: ReturnType<typeof vi.fn>;
|
||||||
} {
|
} {
|
||||||
let nextId = 100;
|
let nextId = 100;
|
||||||
const createMock = vi.fn(async (_url: string) => {
|
const createMock = vi.fn(async (_url: string, _opts?: AgentWindowCreateOptions) => {
|
||||||
const id = nextId++;
|
const id = nextId++;
|
||||||
return id;
|
return id;
|
||||||
});
|
});
|
||||||
@@ -30,7 +30,7 @@ describe("SessionManager", () => {
|
|||||||
const sm = new SessionManager({ agentWindow: aw, now: () => 1700000000000 });
|
const sm = new SessionManager({ agentWindow: aw, now: () => 1700000000000 });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
expect(aw.createMock).toHaveBeenCalledOnce();
|
expect(aw.createMock).toHaveBeenCalledOnce();
|
||||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank");
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {});
|
||||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledOnce();
|
expect(aw.ensureActiveTabMock).toHaveBeenCalledOnce();
|
||||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledWith(100, "about:blank");
|
expect(aw.ensureActiveTabMock).toHaveBeenCalledWith(100, "about:blank");
|
||||||
expect(ctx.sessionId).toBe("aa11");
|
expect(ctx.sessionId).toBe("aa11");
|
||||||
@@ -39,6 +39,24 @@ describe("SessionManager", () => {
|
|||||||
expect(ctx.refStore.isEmpty()).toBe(true);
|
expect(ctx.refStore.isEmpty()).toBe(true);
|
||||||
expect(ctx.borrowedTabs.size).toBe(0);
|
expect(ctx.borrowedTabs.size).toBe(0);
|
||||||
});
|
});
|
||||||
|
it("forwards an optional window size when starting a session", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const ctx = await sm.start("aa11", { size: { width: 1280, height: 800 } });
|
||||||
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {
|
||||||
|
size: { width: 1280, height: 800 },
|
||||||
|
});
|
||||||
|
expect(ctx.agentWindowId).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forwards an explicit unfocused start to the Agent Window", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await sm.start("aa11", { focused: false });
|
||||||
|
|
||||||
|
expect(aw.createMock).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
|
});
|
||||||
|
|
||||||
it("indexes the session by sessionId and agent window id", async () => {
|
it("indexes the session by sessionId and agent window id", async () => {
|
||||||
const aw = fakeAgentWindow();
|
const aw = fakeAgentWindow();
|
||||||
@@ -57,6 +75,52 @@ describe("SessionManager", () => {
|
|||||||
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("removes a newly created Agent Window when startup is aborted", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
let resolveCreate: (windowId: number) => void = () => {};
|
||||||
|
aw.createMock.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise<number>((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const controller = new AbortController();
|
||||||
|
const pending = sm.start("aa11", { signal: controller.signal });
|
||||||
|
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate(777);
|
||||||
|
|
||||||
|
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(aw.removeMock).toHaveBeenCalledWith(777);
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes an incomplete Agent Window when active-tab setup fails", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await expect(sm.start("aa11")).rejects.toThrow("tab setup failed");
|
||||||
|
|
||||||
|
expect(aw.removeMock).toHaveBeenCalledWith(100);
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces the orphan Agent Window id when startup cleanup fails", async () => {
|
||||||
|
const aw = fakeAgentWindow();
|
||||||
|
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||||
|
aw.removeMock.mockRejectedValueOnce(new Error("window removal denied"));
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|
||||||
|
await expect(sm.start("aa11")).rejects.toMatchObject({
|
||||||
|
name: "SessionStartCleanupError",
|
||||||
|
windowId: 100,
|
||||||
|
message: expect.stringMatching(/cleanup of Agent Window 100 failed.*window removal denied/),
|
||||||
|
});
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("stop() closes the Agent Window and forgets the session", async () => {
|
it("stop() closes the Agent Window and forgets the session", async () => {
|
||||||
const aw = fakeAgentWindow();
|
const aw = fakeAgentWindow();
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|||||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
|||||||
expect(s.resolve("@e11")).toBe(11);
|
expect(s.resolve("@e11")).toBe(11);
|
||||||
expect(s.size()).toBe(2);
|
expect(s.size()).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves the child CDP session as part of ref identity", () => {
|
||||||
|
const s = new RefStore();
|
||||||
|
s.set("e1", 42, {
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
expect(s.resolveEntry("e1")).toMatchObject({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
export interface AgentWindowApi {
|
export interface AgentWindowApi {
|
||||||
create(url: string): Promise<number>;
|
create(url: string, opts?: AgentWindowCreateOptions): Promise<number>;
|
||||||
remove(windowId: number): Promise<void>;
|
remove(windowId: number): Promise<void>;
|
||||||
/**
|
/**
|
||||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||||
@@ -18,15 +18,24 @@ export interface AgentWindowApi {
|
|||||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Creation hints for a new Agent Window. */
|
||||||
|
export interface AgentWindowCreateOptions {
|
||||||
|
/** Optional outer size in CSS pixels. */
|
||||||
|
size?: { width: number; height: number };
|
||||||
|
/** Defaults to true so existing sessions keep visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
/** Initial tab URL for every new session's Agent Window. */
|
/** Initial tab URL for every new session's Agent Window. */
|
||||||
export const AGENT_WINDOW_HOME = "about:blank";
|
export const AGENT_WINDOW_HOME = "about:blank";
|
||||||
|
|
||||||
export const chromeAgentWindowApi: AgentWindowApi = {
|
export const chromeAgentWindowApi: AgentWindowApi = {
|
||||||
async create(url: string): Promise<number> {
|
async create(url: string, opts: AgentWindowCreateOptions = {}): Promise<number> {
|
||||||
const win = await chrome.windows.create({
|
const win = await chrome.windows.create({
|
||||||
type: "normal",
|
type: "normal",
|
||||||
focused: true,
|
focused: opts.focused ?? true,
|
||||||
url,
|
url,
|
||||||
|
...(opts.size ? { width: opts.size.width, height: opts.size.height } : {}),
|
||||||
});
|
});
|
||||||
if (typeof win?.id !== "number") {
|
if (typeof win?.id !== "number") {
|
||||||
throw new Error("[bh] chrome.windows.create returned no window id");
|
throw new Error("[bh] chrome.windows.create returned no window id");
|
||||||
@@ -34,12 +43,11 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
|||||||
return win.id;
|
return win.id;
|
||||||
},
|
},
|
||||||
async remove(windowId: number): Promise<void> {
|
async remove(windowId: number): Promise<void> {
|
||||||
try {
|
// Callers decide whether a missing/failed removal is benign. In
|
||||||
await chrome.windows.remove(windowId);
|
// particular, transactional session-start cleanup must be able to
|
||||||
} catch (err) {
|
// surface a window it could not remove instead of reporting a false
|
||||||
// Window may have been closed by the user already; ignore.
|
// cancellation success while the Agent Window remains open.
|
||||||
console.debug("[bh] chrome.windows.remove failed", err);
|
await chrome.windows.remove(windowId);
|
||||||
}
|
|
||||||
},
|
},
|
||||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||||
const tabs = await chrome.tabs.query({ windowId });
|
const tabs = await chrome.tabs.query({ windowId });
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||||
|
import type { SessionManager } from "./manager";
|
||||||
|
|
||||||
|
export interface DisconnectCleanupFailure {
|
||||||
|
sessionId: string;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DisconnectCleanupReport {
|
||||||
|
stoppedSessionIds: string[];
|
||||||
|
failures: DisconnectCleanupFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DisconnectCleanupOptions {
|
||||||
|
manager: SessionManager;
|
||||||
|
sessionStopDeps?: SessionStopDeps;
|
||||||
|
onSessionsChanged?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a coalesced cleanup operation for transport loss.
|
||||||
|
*
|
||||||
|
* The daemon purges its registry when the extension socket disappears. To
|
||||||
|
* keep the extension-side mirror consistent, every local session must follow
|
||||||
|
* the normal safe stop path as well: return borrowed tabs, clear refs, detach
|
||||||
|
* CDP, and only then close the Agent Window.
|
||||||
|
*/
|
||||||
|
export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
||||||
|
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||||
|
|
||||||
|
return (): Promise<DisconnectCleanupReport> => {
|
||||||
|
if (inFlight) return inFlight;
|
||||||
|
inFlight = cleanupSessions(options).finally(() => {
|
||||||
|
inFlight = null;
|
||||||
|
});
|
||||||
|
return inFlight;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupSessions(
|
||||||
|
options: DisconnectCleanupOptions,
|
||||||
|
): Promise<DisconnectCleanupReport> {
|
||||||
|
const stoppedSessionIds: string[] = [];
|
||||||
|
const failures: DisconnectCleanupFailure[] = [];
|
||||||
|
|
||||||
|
for (const ctx of options.manager.list()) {
|
||||||
|
try {
|
||||||
|
const result = await handleSessionStop(
|
||||||
|
options.manager,
|
||||||
|
{ session_id: ctx.sessionId },
|
||||||
|
options.sessionStopDeps,
|
||||||
|
);
|
||||||
|
if ("code" in result) {
|
||||||
|
failures.push({ sessionId: ctx.sessionId, message: result.message });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (result.return_failures?.length) {
|
||||||
|
failures.push({
|
||||||
|
sessionId: ctx.sessionId,
|
||||||
|
message: result.return_failures.map((failure) => failure.message).join("; "),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
stoppedSessionIds.push(ctx.sessionId);
|
||||||
|
} catch (err) {
|
||||||
|
failures.push({
|
||||||
|
sessionId: ctx.sessionId,
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
options.onSessionsChanged?.();
|
||||||
|
return { stoppedSessionIds, failures };
|
||||||
|
}
|
||||||
@@ -25,6 +25,46 @@ export interface SessionManagerOptions {
|
|||||||
now?: () => number;
|
now?: () => number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Options for starting a session's Agent Window. */
|
||||||
|
export interface SessionStartOptions {
|
||||||
|
/** Optional Agent Window outer size in CSS pixels. */
|
||||||
|
size?: { width: number; height: number };
|
||||||
|
/** Defaults to true so existing clients keep visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
|
/** Cancellation for the transactional Agent Window startup sequence. */
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SessionStartCleanupError extends Error {
|
||||||
|
readonly windowId: number;
|
||||||
|
readonly startupError: unknown;
|
||||||
|
readonly cleanupError: unknown;
|
||||||
|
|
||||||
|
constructor(windowId: number, startupError: unknown, cleanupError: unknown) {
|
||||||
|
const startupMessage =
|
||||||
|
startupError instanceof Error ? startupError.message : String(startupError);
|
||||||
|
const cleanupMessage =
|
||||||
|
cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
||||||
|
super(
|
||||||
|
`session_start failed (${startupMessage}) and cleanup of Agent Window ${windowId} failed: ${cleanupMessage}`,
|
||||||
|
);
|
||||||
|
this.name = "SessionStartCleanupError";
|
||||||
|
this.windowId = windowId;
|
||||||
|
this.startupError = startupError;
|
||||||
|
this.cleanupError = cleanupError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionStartAbortError(): Error {
|
||||||
|
const error = new Error("session_start aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfSessionStartAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (signal?.aborted) throw sessionStartAbortError();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Owner of all live agent sessions inside the extension.
|
* Owner of all live agent sessions inside the extension.
|
||||||
*
|
*
|
||||||
@@ -128,22 +168,40 @@ export class SessionManager {
|
|||||||
* Returns the created window id so callers can echo it back to the
|
* Returns the created window id so callers can echo it back to the
|
||||||
* daemon in the `tool.session_start` reply.
|
* daemon in the `tool.session_start` reply.
|
||||||
*/
|
*/
|
||||||
async start(sessionId: string): Promise<SessionContext> {
|
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||||
if (this.sessions.has(sessionId)) {
|
if (this.sessions.has(sessionId)) {
|
||||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||||
}
|
}
|
||||||
const windowId = await this.agentWindow.create(AGENT_WINDOW_HOME);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
|
||||||
const ctx: SessionContext = {
|
let windowId: number | null = null;
|
||||||
sessionId,
|
try {
|
||||||
agentWindowId: windowId,
|
const { signal: _signal, ...createOptions } = opts;
|
||||||
refStore: new RefStore(),
|
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||||
borrowedTabs: new Map(),
|
throwIfSessionStartAborted(opts.signal);
|
||||||
createdAtMs: this.now(),
|
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||||
};
|
throwIfSessionStartAborted(opts.signal);
|
||||||
this.sessions.set(sessionId, ctx);
|
|
||||||
this.windowIndex.set(windowId, sessionId);
|
const ctx: SessionContext = {
|
||||||
return ctx;
|
sessionId,
|
||||||
|
agentWindowId: windowId,
|
||||||
|
refStore: new RefStore(),
|
||||||
|
borrowedTabs: new Map(),
|
||||||
|
createdAtMs: this.now(),
|
||||||
|
};
|
||||||
|
this.sessions.set(sessionId, ctx);
|
||||||
|
this.windowIndex.set(windowId, sessionId);
|
||||||
|
return ctx;
|
||||||
|
} catch (startupError) {
|
||||||
|
if (windowId !== null) {
|
||||||
|
try {
|
||||||
|
await this.agentWindow.remove(windowId);
|
||||||
|
} catch (cleanupError) {
|
||||||
|
throw new SessionStartCleanupError(windowId, startupError, cleanupError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw startupError;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
/**
|
/**
|
||||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||||
*
|
*
|
||||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
* `replace(...)` with the new ref → node address pairs. A node address
|
||||||
* `tool.click` consume the store via `resolve("@e1")`.
|
* includes the owning flat CDP session/frame when the element lives in
|
||||||
|
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||||
*
|
*
|
||||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||||
* returns `null`, never silently leaks. Storing values in different
|
* returns `null`, never silently leaks. Storing values in different
|
||||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
|||||||
export interface RefEntry {
|
export interface RefEntry {
|
||||||
backendNodeId: BackendNodeId;
|
backendNodeId: BackendNodeId;
|
||||||
tabId: number | null;
|
tabId: number | null;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
generation: number;
|
generation: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
export type RefInput =
|
||||||
|
| BackendNodeId
|
||||||
|
| {
|
||||||
|
backendNodeId: BackendNodeId;
|
||||||
|
tabId: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
};
|
||||||
|
|
||||||
export class RefStore {
|
export class RefStore {
|
||||||
private readonly map = new Map<string, RefEntry>();
|
private readonly map = new Map<string, RefEntry>();
|
||||||
@@ -44,7 +54,7 @@ export class RefStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||||
* Used after every fresh `tool.snapshot`.
|
* Used after every fresh `tool.snapshot`.
|
||||||
*/
|
*/
|
||||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||||
@@ -53,10 +63,20 @@ export class RefStore {
|
|||||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
set(
|
||||||
|
ref: string,
|
||||||
|
id: BackendNodeId,
|
||||||
|
opts: {
|
||||||
|
tabId?: number;
|
||||||
|
frameId?: string;
|
||||||
|
cdpSessionId?: string;
|
||||||
|
} = {},
|
||||||
|
): void {
|
||||||
this.map.set(normaliseRef(ref), {
|
this.map.set(normaliseRef(ref), {
|
||||||
backendNodeId: id,
|
backendNodeId: id,
|
||||||
tabId: opts.tabId ?? null,
|
tabId: opts.tabId ?? null,
|
||||||
|
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||||
|
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -80,6 +100,8 @@ export class RefStore {
|
|||||||
return {
|
return {
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
tabId: input.tabId,
|
tabId: input.tabId,
|
||||||
|
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||||
|
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
|
|||||||
).toBe(false);
|
).toBe(false);
|
||||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects the Edge Add-ons store", () => {
|
||||||
|
expect(
|
||||||
|
isInjectableContentScriptUrl(
|
||||||
|
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
// Non-store paths on the same host stay injectable.
|
||||||
|
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import type {
|
|||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
|
||||||
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
function fakeTransport() {
|
function fakeTransport() {
|
||||||
const handlers = new Set<FrameHandler>();
|
const handlers = new Set<FrameHandler>();
|
||||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||||
@@ -67,6 +69,25 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards an unfocused session start to the Agent Window", async () => {
|
||||||
|
const { transport, deliver } = fakeTransport();
|
||||||
|
const create = vi.fn(async () => 4242);
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create,
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_start", { session_id: "aa11", focused: false }));
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
|
expect(create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.session_stop and replies with empty result", async () => {
|
it("routes tool.session_stop and replies with empty result", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -124,8 +145,12 @@ describe("ToolDispatcher", () => {
|
|||||||
truncated: false,
|
truncated: false,
|
||||||
}) satisfies ConsoleResult,
|
}) satisfies ConsoleResult,
|
||||||
),
|
),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
||||||
@@ -143,6 +168,107 @@ describe("ToolDispatcher", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||||
|
const sendMessage = vi.fn(async () => undefined);
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
|
||||||
|
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
|
||||||
|
sendMessage,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ctx = await sessions.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
return { object: { objectId: "trigger-object" } } as T;
|
||||||
|
}
|
||||||
|
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
|
if (declaration.includes("count: state.inputs.length")) {
|
||||||
|
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||||
|
}
|
||||||
|
if (declaration.includes("inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } } as T;
|
||||||
|
}
|
||||||
|
return { result: { value: true } } as T;
|
||||||
|
}
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string }).expression ?? "";
|
||||||
|
if (expression.includes("overlayDetails")) {
|
||||||
|
return { result: { value: { hitIndex: 0 } } } as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("overlayHostPresent")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||||
|
},
|
||||||
|
} as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("count:")) {
|
||||||
|
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||||
|
}
|
||||||
|
if (expression.includes("?.inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } } as T;
|
||||||
|
}
|
||||||
|
return { result: { value: true } } as T;
|
||||||
|
}
|
||||||
|
return {} as T;
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send,
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(
|
||||||
|
makeRequest("tool.upload", {
|
||||||
|
session_id: "aa11",
|
||||||
|
ref: "@e1",
|
||||||
|
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await flushMicrotasks();
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||||
|
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||||
|
type: "bh-automation-bypass",
|
||||||
|
enabled: true,
|
||||||
|
});
|
||||||
|
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||||
|
type: "bh-automation-bypass",
|
||||||
|
enabled: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("detaches CDP state before stopping a session", async () => {
|
it("detaches CDP state before stopping a session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -163,8 +289,12 @@ describe("ToolDispatcher", () => {
|
|||||||
next_since: 0,
|
next_since: 0,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
})),
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
|
||||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||||
@@ -274,6 +404,306 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
|
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const onBrowserControlResumed = vi.fn();
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||||
|
await flushMicrotasks();
|
||||||
|
sent.length = 0;
|
||||||
|
|
||||||
|
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
|
||||||
|
await flushMicrotasks();
|
||||||
|
expect(onBrowserControlResumed).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
deliver({
|
||||||
|
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
|
||||||
|
id: "r-control",
|
||||||
|
});
|
||||||
|
await flushMicrotasks();
|
||||||
|
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const order: string[] = [];
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => {
|
||||||
|
order.push("hover");
|
||||||
|
return {} as T;
|
||||||
|
}),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
|
||||||
|
(
|
||||||
|
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
|
||||||
|
).rememberHover("aa11", {
|
||||||
|
tab_id: 7,
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
|
||||||
|
order.push("observe");
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
expect(order).toEqual(["hover", "observe"]);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.withHoverReassert(
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
async () => {
|
||||||
|
order.push("click");
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
{ releaseAfter: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 10,
|
||||||
|
y: 20,
|
||||||
|
});
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reassert or disable hover latches from another session", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 7,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options?: { releaseAfter?: boolean },
|
||||||
|
) => Promise<T>;
|
||||||
|
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
|
||||||
|
releaseAfter: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(cdp.send).not.toHaveBeenCalled();
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.releaseHoverLatch("aa11", 7);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
await helpers.setHoverBypass("bb22", 7, true);
|
||||||
|
await helpers.setHoverBypass("aa11", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await helpers.setHoverBypass("bb22", 7, false);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("limits default-target hover reassertion to the active tab", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||||
|
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 9,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 30,
|
||||||
|
y: 40,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the current session hover latch before navigation-style work", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 1),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
withHoverReleaseForRequest: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
await helpers.setHoverBypass("aa11", 7, true);
|
||||||
|
|
||||||
|
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||||
|
|
||||||
|
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||||
|
7,
|
||||||
|
expect.objectContaining({ enabled: false }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||||
const { transport, deliver } = fakeTransport();
|
const { transport, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -348,23 +778,31 @@ describe("ToolDispatcher", () => {
|
|||||||
await flushMicrotasks();
|
await flushMicrotasks();
|
||||||
expect(ac?.signal.aborted).toBe(true);
|
expect(ac?.signal.aborted).toBe(true);
|
||||||
|
|
||||||
// Cancel ack arrived synchronously; the slow tool replies with
|
// Cancel ack arrives synchronously, but the original RPC must not reply
|
||||||
// `cancelled` once the dispatcher's race observes the abort.
|
// until the in-progress window creation has completed and been rolled back.
|
||||||
const ack = sent.find(
|
const ack = sent.find(
|
||||||
(m) =>
|
(m) =>
|
||||||
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
||||||
);
|
);
|
||||||
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
sent.find(
|
||||||
|
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||||
|
),
|
||||||
|
).toBeUndefined();
|
||||||
|
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(true);
|
||||||
|
|
||||||
|
resolveCreate(9999);
|
||||||
|
await flushMicrotasks();
|
||||||
|
|
||||||
const slow = sent.find(
|
const slow = sent.find(
|
||||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||||
);
|
);
|
||||||
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
||||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
||||||
|
expect(sessions.has("aa44")).toBe(false);
|
||||||
// Drain the dangling create promise so vitest does not warn.
|
expect(sessions.list()).toEqual([]);
|
||||||
resolveCreate(9999);
|
|
||||||
await flushMicrotasks();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
||||||
|
|||||||
@@ -1,24 +1,54 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
import { nodeContentRegion } from "../element-geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
describe("quadBoundingRect", () => {
|
describe("nodeContentRegion", () => {
|
||||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
const send = async (_tabId: number, method: string) => {
|
||||||
x: 10,
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||||
y: 20,
|
if (method === "DOM.getBoxModel") {
|
||||||
width: 100,
|
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||||
height: 40,
|
}
|
||||||
|
throw new Error(`unexpected CDP call ${method}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 5, y: 10 },
|
||||||
|
{ x: 25, y: 10 },
|
||||||
|
{ x: 25, y: 30 },
|
||||||
|
{ x: 5, y: 30 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves every visible content fragment", async () => {
|
||||||
|
const send = async () => ({
|
||||||
|
quads: [
|
||||||
|
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||||
|
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||||
|
],
|
||||||
});
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
|
[
|
||||||
|
{ x: 10, y: 20 },
|
||||||
|
{ x: 60, y: 20 },
|
||||||
|
{ x: 60, y: 40 },
|
||||||
|
{ x: 10, y: 40 },
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{ x: 15, y: 50 },
|
||||||
|
{ x: 90, y: 50 },
|
||||||
|
{ x: 90, y: 80 },
|
||||||
|
{ x: 15, y: 80 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns null for degenerate quads", () => {
|
|
||||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
|
||||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("nodeBoundingRect", () => {
|
|
||||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||||
const calls: Array<{ method: string; params?: object }> = [];
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
const send = async (_tabId: number, method: string, params?: object) => {
|
const send = async (_tabId: number, method: string, params?: object) => {
|
||||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
|||||||
};
|
};
|
||||||
const cdp = { send: send as CdpRunner["send"] };
|
const cdp = { send: send as CdpRunner["send"] };
|
||||||
|
|
||||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||||
x: 242,
|
[
|
||||||
y: 468,
|
{ x: 242, y: 468 },
|
||||||
width: 769,
|
{ x: 1011, y: 468 },
|
||||||
height: 180,
|
{ x: 1011, y: 648 },
|
||||||
});
|
{ x: 242, y: 648 },
|
||||||
|
],
|
||||||
|
]);
|
||||||
expect(calls.map((c) => c.method)).toEqual([
|
expect(calls.map((c) => c.method)).toEqual([
|
||||||
"DOM.getContentQuads",
|
"DOM.getContentQuads",
|
||||||
"DOM.getBoxModel",
|
"DOM.getBoxModel",
|
||||||
|
|||||||
@@ -0,0 +1,416 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { DeviceMetricsOverride, UserAgentOverride } from "@/browser-driver/chromium-cdp";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import type { EmulateOverrides, RpcError } from "@/transport/types";
|
||||||
|
import {
|
||||||
|
EMULATE_SCOPE_NOTE,
|
||||||
|
type EmulateCdpRunner,
|
||||||
|
handleEmulate,
|
||||||
|
resetEmulateStatesForTests,
|
||||||
|
toCdpUserAgentMetadata,
|
||||||
|
validateOverrides,
|
||||||
|
} from "../emulate";
|
||||||
|
|
||||||
|
function fakeAgentWindow(ids: number[]) {
|
||||||
|
let i = 0;
|
||||||
|
return {
|
||||||
|
create: vi.fn(async () => {
|
||||||
|
const id = ids[i++];
|
||||||
|
if (id === undefined) throw new Error("ran out of fake ids");
|
||||||
|
return id;
|
||||||
|
}),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CdpCalls {
|
||||||
|
metrics: Array<{ tabId: number; metrics: DeviceMetricsOverride }>;
|
||||||
|
clearedMetrics: number[];
|
||||||
|
ua: Array<{ tabId: number; override: UserAgentOverride }>;
|
||||||
|
touch: Array<{ tabId: number; enabled: boolean; maxTouchPoints?: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDeps(opts: { throwOn?: string; tabWindowId?: number } = {}) {
|
||||||
|
const calls: CdpCalls = { metrics: [], clearedMetrics: [], ua: [], touch: [] };
|
||||||
|
const maybeThrow = (kind: string) => {
|
||||||
|
if (opts.throwOn === kind) throw new Error(`simulated ${kind} failure`);
|
||||||
|
};
|
||||||
|
const cdp: EmulateCdpRunner = {
|
||||||
|
setDeviceMetricsOverride: vi.fn(async (tabId, metrics) => {
|
||||||
|
maybeThrow("metrics");
|
||||||
|
calls.metrics.push({ tabId, metrics });
|
||||||
|
}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async (tabId) => {
|
||||||
|
maybeThrow("metrics");
|
||||||
|
calls.clearedMetrics.push(tabId);
|
||||||
|
}),
|
||||||
|
setUserAgentOverride: vi.fn(async (tabId, override) => {
|
||||||
|
maybeThrow("ua");
|
||||||
|
calls.ua.push({ tabId, override });
|
||||||
|
}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async (tabId, enabled, maxTouchPoints) => {
|
||||||
|
maybeThrow("touch");
|
||||||
|
calls.touch.push({ tabId, enabled, maxTouchPoints });
|
||||||
|
}),
|
||||||
|
trackSessionTab: vi.fn(),
|
||||||
|
};
|
||||||
|
const tabsApi = {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) =>
|
||||||
|
({ id: tabId, windowId: opts.tabWindowId ?? 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [
|
||||||
|
{ id: 7, windowId: opts.tabWindowId ?? 100, active: true } as chrome.tabs.Tab,
|
||||||
|
]),
|
||||||
|
};
|
||||||
|
return { cdp, tabsApi, calls };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function makeManager(): Promise<SessionManager> {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
return sm;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullOverrides: EmulateOverrides = {
|
||||||
|
width: 390,
|
||||||
|
height: 844,
|
||||||
|
device_scale_factor: 3,
|
||||||
|
mobile: true,
|
||||||
|
user_agent: "Mozilla/5.0 (iPhone)",
|
||||||
|
accept_language: "zh-CN",
|
||||||
|
touch: true,
|
||||||
|
max_touch_points: 5,
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("handleEmulate", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
resetEmulateStatesForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies viewport, UA and touch overrides to the active tab", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(sm, { session_id: "aa11", overrides: fullOverrides }, deps);
|
||||||
|
expect(result).toEqual({
|
||||||
|
tab_id: 7,
|
||||||
|
cleared: false,
|
||||||
|
applied: fullOverrides,
|
||||||
|
note: EMULATE_SCOPE_NOTE,
|
||||||
|
});
|
||||||
|
expect(deps.calls.metrics).toEqual([
|
||||||
|
{ tabId: 7, metrics: { width: 390, height: 844, deviceScaleFactor: 3, mobile: true } },
|
||||||
|
]);
|
||||||
|
expect(deps.calls.ua).toEqual([
|
||||||
|
{
|
||||||
|
tabId: 7,
|
||||||
|
override: { userAgent: "Mozilla/5.0 (iPhone)", acceptLanguage: "zh-CN" },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(deps.calls.touch).toEqual([{ tabId: 7, enabled: true, maxTouchPoints: 5 }]);
|
||||||
|
expect(deps.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 7);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults deviceScaleFactor to 0 and mobile to false when omitted", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { width: 800, height: 1280 } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 7, cleared: false });
|
||||||
|
expect(deps.calls.metrics).toEqual([
|
||||||
|
{ tabId: 7, metrics: { width: 800, height: 1280, deviceScaleFactor: 0, mobile: false } },
|
||||||
|
]);
|
||||||
|
expect(deps.calls.ua).toEqual([]);
|
||||||
|
expect(deps.calls.touch).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies a UA-only override", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { user_agent: "custom-ua" } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 7, cleared: false });
|
||||||
|
expect(deps.calls.metrics).toEqual([]);
|
||||||
|
expect(deps.calls.ua).toEqual([{ tabId: 7, override: { userAgent: "custom-ua" } }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("converts user_agent_metadata to CDP camelCase", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{
|
||||||
|
session_id: "aa11",
|
||||||
|
overrides: {
|
||||||
|
user_agent: "custom-ua",
|
||||||
|
user_agent_metadata: {
|
||||||
|
brands: [{ brand: "Chromium", version: "120" }],
|
||||||
|
full_version: "120.0.0.0",
|
||||||
|
platform: "Android",
|
||||||
|
platform_version: "14",
|
||||||
|
mobile: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 7 });
|
||||||
|
expect(deps.calls.ua[0].override.userAgentMetadata).toEqual({
|
||||||
|
brands: [{ brand: "Chromium", version: "120" }],
|
||||||
|
fullVersion: "120.0.0.0",
|
||||||
|
platform: "Android",
|
||||||
|
platformVersion: "14",
|
||||||
|
mobile: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("max_touch_points without touch enables touch emulation", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { max_touch_points: 5 } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 7, cleared: false });
|
||||||
|
expect(deps.calls.touch).toEqual([{ tabId: 7, enabled: true, maxTouchPoints: 5 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("touch: false only disables touch emulation", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { touch: false } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 7, cleared: false });
|
||||||
|
expect(deps.calls.touch).toEqual([{ tabId: 7, enabled: false, maxTouchPoints: undefined }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("targets an explicit tab_id", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 9, overrides: { user_agent: "custom-ua" } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ tab_id: 9 });
|
||||||
|
expect(deps.calls.ua).toEqual([{ tabId: 9, override: { userAgent: "custom-ua" } }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("off clears metrics, touch and UA overrides", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(sm, { session_id: "aa11", off: true }, deps);
|
||||||
|
expect(result).toEqual({ tab_id: 7, cleared: true, note: EMULATE_SCOPE_NOTE });
|
||||||
|
expect(deps.calls.clearedMetrics).toEqual([7]);
|
||||||
|
expect(deps.calls.touch).toEqual([{ tabId: 7, enabled: false, maxTouchPoints: undefined }]);
|
||||||
|
expect(deps.calls.ua).toEqual([{ tabId: 7, override: { userAgent: "" } }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects off combined with overrides", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", off: true, overrides: { touch: true } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(deps.calls.clearedMetrics).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects missing overrides and missing off", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(sm, { session_id: "aa11" }, deps);
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an unknown session", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
const result = await handleEmulate(sm, { session_id: "zz99", off: true }, deps);
|
||||||
|
expect(result).toMatchObject({ code: "not_found" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a tab outside the Agent Window", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps({ tabWindowId: 200 });
|
||||||
|
const result = await handleEmulate(sm, { session_id: "aa11", tab_id: 9, off: true }, deps);
|
||||||
|
expect(result).toMatchObject({ code: "permission_denied" });
|
||||||
|
expect(deps.calls.clearedMetrics).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("merges onto the tab's stored state: later width/height keep the earlier dpr/mobile", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
await handleEmulate(sm, { session_id: "aa11", overrides: fullOverrides }, deps);
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { width: 500, height: 900 } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
tab_id: 7,
|
||||||
|
cleared: false,
|
||||||
|
applied: { width: 500, height: 900, device_scale_factor: 3, mobile: true },
|
||||||
|
});
|
||||||
|
expect(deps.calls.metrics).toEqual([
|
||||||
|
{ tabId: 7, metrics: { width: 390, height: 844, deviceScaleFactor: 3, mobile: true } },
|
||||||
|
{ tabId: 7, metrics: { width: 500, height: 900, deviceScaleFactor: 3, mobile: true } },
|
||||||
|
]);
|
||||||
|
// The merged state is applied as a whole: UA and touch are re-applied too.
|
||||||
|
expect(deps.calls.ua).toHaveLength(2);
|
||||||
|
expect(deps.calls.touch).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps emulation state per tab", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
await handleEmulate(sm, { session_id: "aa11", overrides: fullOverrides }, deps);
|
||||||
|
await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 9, overrides: { width: 500, height: 900 } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
// Tab 9 has no stored state, so dpr/mobile fall back to the CDP defaults.
|
||||||
|
expect(deps.calls.metrics.at(-1)).toEqual({
|
||||||
|
tabId: 9,
|
||||||
|
metrics: { width: 500, height: 900, deviceScaleFactor: 0, mobile: false },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("off clears the stored state: a later request starts from scratch", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps();
|
||||||
|
await handleEmulate(sm, { session_id: "aa11", overrides: fullOverrides }, deps);
|
||||||
|
await handleEmulate(sm, { session_id: "aa11", off: true }, deps);
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { width: 500, height: 900 } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toEqual({
|
||||||
|
tab_id: 7,
|
||||||
|
cleared: false,
|
||||||
|
applied: { width: 500, height: 900 },
|
||||||
|
note: EMULATE_SCOPE_NOTE,
|
||||||
|
});
|
||||||
|
// dpr/mobile fell back to the CDP defaults instead of the cleared values,
|
||||||
|
// and the cleared UA/touch were not re-applied.
|
||||||
|
expect(deps.calls.metrics.at(-1)).toEqual({
|
||||||
|
tabId: 7,
|
||||||
|
metrics: { width: 500, height: 900, deviceScaleFactor: 0, mobile: false },
|
||||||
|
});
|
||||||
|
expect(deps.calls.ua).toHaveLength(2); // preset + the clearing empty UA
|
||||||
|
expect(deps.calls.touch).toHaveLength(2); // preset + the disabling call
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps CDP failures to cdp_failed and points at --off", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps({ throwOn: "ua" });
|
||||||
|
const result = await handleEmulate(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", overrides: { user_agent: "custom-ua" } },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "cdp_failed" });
|
||||||
|
const message = (result as RpcError).message;
|
||||||
|
expect(message).toContain("simulated ua failure");
|
||||||
|
expect(message).toContain("not rolled back");
|
||||||
|
expect(message).toContain("--off");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports an off-branch CDP failure with the same not-rolled-back note", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const deps = makeDeps({ throwOn: "metrics" });
|
||||||
|
const result = await handleEmulate(sm, { session_id: "aa11", off: true }, deps);
|
||||||
|
expect(result).toMatchObject({ code: "cdp_failed" });
|
||||||
|
const message = (result as RpcError).message;
|
||||||
|
expect(message).toContain("simulated metrics failure");
|
||||||
|
expect(message).toContain("not rolled back");
|
||||||
|
expect(message).toContain("--off");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("validateOverrides", () => {
|
||||||
|
it("accepts a full override set", () => {
|
||||||
|
expect(validateOverrides(fullOverrides)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects width without height and vice versa", () => {
|
||||||
|
expect(validateOverrides({ width: 390 })).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(validateOverrides({ height: 844 })).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects non-positive or non-integer dimensions", () => {
|
||||||
|
expect(validateOverrides({ width: 0, height: 844 })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
expect(validateOverrides({ width: 390.5, height: 844 })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects metrics extras without dimensions", () => {
|
||||||
|
expect(validateOverrides({ device_scale_factor: 3 })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
expect(validateOverrides({ mobile: true })).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a non-positive device_scale_factor", () => {
|
||||||
|
expect(validateOverrides({ width: 390, height: 844, device_scale_factor: 0 })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
validateOverrides({ width: 390, height: 844, device_scale_factor: Number.NaN }),
|
||||||
|
).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects accept_language / user_agent_metadata without user_agent", () => {
|
||||||
|
expect(validateOverrides({ accept_language: "zh-CN" })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
expect(validateOverrides({ user_agent_metadata: { platform: "Android" } })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects invalid max_touch_points", () => {
|
||||||
|
expect(validateOverrides({ touch: true, max_touch_points: 0 })).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an empty override set", () => {
|
||||||
|
expect(validateOverrides({})).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("toCdpUserAgentMetadata", () => {
|
||||||
|
it("maps snake_case fields to camelCase and drops absent ones", () => {
|
||||||
|
expect(
|
||||||
|
toCdpUserAgentMetadata({
|
||||||
|
brands: [{ brand: "Chromium", version: "120" }],
|
||||||
|
platform: "Android",
|
||||||
|
mobile: true,
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
brands: [{ brand: "Chromium", version: "120" }],
|
||||||
|
platform: "Android",
|
||||||
|
mobile: true,
|
||||||
|
});
|
||||||
|
expect(toCdpUserAgentMetadata({})).toEqual({});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,559 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { type DownloadsApi, handleDownload } from "../download";
|
||||||
|
import { captureBrowserDownload } from "../download-capture";
|
||||||
|
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||||
|
import type { ResolvedActionTarget } from "../interaction";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { handleUpload } from "../upload";
|
||||||
|
|
||||||
|
function sessions() {
|
||||||
|
return new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 100),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function tabsApi() {
|
||||||
|
return {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||||
|
const listeners = new Set<T>();
|
||||||
|
return {
|
||||||
|
addListener: (listener: T) => listeners.add(listener),
|
||||||
|
removeListener: (listener: T) => listeners.delete(listener),
|
||||||
|
emit: (...args: Parameters<T>) => {
|
||||||
|
for (const listener of listeners) listener(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionTarget(frameId?: string): ResolvedActionTarget {
|
||||||
|
return {
|
||||||
|
tab: { tabId: 4, windowId: 100, active: true },
|
||||||
|
backendNodeId: 123,
|
||||||
|
cdpTarget: { tabId: 4 },
|
||||||
|
...(frameId ? { frameId } : {}),
|
||||||
|
usedRef: "e3",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function uploadCdp(
|
||||||
|
options: {
|
||||||
|
inputCount?: number;
|
||||||
|
multiple?: boolean;
|
||||||
|
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||||
|
pendingResolve?: boolean;
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
calls.push({ method, params });
|
||||||
|
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
if (method === "DOM.resolveNode") {
|
||||||
|
if (options.pendingResolve) return new Promise<never>(() => {});
|
||||||
|
return { object: { objectId: "trigger-object" } };
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
|
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
|
||||||
|
if (declaration.includes("count: state.inputs.length")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
count: options.inputCount ?? 1,
|
||||||
|
multiple: options.multiple ?? true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (declaration.includes("inputs[0]")) {
|
||||||
|
return { result: { objectId: "input-object" } };
|
||||||
|
}
|
||||||
|
return { result: { value: true } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchMouseEvent" &&
|
||||||
|
(params as { type?: string }).type === "mousePressed" &&
|
||||||
|
options.chooser
|
||||||
|
) {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
|
||||||
|
cdp,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("file transfer tools", () => {
|
||||||
|
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||||
|
const manager = sessions();
|
||||||
|
const ctx = await manager.start("s1");
|
||||||
|
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||||
|
const { cdp, calls } = uploadCdp();
|
||||||
|
|
||||||
|
const result = await handleUpload(
|
||||||
|
manager,
|
||||||
|
{
|
||||||
|
session_id: "s1",
|
||||||
|
ref: "@e3",
|
||||||
|
files: [
|
||||||
|
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
|
||||||
|
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{ cdp, tabsApi: tabsApi() },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||||
|
expect(calls[0]).toEqual({
|
||||||
|
method: "Page.setInterceptFileChooserDialog",
|
||||||
|
params: { enabled: true },
|
||||||
|
});
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "Page.setInterceptFileChooserDialog",
|
||||||
|
params: { enabled: false, cancel: true },
|
||||||
|
});
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "DOM.setFileInputFiles",
|
||||||
|
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails immediately when the trigger does not activate a file input", async () => {
|
||||||
|
const { cdp } = uploadCdp({ inputCount: 0 });
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "unsupported",
|
||||||
|
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails before clicking when chooser interception is unavailable", async () => {
|
||||||
|
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Page.setInterceptFileChooserDialog") {
|
||||||
|
throw new Error("method unavailable");
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { effect_state: "none", phase: "arm_interception" },
|
||||||
|
});
|
||||||
|
expect(trigger).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses an exact chooser event as an independent input-location signal", async () => {
|
||||||
|
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget("f1"),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ multiple: false });
|
||||||
|
expect(calls).toContainEqual({
|
||||||
|
method: "DOM.setFileInputFiles",
|
||||||
|
params: { files: ["/private/stage/one"], backendNodeId: 789 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a File System Access picker without waiting for a timeout", async () => {
|
||||||
|
const { cdp, emitChooser } = uploadCdp({
|
||||||
|
inputCount: 0,
|
||||||
|
});
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget("f1"),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
emitChooser({ frameId: "f1", mode: "selectSingle" });
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "unsupported",
|
||||||
|
message: "upload trigger invoked a non-input file picker",
|
||||||
|
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds a stuck file-input probe", async () => {
|
||||||
|
const { cdp } = uploadCdp({ pendingResolve: true });
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 5,
|
||||||
|
trigger: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||||
|
const fixture = uploadCdp();
|
||||||
|
const originalSend = fixture.cdp.send;
|
||||||
|
const detach = vi.fn(async () => {});
|
||||||
|
fixture.cdp.detach = detach;
|
||||||
|
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
|
||||||
|
return originalSend(tabId, method, params);
|
||||||
|
}) as CdpRunner["send"];
|
||||||
|
|
||||||
|
const result = await uploadThroughActivatedFileInput({
|
||||||
|
cdp: fixture.cdp,
|
||||||
|
actionTarget: actionTarget(),
|
||||||
|
files: ["/private/stage/one"],
|
||||||
|
timeoutMs: 10,
|
||||||
|
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
data: { effect_state: "unknown", phase: "set_files" },
|
||||||
|
});
|
||||||
|
expect(detach).toHaveBeenCalledWith(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||||
|
const manager = sessions();
|
||||||
|
const ctx = await manager.start("s1");
|
||||||
|
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 9,
|
||||||
|
url: "https://example.test/result.zip",
|
||||||
|
finalUrl: "https://example.test/result.zip",
|
||||||
|
filename: "result.zip",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: 12,
|
||||||
|
mime: "application/zip",
|
||||||
|
danger: "safe",
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const completed = {
|
||||||
|
...initial,
|
||||||
|
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 12,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => [completed]),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchMouseEvent" &&
|
||||||
|
(params as { type?: string }).type === "mousePressed"
|
||||||
|
) {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: "result.zip",
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, (value) => {
|
||||||
|
suggested = value;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
onCreated.emit(completed);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await handleDownload(
|
||||||
|
manager,
|
||||||
|
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||||
|
{ cdp, tabsApi: tabsApi(), downloads },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(suggested).toEqual({
|
||||||
|
filename: "BrowserSkill/tr_1/result.zip",
|
||||||
|
conflictAction: "overwrite",
|
||||||
|
});
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
tab_id: 4,
|
||||||
|
suggested_filename: "result.zip",
|
||||||
|
byte_size: 12,
|
||||||
|
browser_path: completed.filename,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not claim a download without an intent from the exact target", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => []),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const unrelated = {
|
||||||
|
id: 17,
|
||||||
|
url: "https://example.test/unrelated.zip",
|
||||||
|
finalUrl: "https://example.test/unrelated.zip",
|
||||||
|
filename: "unrelated.zip",
|
||||||
|
state: "in_progress",
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
let defaultSuggestionCalled = false;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4, sessionId: "expected-child" },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_1",
|
||||||
|
timeoutMs: 5,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
|
||||||
|
url: unrelated.url,
|
||||||
|
suggestedFilename: unrelated.filename,
|
||||||
|
});
|
||||||
|
onDeterminingFilename.emit(unrelated, (suggestion) => {
|
||||||
|
defaultSuggestionCalled = suggestion === undefined;
|
||||||
|
});
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(defaultSuggestionCalled).toBe(true);
|
||||||
|
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "download_capture_failed" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("correlates a filename candidate that arrives before the CDP intent", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const initial = {
|
||||||
|
id: 21,
|
||||||
|
url: "https://example.test/candidate-first.bin",
|
||||||
|
finalUrl: "https://example.test/candidate-first.bin",
|
||||||
|
filename: "candidate-first.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
fileSize: -1,
|
||||||
|
totalBytes: 4,
|
||||||
|
bytesReceived: 0,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const complete = {
|
||||||
|
...initial,
|
||||||
|
state: "complete",
|
||||||
|
fileSize: 4,
|
||||||
|
} as chrome.downloads.DownloadItem;
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => [complete]),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_21",
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
trigger: async () => {
|
||||||
|
const suggested = new Promise<void>((resolve) => {
|
||||||
|
onDeterminingFilename.emit(initial, (value) => {
|
||||||
|
suggestion = value;
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: initial.url,
|
||||||
|
suggestedFilename: initial.filename,
|
||||||
|
});
|
||||||
|
await suggested;
|
||||||
|
onCreated.emit(complete);
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(suggestion).toEqual({
|
||||||
|
filename: "BrowserSkill/tr_21/candidate-first.bin",
|
||||||
|
conflictAction: "overwrite",
|
||||||
|
});
|
||||||
|
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
|
||||||
|
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||||
|
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||||
|
const onDeterminingFilename =
|
||||||
|
fakeEvent<
|
||||||
|
(
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true
|
||||||
|
>();
|
||||||
|
const downloads: DownloadsApi = {
|
||||||
|
onCreated,
|
||||||
|
onChanged,
|
||||||
|
onDeterminingFilename,
|
||||||
|
search: vi.fn(async () => []),
|
||||||
|
cancel: vi.fn(async () => {}),
|
||||||
|
removeFile: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
onEvent: (handler) => {
|
||||||
|
cdpEvent = handler;
|
||||||
|
return { dispose: vi.fn() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const defaults: number[] = [];
|
||||||
|
const candidate = (id: number) =>
|
||||||
|
({
|
||||||
|
id,
|
||||||
|
url: "https://example.test/same.bin",
|
||||||
|
finalUrl: "https://example.test/same.bin",
|
||||||
|
filename: "same.bin",
|
||||||
|
state: "in_progress",
|
||||||
|
}) as chrome.downloads.DownloadItem;
|
||||||
|
|
||||||
|
const result = await captureBrowserDownload({
|
||||||
|
cdp,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
downloads,
|
||||||
|
browserRelativeDir: "BrowserSkill/tr_ambiguous",
|
||||||
|
timeoutMs: 100,
|
||||||
|
trigger: async () => {
|
||||||
|
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||||
|
url: "https://example.test/same.bin",
|
||||||
|
suggestedFilename: "same.bin",
|
||||||
|
});
|
||||||
|
for (const id of [31, 32]) {
|
||||||
|
onDeterminingFilename.emit(candidate(id), (value) => {
|
||||||
|
if (value === undefined) defaults.push(id);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { tab_id: 4, x: 10, y: 10 };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(defaults.sort()).toEqual([31, 32]);
|
||||||
|
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { effect_state: "unknown", phase: "attribution" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||||
|
import {
|
||||||
|
clipPolygon,
|
||||||
|
polygonArea,
|
||||||
|
polygonBounds,
|
||||||
|
projectAndClipRegion,
|
||||||
|
projectUnitPoint,
|
||||||
|
type Quad,
|
||||||
|
rectPolygon,
|
||||||
|
regionBounds,
|
||||||
|
} from "../geometry";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
describe("frame geometry projection", () => {
|
||||||
|
it("keeps region bounds separate from polygon area", () => {
|
||||||
|
const region = [
|
||||||
|
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||||
|
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||||
|
];
|
||||||
|
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||||
|
expect(polygonArea(region[0])).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps through the iframe content box instead of its border box", () => {
|
||||||
|
const contentQuad: Quad = [
|
||||||
|
{ x: 204, y: 306 },
|
||||||
|
{ x: 604, y: 306 },
|
||||||
|
{ x: 604, y: 506 },
|
||||||
|
{ x: 204, y: 506 },
|
||||||
|
];
|
||||||
|
const projected = projectAndClipRegion(
|
||||||
|
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||||
|
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||||
|
{ width: 1000, height: 800 },
|
||||||
|
);
|
||||||
|
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||||
|
const quad: Quad = [
|
||||||
|
{ x: 0, y: 0 },
|
||||||
|
{ x: 200, y: 20 },
|
||||||
|
{ x: 180, y: 140 },
|
||||||
|
{ x: 20, y: 100 },
|
||||||
|
];
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||||
|
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||||
|
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||||
|
const frameQuad: Quad = [
|
||||||
|
{ x: 50, y: 0 },
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 50, y: 100 },
|
||||||
|
{ x: 0, y: 50 },
|
||||||
|
];
|
||||||
|
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||||
|
const send = vi.fn(async (_tabId, method, params) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||||
|
return backendNodeId === 20
|
||||||
|
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||||
|
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
});
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: send as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
const projection = await resolveFrameProjection(
|
||||||
|
cdp,
|
||||||
|
{
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "same-process-parent",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "oopif",
|
||||||
|
parentFrameId: "same-process-parent",
|
||||||
|
ownerBackendNodeId: 20,
|
||||||
|
target: { tabId: 4, sessionId: "oopif-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"oopif",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(projection?.edges).toEqual([
|
||||||
|
{
|
||||||
|
sourceViewport: { width: 200, height: 100 },
|
||||||
|
destinationQuad: [
|
||||||
|
{ x: 110, y: 70 },
|
||||||
|
{ x: 310, y: 70 },
|
||||||
|
{ x: 310, y: 170 },
|
||||||
|
{ x: 110, y: 170 },
|
||||||
|
],
|
||||||
|
destinationClips: [
|
||||||
|
[
|
||||||
|
{ x: 100, y: 50 },
|
||||||
|
{ x: 400, y: 50 },
|
||||||
|
{ x: 400, y: 250 },
|
||||||
|
{ x: 100, y: 250 },
|
||||||
|
],
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||||
|
const getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root command ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child command ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
getFrameGraph,
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(
|
||||||
|
cdp,
|
||||||
|
4,
|
||||||
|
{
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
},
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
),
|
||||||
|
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||||
|
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["send"],
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 10,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
|
||||||
|
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4 },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(geometry).toMatchObject({
|
||||||
|
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||||
|
actionPoint: { x: 200, y: 150 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(),
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveNodeGeometry(cdp, 4, {
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
frameId: "child",
|
||||||
|
backendNodeId: 101,
|
||||||
|
}),
|
||||||
|
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,14 +1,60 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { RefStore } from "@/session-manager/ref-store";
|
||||||
import type { RequestHelpParams } from "@/transport/types";
|
import type { RequestHelpParams } from "@/transport/types";
|
||||||
import { handleRequestHelp, type RequestHelpDeps } from "../human-loop";
|
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
|
|
||||||
|
function chromeEvent<T extends (...args: never[]) => unknown>() {
|
||||||
|
const listeners = new Set<T>();
|
||||||
|
return {
|
||||||
|
addListener: vi.fn((listener: T) => {
|
||||||
|
listeners.add(listener);
|
||||||
|
}),
|
||||||
|
removeListener: vi.fn((listener: T) => {
|
||||||
|
listeners.delete(listener);
|
||||||
|
}),
|
||||||
|
emit: (...args: Parameters<T>) => {
|
||||||
|
for (const listener of [...listeners]) listener(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function installHelpLifecycleChrome() {
|
||||||
|
const runtimeOnMessage =
|
||||||
|
chromeEvent<
|
||||||
|
(
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => unknown
|
||||||
|
>();
|
||||||
|
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
|
||||||
|
const tabsOnCreated = chromeEvent<(tab: chrome.tabs.Tab) => unknown>();
|
||||||
|
const tabsOnUpdated =
|
||||||
|
chromeEvent<(tabId: number, changeInfo: chrome.tabs.TabChangeInfo) => unknown>();
|
||||||
|
const webNavigationOnCompleted =
|
||||||
|
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
|
||||||
|
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
runtime: { onMessage: runtimeOnMessage },
|
||||||
|
tabs: { onActivated: tabsOnActivated, onCreated: tabsOnCreated, onUpdated: tabsOnUpdated },
|
||||||
|
webNavigation: { onCompleted: webNavigationOnCompleted },
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
runtimeOnMessage,
|
||||||
|
tabsOnActivated,
|
||||||
|
tabsOnCreated,
|
||||||
|
tabsOnUpdated,
|
||||||
|
webNavigationOnCompleted,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||||
|
const refStore = new RefStore();
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === sessionId
|
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
|
||||||
: null,
|
|
||||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
return mgr;
|
return mgr;
|
||||||
@@ -27,14 +73,19 @@ function baseDeps(over: Partial<RequestHelpDeps> = {}): RequestHelpDeps {
|
|||||||
windows: { update: vi.fn(async () => ({}) as never) },
|
windows: { update: vi.fn(async () => ({}) as never) },
|
||||||
activateTab: vi.fn(async () => {}),
|
activateTab: vi.fn(async () => {}),
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "continued", note: "ok" })),
|
||||||
watchTabNavigation: () => () => {},
|
|
||||||
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
cdp: { send: vi.fn(async () => ({})) } as unknown as RequestHelpDeps["cdp"],
|
||||||
notifications: null,
|
notifications: null,
|
||||||
|
autoAttachLifecycle: false,
|
||||||
...over,
|
...over,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleRequestHelp", () => {
|
describe("handleRequestHelp", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
resetHelpLifecycleForTests();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects unknown session", async () => {
|
it("rejects unknown session", async () => {
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
fakeManager("abcd", 99, 5),
|
fakeManager("abcd", 99, 5),
|
||||||
@@ -56,6 +107,30 @@ describe("handleRequestHelp", () => {
|
|||||||
expect(res).toMatchObject({ outcome: "continued", note: "ok", tab_id: 5 });
|
expect(res).toMatchObject({ outcome: "continued", note: "ok", tab_id: 5 });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not refresh observations after the user returns control", async () => {
|
||||||
|
const cdpSend = vi.fn(async () => ({}));
|
||||||
|
const deps = baseDeps({
|
||||||
|
cdp: { send: cdpSend } as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({ tab_id: 5 }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({ outcome: "continued", tab_id: 5 });
|
||||||
|
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||||
|
expect.any(Number),
|
||||||
|
"Accessibility.getFullAXTree",
|
||||||
|
expect.anything(),
|
||||||
|
);
|
||||||
|
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||||
|
expect.any(Number),
|
||||||
|
"DOMSnapshot.captureSnapshot",
|
||||||
|
expect.anything(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("forwards title into the help request message when provided", async () => {
|
it("forwards title into the help request message when provided", async () => {
|
||||||
const deps = baseDeps();
|
const deps = baseDeps();
|
||||||
await handleRequestHelp(
|
await handleRequestHelp(
|
||||||
@@ -78,22 +153,29 @@ describe("handleRequestHelp", () => {
|
|||||||
expect(sentMsg.title).toBeUndefined();
|
expect(sentMsg.title).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns navigated when the tab navigates during the wait", async () => {
|
it("does not complete merely because the tab navigates during the wait", async () => {
|
||||||
const unwatch = vi.fn();
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(() => new Promise(() => {})),
|
sendToTab: vi.fn(() => new Promise(() => {})),
|
||||||
watchTabNavigation: vi.fn((_tabId, cb) => {
|
|
||||||
queueMicrotask(() => cb());
|
|
||||||
return unwatch;
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const pending = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect(res).toMatchObject({ outcome: "navigated", tab_id: 5 });
|
);
|
||||||
expect(unwatch).toHaveBeenCalled();
|
await vi.advanceTimersByTimeAsync(5);
|
||||||
|
let settled = false;
|
||||||
|
void pending.then(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
await expect(pending).resolves.toMatchObject({ outcome: "timed_out", tab_id: 5 });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns timed_out when the wait expires", async () => {
|
it("returns timed_out when the wait expires", async () => {
|
||||||
@@ -115,14 +197,165 @@ describe("handleRequestHelp", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("tags ref targets via CDP and reports them matched", async () => {
|
it("returns completed when explicit completion criteria match", async () => {
|
||||||
|
const deps = baseDeps({
|
||||||
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||||
|
cdp: {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "Runtime.evaluate") return { result: { value: true } };
|
||||||
|
return {};
|
||||||
|
}),
|
||||||
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({
|
||||||
|
tab_id: 5,
|
||||||
|
completion_criteria: {
|
||||||
|
any: [{ selector_exists: "#account-menu" }],
|
||||||
|
stable_for_ms: 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
expect(res).toMatchObject({ outcome: "completed", completed_by: "system", tab_id: 5 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps user control active on new tabs without moving completion off the primary tab", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const chromeEvents = installHelpLifecycleChrome();
|
||||||
|
const sendToTab = vi.fn(async () => ({ type: "bsk-help-ack", ok: true }));
|
||||||
|
const deps = baseDeps({
|
||||||
|
autoAttachLifecycle: undefined,
|
||||||
|
sendToTab,
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(async (tabId: number) => ({
|
||||||
|
id: tabId,
|
||||||
|
windowId: 99,
|
||||||
|
active: tabId === 6,
|
||||||
|
url: tabId === 6 ? "https://app.example/reset/success" : "https://app.example/login",
|
||||||
|
})) as never,
|
||||||
|
query: vi.fn(async () => [{ id: 5, windowId: 99, active: true }] as never),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pending = handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({
|
||||||
|
tab_id: 5,
|
||||||
|
timeout_ms: 1_000,
|
||||||
|
completion_criteria: {
|
||||||
|
any: [{ url_contains: "/reset/success" }],
|
||||||
|
stable_for_ms: 0,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||||
|
|
||||||
|
chromeEvents.tabsOnActivated.emit({ tabId: 6, windowId: 99 });
|
||||||
|
await vi.advanceTimersByTimeAsync(200);
|
||||||
|
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
expect.objectContaining({
|
||||||
|
type: "bsk-help-request",
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: [],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
await expect(pending).resolves.toMatchObject({
|
||||||
|
outcome: "timed_out",
|
||||||
|
tab_id: 5,
|
||||||
|
});
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
expect.objectContaining({ type: "bsk-help-cancel" }),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets same-window tabs query active help after content-script load", async () => {
|
||||||
|
const chromeEvents = installHelpLifecycleChrome();
|
||||||
|
const ac = new AbortController();
|
||||||
|
const cdpSend = vi.fn(async () => ({ root: { nodeId: 1 }, nodeIds: [] }));
|
||||||
|
const deps = baseDeps({
|
||||||
|
autoAttachLifecycle: undefined,
|
||||||
|
signal: ac.signal,
|
||||||
|
sendToTab: vi.fn(async () => ({ type: "bsk-help-ack", ok: true })),
|
||||||
|
cdp: {
|
||||||
|
send: cdpSend,
|
||||||
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const pending = handleRequestHelp(
|
||||||
|
fakeManager("abcd", 99, 5),
|
||||||
|
baseParams({ tab_id: 5, timeout_ms: 60_000 }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(deps.sendToTab).toHaveBeenCalledWith(5, expect.anything()));
|
||||||
|
|
||||||
|
const sameWindowResponse = vi.fn();
|
||||||
|
chromeEvents.runtimeOnMessage.emit(
|
||||||
|
{ type: "bsk-help-query" },
|
||||||
|
{ tab: { id: 6 } as chrome.tabs.Tab },
|
||||||
|
sameWindowResponse,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(sameWindowResponse).toHaveBeenCalledWith({
|
||||||
|
active: true,
|
||||||
|
request: expect.objectContaining({
|
||||||
|
requestId: expect.any(String),
|
||||||
|
prompt: "log in",
|
||||||
|
displayMode: "compact",
|
||||||
|
selectors: [],
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const subjectResponse = vi.fn();
|
||||||
|
chromeEvents.runtimeOnMessage.emit(
|
||||||
|
{ type: "bsk-help-query" },
|
||||||
|
{ tab: { id: 5 } as chrome.tabs.Tab },
|
||||||
|
subjectResponse,
|
||||||
|
);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(subjectResponse).toHaveBeenCalledWith({
|
||||||
|
active: true,
|
||||||
|
request: expect.objectContaining({
|
||||||
|
requestId: expect.any(String),
|
||||||
|
prompt: "log in",
|
||||||
|
displayMode: "full",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
ac.abort();
|
||||||
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
|
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||||
|
expect.anything(),
|
||||||
|
"DOM.removeAttribute",
|
||||||
|
expect.anything(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 5 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: { resolve: () => 42 },
|
refStore,
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
@@ -130,7 +363,17 @@ describe("handleRequestHelp", () => {
|
|||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
cdp: {
|
cdp: {
|
||||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
send: vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected ${method}`);
|
||||||
|
}),
|
||||||
} as unknown as RequestHelpDeps["cdp"],
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -139,14 +382,97 @@ describe("handleRequestHelp", () => {
|
|||||||
deps,
|
deps,
|
||||||
);
|
);
|
||||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
expect(sentMsg.selectors).toEqual([]);
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||||
expect(res).toMatchObject({
|
expect(res).toMatchObject({
|
||||||
outcome: "continued",
|
outcome: "continued",
|
||||||
tab_id: 5,
|
tab_id: 5,
|
||||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
resolved_targets: [
|
||||||
|
{
|
||||||
|
matched: true,
|
||||||
|
ref: "@e1",
|
||||||
|
},
|
||||||
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||||
|
const mgr = {
|
||||||
|
get: (id: string) =>
|
||||||
|
id === "abcd"
|
||||||
|
? {
|
||||||
|
sessionId: "abcd",
|
||||||
|
agentWindowId: 99,
|
||||||
|
refStore: {
|
||||||
|
resolveEntry: () => ({
|
||||||
|
backendNodeId: 42,
|
||||||
|
tabId: 5,
|
||||||
|
frameId: "child",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
generation: 1,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
borrowedTabs: new Map(),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||||
|
} as unknown as SessionManager;
|
||||||
|
const send = vi.fn(async (_tabId, method) => {
|
||||||
|
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getBoxModel") {
|
||||||
|
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected root ${method}`);
|
||||||
|
});
|
||||||
|
const sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child ${method}`);
|
||||||
|
});
|
||||||
|
const deps = baseDeps({
|
||||||
|
cdp: {
|
||||||
|
send,
|
||||||
|
sendToTarget,
|
||||||
|
getFrameGraph: vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 5 } },
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 5, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleRequestHelp(
|
||||||
|
mgr,
|
||||||
|
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
|
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||||
|
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||||
|
expect(sendToTarget).toHaveBeenCalledWith(
|
||||||
|
{ tabId: 5, sessionId: "child-session" },
|
||||||
|
"DOM.getContentQuads",
|
||||||
|
{ backendNodeId: 42 },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||||
const deps = baseDeps();
|
const deps = baseDeps();
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -162,16 +488,15 @@ describe("handleRequestHelp", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||||
|
const refStore = new RefStore();
|
||||||
|
refStore.set("e1", 42, { tabId: 4 });
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore: {
|
refStore,
|
||||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
|
||||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
|
||||||
},
|
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
@@ -190,38 +515,40 @@ describe("handleRequestHelp", () => {
|
|||||||
expect("code" in res).toBe(false);
|
expect("code" in res).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error when sendToTab rejects", async () => {
|
it("keeps waiting when the initial overlay delivery rejects", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
sendToTab: vi.fn(async () => {
|
sendToTab: vi.fn(async () => {
|
||||||
throw new Error("no receiver");
|
throw new Error("no receiver");
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
try {
|
||||||
fakeManager("abcd", 99, 5),
|
const res = handleRequestHelp(
|
||||||
baseParams({ tab_id: 5 }),
|
fakeManager("abcd", 99, 5),
|
||||||
deps,
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
);
|
deps,
|
||||||
expect("code" in res && res.code).toBe("protocol_error");
|
);
|
||||||
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns protocol_error for a malformed help response", async () => {
|
it("keeps waiting for explicit completion after a malformed help response", async () => {
|
||||||
const undefinedReply = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
vi.useFakeTimers();
|
||||||
const res1 = await handleRequestHelp(
|
const deps = baseDeps({ sendToTab: vi.fn(async () => undefined) });
|
||||||
fakeManager("abcd", 99, 5),
|
try {
|
||||||
baseParams({ tab_id: 5 }),
|
const res = handleRequestHelp(
|
||||||
undefinedReply,
|
fakeManager("abcd", 99, 5),
|
||||||
);
|
baseParams({ tab_id: 5, timeout_ms: 10 }),
|
||||||
expect("code" in res1 && res1.code).toBe("protocol_error");
|
deps,
|
||||||
|
);
|
||||||
const badOutcome = baseDeps({
|
await vi.advanceTimersByTimeAsync(20);
|
||||||
sendToTab: vi.fn(async () => ({ type: "bsk-help-response", outcome: "weird" })),
|
await expect(res).resolves.toMatchObject({ outcome: "timed_out" });
|
||||||
});
|
} finally {
|
||||||
const res2 = await handleRequestHelp(
|
vi.useRealTimers();
|
||||||
fakeManager("abcd", 99, 5),
|
}
|
||||||
baseParams({ tab_id: 5 }),
|
|
||||||
badOutcome,
|
|
||||||
);
|
|
||||||
expect("code" in res2 && res2.code).toBe("protocol_error");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reports selector match status from CDP", async () => {
|
it("reports selector match status from CDP", async () => {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
|||||||
import {
|
import {
|
||||||
handleClick,
|
handleClick,
|
||||||
handleFill,
|
handleFill,
|
||||||
|
handleHover,
|
||||||
handlePress,
|
handlePress,
|
||||||
handleSelect,
|
handleSelect,
|
||||||
modifiersBitfield,
|
modifiersBitfield,
|
||||||
@@ -29,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||||
sent.push({ tabId, method, params });
|
sent.push({ tabId, method, params });
|
||||||
const h = handlers[method];
|
const h = handlers[method];
|
||||||
|
if (!h && method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
|
}
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -163,6 +167,63 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getBoxModel": () => ({
|
||||||
|
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||||
|
targetCalls.push({ sessionId: target.sessionId, method });
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child CDP call ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handleClick(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||||
|
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||||
|
expect(targetCalls).toEqual([
|
||||||
|
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||||
|
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||||
|
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||||
|
]);
|
||||||
|
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||||
const order: string[] = [];
|
const order: string[] = [];
|
||||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
@@ -435,6 +496,50 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleHover", () => {
|
||||||
|
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||||
|
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||||
|
"Runtime.evaluate": (params: unknown) => {
|
||||||
|
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||||
|
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||||
|
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||||
|
}
|
||||||
|
if (expr.includes("hitIndex")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||||
|
},
|
||||||
|
"Input.dispatchMouseEvent": () => ({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleHover(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||||
|
{
|
||||||
|
cdp: fake.cdp,
|
||||||
|
tabsApi: fake.tabsApi,
|
||||||
|
bypassOverlay,
|
||||||
|
keepOverlayBypassAfterHover: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||||
|
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("handleFill", () => {
|
describe("handleFill", () => {
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
|||||||
@@ -49,6 +49,8 @@ function makeFakeCdp(opts?: {
|
|||||||
beforeLoaderId?: string;
|
beforeLoaderId?: string;
|
||||||
readyState?: string;
|
readyState?: string;
|
||||||
readyStateSequence?: string[];
|
readyStateSequence?: string[];
|
||||||
|
/** Make the readyState probe's `Runtime.evaluate` throw (returns null). */
|
||||||
|
evaluateThrows?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const readyStates = opts?.readyStateSequence ?? (opts?.readyState ? [opts.readyState] : []);
|
const readyStates = opts?.readyStateSequence ?? (opts?.readyState ? [opts.readyState] : []);
|
||||||
let readyStateIdx = 0;
|
let readyStateIdx = 0;
|
||||||
@@ -94,7 +96,10 @@ function makeFakeCdp(opts?: {
|
|||||||
return {};
|
return {};
|
||||||
},
|
},
|
||||||
"Runtime.enable": () => ({}),
|
"Runtime.enable": () => ({}),
|
||||||
"Runtime.evaluate": () => ({ result: { value: nextReadyState() } }),
|
"Runtime.evaluate": () => {
|
||||||
|
if (opts?.evaluateThrows) throw new Error("Cannot execute in a destroyed execution context");
|
||||||
|
return { result: { value: nextReadyState() } };
|
||||||
|
},
|
||||||
"Page.getNavigationHistory": () => ({
|
"Page.getNavigationHistory": () => ({
|
||||||
currentIndex: opts?.historyIndex ?? 1,
|
currentIndex: opts?.historyIndex ?? 1,
|
||||||
entries: opts?.historyEntries ?? [
|
entries: opts?.historyEntries ?? [
|
||||||
@@ -168,8 +173,13 @@ function makeFakeCdp(opts?: {
|
|||||||
const payload = { name, frameId, loaderId };
|
const payload = { name, frameId, loaderId };
|
||||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload);
|
for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload);
|
||||||
},
|
},
|
||||||
fireFrameNavigated(frameId = opts?.navigateFrameId ?? "frame-1") {
|
fireFrameNavigated(
|
||||||
const payload = { frame: { id: frameId, url: opts?.finalUrl ?? "https://example.com/" } };
|
frameId = opts?.navigateFrameId ?? "frame-1",
|
||||||
|
loaderId = opts?.navigateLoaderId ?? "loader-after",
|
||||||
|
) {
|
||||||
|
const payload = {
|
||||||
|
frame: { id: frameId, loaderId, url: opts?.finalUrl ?? "https://example.com/" },
|
||||||
|
};
|
||||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload);
|
for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload);
|
||||||
},
|
},
|
||||||
listeners: events,
|
listeners: events,
|
||||||
@@ -457,6 +467,34 @@ describe("handleNavigate", () => {
|
|||||||
expect(res.reached).toBe("commit");
|
expect(res.reached).toBe("commit");
|
||||||
expect(fake.listeners.length).toBe(0);
|
expect(fake.listeners.length).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resolves wait_until=commit from Page.frameNavigated when the readyState probe fails", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
// Cross-origin-style navigation: the readyState probe can't read the
|
||||||
|
// (already destroyed) old execution context, so commit must come from
|
||||||
|
// the Page.frameNavigated event. Before the fix this timed out because
|
||||||
|
// the handler dropped the frame's loaderId and the loader guard then
|
||||||
|
// rejected the event.
|
||||||
|
const fake = makeFakeCdp({ evaluateThrows: true });
|
||||||
|
const navP = handleNavigate(
|
||||||
|
sm,
|
||||||
|
{
|
||||||
|
session_id: "aa11",
|
||||||
|
url: "https://example.com/",
|
||||||
|
wait_until: "commit",
|
||||||
|
timeout_ms: 1_000,
|
||||||
|
},
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 5));
|
||||||
|
fake.fireFrameNavigated("frame-1", "loader-after");
|
||||||
|
const res = await navP;
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.reached).toBe("commit");
|
||||||
|
expect(fake.listeners.length).toBe(0);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("handleNavigateBack / Forward", () => {
|
describe("handleNavigateBack / Forward", () => {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -60,6 +60,7 @@ function makeApis(
|
|||||||
}),
|
}),
|
||||||
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
||||||
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
};
|
};
|
||||||
return { tabs, windows };
|
return { tabs, windows };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import { lookupSession, parseBufferedReadBounds } from "../shared";
|
import {
|
||||||
|
cdpBlockedUrlReason,
|
||||||
|
enforceCdpAccessibleTarget,
|
||||||
|
lookupSession,
|
||||||
|
parseBufferedReadBounds,
|
||||||
|
resolveCdpAccessibleTargetTab,
|
||||||
|
} from "../shared";
|
||||||
|
|
||||||
function fakeAgentWindow() {
|
function fakeAgentWindow() {
|
||||||
return {
|
return {
|
||||||
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("CDP target URL guard", () => {
|
||||||
|
it("allows ordinary pages and about:blank", () => {
|
||||||
|
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
|
||||||
|
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
|
||||||
|
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("blocks browser and extension internal pages", () => {
|
||||||
|
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
|
||||||
|
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
|
||||||
|
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns a structured permission error before page CDP access", () => {
|
||||||
|
expect(
|
||||||
|
enforceCdpAccessibleTarget(
|
||||||
|
{
|
||||||
|
tabId: 7,
|
||||||
|
windowId: 100,
|
||||||
|
active: true,
|
||||||
|
url: "chrome-extension://other-extension/page.html",
|
||||||
|
},
|
||||||
|
"snapshot",
|
||||||
|
),
|
||||||
|
).toMatchObject({
|
||||||
|
code: "permission_denied",
|
||||||
|
data: { reason: "restricted_tab_url" },
|
||||||
|
message: expect.stringContaining("snapshot cannot access tab 7"),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
const tabsApi = {
|
||||||
|
get: vi.fn(),
|
||||||
|
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
|
||||||
|
if (query.active) {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
id: 7,
|
||||||
|
windowId: ctx.agentWindowId,
|
||||||
|
active: true,
|
||||||
|
url: "chrome-extension://other-extension/page.html",
|
||||||
|
} as chrome.tabs.Tab,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
id: 7,
|
||||||
|
windowId: ctx.agentWindowId,
|
||||||
|
active: true,
|
||||||
|
url: "chrome-extension://other-extension/page.html",
|
||||||
|
} as chrome.tabs.Tab,
|
||||||
|
{
|
||||||
|
id: 8,
|
||||||
|
windowId: ctx.agentWindowId,
|
||||||
|
active: false,
|
||||||
|
url: "https://example.test/",
|
||||||
|
} as chrome.tabs.Tab,
|
||||||
|
];
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
|
||||||
|
).resolves.toMatchObject({
|
||||||
|
tabId: 8,
|
||||||
|
url: "https://example.test/",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not replace an explicitly requested restricted tab", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
const tabsApi = {
|
||||||
|
get: vi.fn(
|
||||||
|
async () =>
|
||||||
|
({
|
||||||
|
id: 7,
|
||||||
|
windowId: ctx.agentWindowId,
|
||||||
|
active: true,
|
||||||
|
url: "chrome-extension://other-extension/page.html",
|
||||||
|
}) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
|
||||||
|
).resolves.toMatchObject({
|
||||||
|
code: "permission_denied",
|
||||||
|
data: { reason: "restricted_tab_url" },
|
||||||
|
});
|
||||||
|
expect(tabsApi.query).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("resolveSnapshotRef", () => {
|
describe("resolveSnapshotRef", () => {
|
||||||
|
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e3", 1234, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
const expected = {
|
||||||
|
backendNodeId: 1234,
|
||||||
|
refKey: "e3",
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
};
|
||||||
|
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
|
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -179,6 +179,7 @@ function makeWindowsApi(
|
|||||||
get: ReturnType<typeof vi.fn>;
|
get: ReturnType<typeof vi.fn>;
|
||||||
lastFocused: ReturnType<typeof vi.fn>;
|
lastFocused: ReturnType<typeof vi.fn>;
|
||||||
create: ReturnType<typeof vi.fn>;
|
create: ReturnType<typeof vi.fn>;
|
||||||
|
remove: ReturnType<typeof vi.fn>;
|
||||||
};
|
};
|
||||||
} {
|
} {
|
||||||
const get = vi.fn(async (windowId: number) => {
|
const get = vi.fn(async (windowId: number) => {
|
||||||
@@ -197,7 +198,13 @@ function makeWindowsApi(
|
|||||||
const id = opts?.createWindowId ?? 999;
|
const id = opts?.createWindowId ?? 999;
|
||||||
return { id } as chrome.windows.Window;
|
return { id } as chrome.windows.Window;
|
||||||
});
|
});
|
||||||
return { api: { get, getLastFocused: lastFocused, create }, spies: { get, lastFocused, create } };
|
const remove = vi.fn(async (windowId: number) => {
|
||||||
|
state.windowsClosed.add(windowId);
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
api: { get, getLastFocused: lastFocused, create, remove },
|
||||||
|
spies: { get, lastFocused, create, remove },
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("handleTabCreate", () => {
|
describe("handleTabCreate", () => {
|
||||||
@@ -601,6 +608,86 @@ describe("handleTabReturn", () => {
|
|||||||
expect(winSpies.create).toHaveBeenCalledOnce();
|
expect(winSpies.create).toHaveBeenCalledOnce();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("closes a newly created fallback window when cancellation wins before the move", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||||
|
const state: FakeTabState = {
|
||||||
|
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||||
|
nextTabId: 50,
|
||||||
|
windowsClosed: new Set([200]),
|
||||||
|
};
|
||||||
|
const { api, spies } = makeTabMutationApi(state);
|
||||||
|
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||||
|
lastFocused: 100,
|
||||||
|
createWindowId: 777,
|
||||||
|
});
|
||||||
|
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||||
|
winSpies.create.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
const pending = handleTabReturn(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||||
|
|
||||||
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
|
expect(winSpies.remove).toHaveBeenCalledWith(777);
|
||||||
|
expect(spies.move).not.toHaveBeenCalled();
|
||||||
|
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces the fallback window id when cancellation cleanup fails", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||||
|
const state: FakeTabState = {
|
||||||
|
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||||
|
nextTabId: 50,
|
||||||
|
windowsClosed: new Set([200]),
|
||||||
|
};
|
||||||
|
const { api, spies } = makeTabMutationApi(state);
|
||||||
|
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||||
|
lastFocused: 100,
|
||||||
|
createWindowId: 777,
|
||||||
|
});
|
||||||
|
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||||
|
winSpies.create.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolveCreate = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
winSpies.remove.mockRejectedValueOnce(new Error("window removal denied"));
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
const pending = handleTabReturn(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", tab_id: 7 },
|
||||||
|
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||||
|
controller.abort();
|
||||||
|
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||||
|
|
||||||
|
await expect(pending).resolves.toMatchObject({
|
||||||
|
code: "protocol_error",
|
||||||
|
data: { reason: "cleanup_failed", resource_type: "window", resource_id: 777 },
|
||||||
|
message: expect.stringMatching(/fallback window 777.*window removal denied/),
|
||||||
|
});
|
||||||
|
expect(spies.move).not.toHaveBeenCalled();
|
||||||
|
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
+146
@@ -0,0 +1,146 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { handleSessionStart } from "../session";
|
||||||
|
import { handleWindowResize, type WindowResizeApi } from "../window";
|
||||||
|
|
||||||
|
function fakeAgentWindow(ids: number[]) {
|
||||||
|
let i = 0;
|
||||||
|
const create = vi.fn(async () => {
|
||||||
|
const id = ids[i++];
|
||||||
|
if (id === undefined) throw new Error("ran out of fake ids");
|
||||||
|
return id;
|
||||||
|
});
|
||||||
|
const remove = vi.fn(async () => {});
|
||||||
|
const ensureActiveTab = vi.fn(async () => {});
|
||||||
|
return { create, remove, ensureActiveTab };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function makeManager(): Promise<SessionManager> {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
return sm;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeUpdateApi(opts?: { throw?: boolean }) {
|
||||||
|
const calls: Array<{ windowId: number; width: number; height: number }> = [];
|
||||||
|
const api: WindowResizeApi = {
|
||||||
|
update: vi.fn(async (windowId: number, updateInfo: { width: number; height: number }) => {
|
||||||
|
if (opts?.throw) throw new Error("simulated chrome failure");
|
||||||
|
calls.push({ windowId, width: updateInfo.width, height: updateInfo.height });
|
||||||
|
return undefined;
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
return { api, calls };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("handleWindowResize", () => {
|
||||||
|
it("resizes the session's Agent Window", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api, calls } = fakeUpdateApi();
|
||||||
|
const result = await handleWindowResize(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", width: 1280, height: 800 },
|
||||||
|
api,
|
||||||
|
);
|
||||||
|
expect(result).toEqual({ window_id: 100, width: 1280, height: 800 });
|
||||||
|
expect(calls).toEqual([{ windowId: 100, width: 1280, height: 800 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an unknown session", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api } = fakeUpdateApi();
|
||||||
|
const result = await handleWindowResize(
|
||||||
|
sm,
|
||||||
|
{ session_id: "zz99", width: 1280, height: 800 },
|
||||||
|
api,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "not_found" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects missing session_id", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api } = fakeUpdateApi();
|
||||||
|
const result = await handleWindowResize(sm, { session_id: "", width: 1280, height: 800 }, api);
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects missing dimensions", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api, calls } = fakeUpdateApi();
|
||||||
|
const result = await handleWindowResize(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" } as unknown as { session_id: string; width: number; height: number },
|
||||||
|
api,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(calls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
[99, 800],
|
||||||
|
[100, 7681],
|
||||||
|
[1280.5, 800],
|
||||||
|
[Number.NaN, 800],
|
||||||
|
])("rejects out-of-range or non-integer dimensions (%s, %s)", async (width, height) => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api, calls } = fakeUpdateApi();
|
||||||
|
const result = await handleWindowResize(sm, { session_id: "aa11", width, height }, api);
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(calls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps chrome API failures to protocol_error", async () => {
|
||||||
|
const sm = await makeManager();
|
||||||
|
const { api } = fakeUpdateApi({ throw: true });
|
||||||
|
const result = await handleWindowResize(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", width: 1280, height: 800 },
|
||||||
|
api,
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ code: "protocol_error", message: "simulated chrome failure" });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("handleSessionStart window size", () => {
|
||||||
|
it("passes width/height through to Agent Window creation", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11", width: 1280, height: 800 });
|
||||||
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
|
expect(aw.create).toHaveBeenCalledWith("about:blank", { size: { width: 1280, height: 800 } });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("creates the window without size when width/height are omitted", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11" });
|
||||||
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
|
expect(aw.create).toHaveBeenCalledWith("about:blank", {});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forwards focused: false to the Agent Window", async () => {
|
||||||
|
const aw = fakeAgentWindow([100]);
|
||||||
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11", focused: false });
|
||||||
|
expect(result).toEqual({ agent_window_id: 100 });
|
||||||
|
expect(aw.create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a lone width without height", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11", width: 1280 });
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
[99, 800],
|
||||||
|
[1280, 7681],
|
||||||
|
[1280.5, 800],
|
||||||
|
])("rejects invalid dimensions (%s, %s)", async (width, height) => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const result = await handleSessionStart(sm, { session_id: "aa11", width, height });
|
||||||
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
|
expect(sm.has("aa11")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
|||||||
// URL injection-eligibility test
|
// URL injection-eligibility test
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const CHROME_WEB_STORE_RES = [
|
const EXTENSION_STORE_RES = [
|
||||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||||
/^https:\/\/chromewebstore\.google\.com/i,
|
/^https:\/\/chromewebstore\.google\.com/i,
|
||||||
|
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
|
|||||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||||
|
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||||
*
|
*
|
||||||
* Mirrors the implicit scheme list documented in
|
* Mirrors the implicit scheme list documented in
|
||||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||||
*/
|
*/
|
||||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||||
if (!url) return false;
|
if (!url) return false;
|
||||||
for (const re of CHROME_WEB_STORE_RES) {
|
for (const re of EXTENSION_STORE_RES) {
|
||||||
if (re.test(url)) return false;
|
if (re.test(url)) return false;
|
||||||
}
|
}
|
||||||
return /^https?:\/\//i.test(url);
|
return /^https?:\/\//i.test(url);
|
||||||
|
|||||||
@@ -27,19 +27,23 @@ export async function handleConsole(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: ConsoleParams,
|
params: ConsoleParams,
|
||||||
deps: ConsoleDeps = defaultConsoleDeps(),
|
deps: ConsoleDeps = defaultConsoleDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<ConsoleResult | RpcError> {
|
): Promise<ConsoleResult | RpcError> {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
const ctxOrErr = lookupSession(manager, params, "console");
|
const ctxOrErr = lookupSession(manager, params, "console");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const bounds = parseBufferedReadBounds(params);
|
const bounds = parseBufferedReadBounds(params);
|
||||||
if (isRpcError(bounds)) return bounds;
|
if (isRpcError(bounds)) return bounds;
|
||||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
||||||
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await deps.cdp.ensureConsoleCapture(target.tabId);
|
await deps.cdp.ensureConsoleCapture(target.tabId);
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||||
return deps.cdp.consoleEntriesSince(
|
return deps.cdp.consoleEntriesSince(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
|
|||||||
@@ -4,13 +4,18 @@ import type { Transport } from "@/transport/transport";
|
|||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
ConsoleParams,
|
ConsoleParams,
|
||||||
|
DownloadParams,
|
||||||
|
EmulateParams,
|
||||||
EvaluateParams,
|
EvaluateParams,
|
||||||
FillParams,
|
FillParams,
|
||||||
GetHtmlParams,
|
GetHtmlParams,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
NavigateBackParams,
|
NavigateBackParams,
|
||||||
NavigateForwardParams,
|
NavigateForwardParams,
|
||||||
NavigateParams,
|
NavigateParams,
|
||||||
NetworkParams,
|
NetworkParams,
|
||||||
|
ObserveParams,
|
||||||
PressParams,
|
PressParams,
|
||||||
ProtocolFrame,
|
ProtocolFrame,
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
@@ -24,13 +29,16 @@ import type {
|
|||||||
ScreenshotParams,
|
ScreenshotParams,
|
||||||
SelectParams,
|
SelectParams,
|
||||||
SnapshotParams,
|
SnapshotParams,
|
||||||
|
UploadParams,
|
||||||
WaitForNavigationParams,
|
WaitForNavigationParams,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { isRequestFrame } from "@/transport/types";
|
import { isRequestFrame } from "@/transport/types";
|
||||||
import { handleConsole } from "./console";
|
import { handleConsole } from "./console";
|
||||||
|
import { handleDownload } from "./download";
|
||||||
|
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { defaultWatchTabNavigation, handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
handleNavigateBack,
|
handleNavigateBack,
|
||||||
@@ -42,6 +50,7 @@ import {
|
|||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
chromeTabsCaptureApi,
|
chromeTabsCaptureApi,
|
||||||
handleGetHtml,
|
handleGetHtml,
|
||||||
|
handleObserve,
|
||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
handleSnapshot,
|
handleSnapshot,
|
||||||
} from "./observation";
|
} from "./observation";
|
||||||
@@ -52,7 +61,7 @@ import {
|
|||||||
type SessionStartParams,
|
type SessionStartParams,
|
||||||
type SessionStopParams,
|
type SessionStopParams,
|
||||||
} from "./session";
|
} from "./session";
|
||||||
import { chromeTabsApi } from "./shared";
|
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||||
import {
|
import {
|
||||||
type BorrowConfirmationApprover,
|
type BorrowConfirmationApprover,
|
||||||
handleTabBorrow,
|
handleTabBorrow,
|
||||||
@@ -68,13 +77,28 @@ import {
|
|||||||
type TabReturnParams,
|
type TabReturnParams,
|
||||||
type TabSelectParams,
|
type TabSelectParams,
|
||||||
} from "./tabs";
|
} from "./tabs";
|
||||||
|
import { handleUpload } from "./upload";
|
||||||
import { handleWaitForNavigation } from "./waits";
|
import { handleWaitForNavigation } from "./waits";
|
||||||
|
import { handleWindowResize, type WindowResizeParams } from "./window";
|
||||||
|
|
||||||
type DispatcherCdpRunner = CdpRunner &
|
type DispatcherCdpRunner = CdpRunner &
|
||||||
NetworkCdpRunner & {
|
NetworkCdpRunner &
|
||||||
|
EmulateCdpRunner & {
|
||||||
detachSession(sessionId: string): Promise<void>;
|
detachSession(sessionId: string): Promise<void>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
interface HoverLatch {
|
||||||
|
sessionId: string;
|
||||||
|
tabId: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HoverLatchScope {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DispatcherDeps {
|
export interface DispatcherDeps {
|
||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
@@ -86,6 +110,8 @@ export interface DispatcherDeps {
|
|||||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||||
*/
|
*/
|
||||||
onSessionsChanged?: () => void;
|
onSessionsChanged?: () => void;
|
||||||
|
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||||
|
onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||||
approveBorrow?: BorrowConfirmationApprover;
|
approveBorrow?: BorrowConfirmationApprover;
|
||||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||||
@@ -103,20 +129,22 @@ export interface DispatcherDeps {
|
|||||||
* `AbortController` keyed by its wire `id` in
|
* `AbortController` keyed by its wire `id` in
|
||||||
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
||||||
* request the dispatcher trips the matching controller; tool
|
* request the dispatcher trips the matching controller; tool
|
||||||
* handlers that already accept a `signal` (waits, navigation,
|
* handlers observe that signal between awaited operations. The
|
||||||
* interaction, evaluate, tabs) react in line, and the dispatcher
|
* original RPC remains pending until its handler has stopped or
|
||||||
* additionally races the in-flight invocation against the abort
|
* completed compensation; only the separate cancel acknowledgement
|
||||||
* promise so handlers without explicit signal plumbing still respond
|
* takes the fast path.
|
||||||
* promptly with `cancelled`.
|
|
||||||
*/
|
*/
|
||||||
export class ToolDispatcher {
|
export class ToolDispatcher {
|
||||||
private readonly transport: Transport;
|
private readonly transport: Transport;
|
||||||
private readonly sessions: SessionManager;
|
private readonly sessions: SessionManager;
|
||||||
private readonly cdp?: DispatcherCdpRunner;
|
private readonly cdp?: DispatcherCdpRunner;
|
||||||
private readonly onSessionsChanged?: () => void;
|
private readonly onSessionsChanged?: () => void;
|
||||||
|
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||||
private subscription: { dispose(): void } | null = null;
|
private subscription: { dispose(): void } | null = null;
|
||||||
|
private readonly hoverBypassTabs = new Map<number, string>();
|
||||||
|
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||||
/**
|
/**
|
||||||
* Per-rpc-id `AbortController` registry. Populated inside
|
* Per-rpc-id `AbortController` registry. Populated inside
|
||||||
* [`dispatch`] before we await the tool handler and torn down in
|
* [`dispatch`] before we await the tool handler and torn down in
|
||||||
@@ -130,6 +158,7 @@ export class ToolDispatcher {
|
|||||||
this.sessions = deps.sessions;
|
this.sessions = deps.sessions;
|
||||||
this.cdp = deps.cdp;
|
this.cdp = deps.cdp;
|
||||||
this.onSessionsChanged = deps.onSessionsChanged;
|
this.onSessionsChanged = deps.onSessionsChanged;
|
||||||
|
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||||
this.approveBorrow = deps.approveBorrow;
|
this.approveBorrow = deps.approveBorrow;
|
||||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||||
}
|
}
|
||||||
@@ -193,7 +222,9 @@ export class ToolDispatcher {
|
|||||||
let body: ResponseFrame;
|
let body: ResponseFrame;
|
||||||
let startedSession: string | null = null;
|
let startedSession: string | null = null;
|
||||||
try {
|
try {
|
||||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||||
|
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||||
|
const result = await this.invoke(req, ac.signal);
|
||||||
if (isRpcError(result)) {
|
if (isRpcError(result)) {
|
||||||
body = { id: req.id, error: result };
|
body = { id: req.id, error: result };
|
||||||
} else {
|
} else {
|
||||||
@@ -258,26 +289,45 @@ export class ToolDispatcher {
|
|||||||
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.session_start":
|
case "tool.session_start":
|
||||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
return handleSessionStart(this.sessions, req.params as SessionStartParams, { signal });
|
||||||
case "tool.session_stop":
|
case "tool.session_stop": {
|
||||||
|
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
case "tool.tab_list":
|
case "tool.tab_list":
|
||||||
return handleTabList(this.sessions, req.params as TabListParams);
|
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||||
case "tool.tab_close":
|
case "tool.tab_close":
|
||||||
return handleTabClose(this.sessions, req.params as TabCloseParams);
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as TabCloseParams,
|
||||||
|
() => handleTabClose(this.sessions, req.params as TabCloseParams, { signal }),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
case "tool.tab_select":
|
case "tool.tab_select":
|
||||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||||
case "tool.tab_borrow":
|
case "tool.tab_borrow":
|
||||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||||
signal,
|
signal,
|
||||||
approveBorrow: this.approveBorrow,
|
approveBorrow: this.approveBorrow,
|
||||||
});
|
});
|
||||||
case "tool.tab_return":
|
case "tool.tab_return":
|
||||||
return handleTabReturn(this.sessions, req.params as TabReturnParams);
|
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||||
|
case "tool.window_resize":
|
||||||
|
return handleWindowResize(
|
||||||
|
this.sessions,
|
||||||
|
req.params as WindowResizeParams,
|
||||||
|
undefined,
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
case "tool.emulate":
|
||||||
|
return handleEmulate(
|
||||||
|
this.sessions,
|
||||||
|
req.params as EmulateParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
);
|
||||||
case "tool.screenshot":
|
case "tool.screenshot":
|
||||||
return handleScreenshot(
|
return handleScreenshot(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
@@ -285,94 +335,212 @@ export class ToolDispatcher {
|
|||||||
this.cdp
|
this.cdp
|
||||||
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
||||||
: undefined,
|
: undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.console":
|
case "tool.console":
|
||||||
return handleConsole(
|
return handleConsole(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as ConsoleParams,
|
req.params as ConsoleParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.network":
|
case "tool.network":
|
||||||
return handleNetwork(
|
return handleNetwork(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as NetworkParams,
|
req.params as NetworkParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.snapshot":
|
case "tool.snapshot":
|
||||||
return handleSnapshot(
|
return this.withHoverReassert(
|
||||||
this.sessions,
|
|
||||||
req.params as SnapshotParams,
|
req.params as SnapshotParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
() =>
|
||||||
|
handleSnapshot(
|
||||||
|
this.sessions,
|
||||||
|
req.params as SnapshotParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
|
signal,
|
||||||
|
),
|
||||||
|
{},
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
|
case "tool.observe": {
|
||||||
|
const params = req.params as ObserveParams;
|
||||||
|
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
return this.withHoverReassert(
|
||||||
|
params,
|
||||||
|
() =>
|
||||||
|
handleObserve(
|
||||||
|
this.sessions,
|
||||||
|
params,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsCaptureApi,
|
||||||
|
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||||
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
signal,
|
||||||
|
),
|
||||||
|
{},
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
}
|
||||||
case "tool.get_html":
|
case "tool.get_html":
|
||||||
return handleGetHtml(
|
return handleGetHtml(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as GetHtmlParams,
|
req.params as GetHtmlParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate":
|
case "tool.navigate":
|
||||||
return handleNavigate(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateParams,
|
req.params as NavigateParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigate(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate_back":
|
case "tool.navigate_back":
|
||||||
return handleNavigateBack(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateBackParams,
|
req.params as NavigateBackParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigateBack(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateBackParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.navigate_forward":
|
case "tool.navigate_forward":
|
||||||
return handleNavigateForward(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as NavigateForwardParams,
|
req.params as NavigateForwardParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleNavigateForward(
|
||||||
|
this.sessions,
|
||||||
|
req.params as NavigateForwardParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.reload":
|
case "tool.reload":
|
||||||
return handleReload(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as ReloadParams,
|
req.params as ReloadParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleReload(
|
||||||
|
this.sessions,
|
||||||
|
req.params as ReloadParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.click":
|
case "tool.click":
|
||||||
return handleClick(
|
return this.withHoverReassert(
|
||||||
this.sessions,
|
|
||||||
req.params as ClickParams,
|
req.params as ClickParams,
|
||||||
|
() =>
|
||||||
|
handleClick(
|
||||||
|
this.sessions,
|
||||||
|
req.params as ClickParams,
|
||||||
|
this.cdp
|
||||||
|
? {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
signal,
|
||||||
|
bypassOverlay,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
),
|
||||||
|
{ releaseAfter: true },
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
case "tool.hover": {
|
||||||
|
const result = await handleHover(
|
||||||
|
this.sessions,
|
||||||
|
req.params as HoverParams,
|
||||||
this.cdp
|
this.cdp
|
||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
signal,
|
signal,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
bypassOverlay: (tabId, enabled) =>
|
||||||
try {
|
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
keepOverlayBypassAfterHover: true,
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
);
|
);
|
||||||
|
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||||
|
}
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
return handleFill(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as FillParams,
|
req.params as FillParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleFill(
|
||||||
|
this.sessions,
|
||||||
|
req.params as FillParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
return handlePress(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as PressParams,
|
req.params as PressParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handlePress(
|
||||||
|
this.sessions,
|
||||||
|
req.params as PressParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
return handleSelect(
|
return this.withHoverReleaseForRequest(
|
||||||
this.sessions,
|
|
||||||
req.params as SelectParams,
|
req.params as SelectParams,
|
||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
() =>
|
||||||
|
handleSelect(
|
||||||
|
this.sessions,
|
||||||
|
req.params as SelectParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
case "tool.upload":
|
||||||
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as UploadParams,
|
||||||
|
() =>
|
||||||
|
this.cdp
|
||||||
|
? handleUpload(this.sessions, req.params as UploadParams, {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
signal,
|
||||||
|
bypassOverlay,
|
||||||
|
})
|
||||||
|
: Promise.resolve({
|
||||||
|
code: "unsupported",
|
||||||
|
message: "upload requires CDP",
|
||||||
|
} satisfies RpcError),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
case "tool.download":
|
||||||
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as DownloadParams,
|
||||||
|
() =>
|
||||||
|
this.cdp
|
||||||
|
? handleDownload(this.sessions, req.params as DownloadParams, {
|
||||||
|
cdp: this.cdp,
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
signal,
|
||||||
|
bypassOverlay,
|
||||||
|
})
|
||||||
|
: Promise.resolve({
|
||||||
|
code: "unsupported",
|
||||||
|
message: "download requires CDP",
|
||||||
|
} satisfies RpcError),
|
||||||
|
signal,
|
||||||
);
|
);
|
||||||
case "tool.evaluate":
|
case "tool.evaluate":
|
||||||
return handleEvaluate(
|
return handleEvaluate(
|
||||||
@@ -394,7 +562,6 @@ export class ToolDispatcher {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||||
notifications: makeHelpNotifications(),
|
notifications: makeHelpNotifications(),
|
||||||
notificationCopy: this.helpNotificationCopy?.(),
|
notificationCopy: this.helpNotificationCopy?.(),
|
||||||
@@ -446,6 +613,122 @@ export class ToolDispatcher {
|
|||||||
} satisfies RpcError;
|
} satisfies RpcError;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
|
||||||
|
const owner = this.hoverBypassTabs.get(tabId);
|
||||||
|
if (enabled) {
|
||||||
|
if (owner === sessionId) return;
|
||||||
|
if (owner === undefined) await bypassOverlay(tabId, true);
|
||||||
|
this.hoverBypassTabs.set(tabId, sessionId);
|
||||||
|
} else {
|
||||||
|
if (owner !== sessionId) return;
|
||||||
|
await bypassOverlay(tabId, false);
|
||||||
|
this.hoverBypassTabs.delete(tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
|
||||||
|
if (!isRpcError(result)) {
|
||||||
|
this.hoverLatches.set(result.tab_id, {
|
||||||
|
sessionId,
|
||||||
|
tabId: result.tab_id,
|
||||||
|
x: result.x,
|
||||||
|
y: result.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||||
|
return this.hoverLatchesForRequest(scope).length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReassert<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
options: { releaseAfter?: boolean } = {},
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<T> {
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
await this.reassertHover(scope);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
if (options.releaseAfter) {
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withHoverReleaseForRequest<T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<T> {
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
|
throwIfDispatchAborted(signal);
|
||||||
|
return work();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveHoverLatchScope(params: {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}): Promise<HoverLatchScope> {
|
||||||
|
if (params.tab_id !== undefined) return params;
|
||||||
|
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||||
|
if (isRpcError(ctx)) return params;
|
||||||
|
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||||
|
if (isRpcError(target)) return params;
|
||||||
|
return { session_id: params.session_id, tab_id: target.tabId };
|
||||||
|
}
|
||||||
|
|
||||||
|
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||||
|
return [...this.hoverLatches.values()].filter((latch) => {
|
||||||
|
if (latch.sessionId !== params.session_id) return false;
|
||||||
|
return params.tab_id === undefined || latch.tabId === params.tab_id;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
|
||||||
|
if (!this.cdp) return;
|
||||||
|
await Promise.all(
|
||||||
|
this.hoverLatchesForRequest(params).map((latch) =>
|
||||||
|
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: latch.x,
|
||||||
|
y: latch.y,
|
||||||
|
}).catch((err) => {
|
||||||
|
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
|
||||||
|
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
|
||||||
|
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
|
||||||
|
return tabId === undefined || entryTabId === tabId;
|
||||||
|
};
|
||||||
|
const tabs = new Set<number>();
|
||||||
|
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
|
||||||
|
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
|
||||||
|
tabs.add(bypassTabId);
|
||||||
|
this.hoverBypassTabs.delete(bypassTabId);
|
||||||
|
}
|
||||||
|
for (const latch of this.hoverLatches.values()) {
|
||||||
|
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
|
||||||
|
tabs.add(latch.tabId);
|
||||||
|
this.hoverLatches.delete(latch.tabId);
|
||||||
|
}
|
||||||
|
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isRpcError(v: unknown): v is RpcError {
|
function isRpcError(v: unknown): v is RpcError {
|
||||||
@@ -458,41 +741,57 @@ function isRpcError(v: unknown): v is RpcError {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
switch (req.method) {
|
||||||
* fires (or immediately if it is already aborted). Used by the
|
case "tool.tab_create":
|
||||||
* dispatcher to race the tool invocation so handlers without explicit
|
case "tool.tab_close":
|
||||||
* signal plumbing still surface a `cancelled` reply promptly.
|
case "tool.tab_select":
|
||||||
*/
|
case "tool.tab_borrow":
|
||||||
function abortPromise(signal: AbortSignal): Promise<never> {
|
case "tool.tab_return":
|
||||||
return new Promise<never>((_, reject) => {
|
case "tool.window_resize":
|
||||||
if (signal.aborted) {
|
case "tool.emulate":
|
||||||
reject(new AbortLikeError());
|
case "tool.navigate":
|
||||||
return;
|
case "tool.navigate_back":
|
||||||
|
case "tool.navigate_forward":
|
||||||
|
case "tool.reload":
|
||||||
|
case "tool.click":
|
||||||
|
case "tool.hover":
|
||||||
|
case "tool.fill":
|
||||||
|
case "tool.press":
|
||||||
|
case "tool.select":
|
||||||
|
case "tool.upload":
|
||||||
|
case "tool.download":
|
||||||
|
case "tool.evaluate":
|
||||||
|
case "tool.observe":
|
||||||
|
case "tool.request_help":
|
||||||
|
case "tool.record_start": {
|
||||||
|
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||||
|
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
|
||||||
}
|
}
|
||||||
signal.addEventListener(
|
default:
|
||||||
"abort",
|
return null;
|
||||||
() => {
|
|
||||||
reject(new AbortLikeError());
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sentinel error class so [`isAbortLikeError`] can recognise our own
|
|
||||||
* race-rejection without confusing it with a real CDP failure.
|
|
||||||
*/
|
|
||||||
class AbortLikeError extends Error {
|
|
||||||
constructor() {
|
|
||||||
super("rpc aborted by daemon cancel");
|
|
||||||
this.name = "BhAbortError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfDispatchAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (!signal?.aborted) return;
|
||||||
|
const error = new Error("rpc aborted by daemon cancel");
|
||||||
|
error.name = "AbortError";
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
function isAbortLikeError(err: unknown): boolean {
|
function isAbortLikeError(err: unknown): boolean {
|
||||||
if (err instanceof AbortLikeError) return true;
|
|
||||||
if (err instanceof DOMException && err.name === "AbortError") return true;
|
if (err instanceof DOMException && err.name === "AbortError") return true;
|
||||||
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -0,0 +1,349 @@
|
|||||||
|
// Order-independent coordinator for one browser download. CDP supplies the
|
||||||
|
// exact target/frame intent while chrome.downloads supplies the download id
|
||||||
|
// and filename routing hook; neither event is assumed to arrive first.
|
||||||
|
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||||
|
import { transferError } from "./errors";
|
||||||
|
import { type CdpRunner, isRpcError } from "./shared";
|
||||||
|
|
||||||
|
const CORRELATION_GRACE_MS = 750;
|
||||||
|
const UNIQUE_SETTLE_MS = 50;
|
||||||
|
const SIZE_POLL_MS = 250;
|
||||||
|
|
||||||
|
type DeterminingFilenameListener = (
|
||||||
|
item: chrome.downloads.DownloadItem,
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||||
|
) => void | true;
|
||||||
|
|
||||||
|
interface ListenerEvent<T> {
|
||||||
|
addListener(listener: T): void;
|
||||||
|
removeListener(listener: T): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadsApi {
|
||||||
|
onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>;
|
||||||
|
onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>;
|
||||||
|
onDeterminingFilename: ListenerEvent<DeterminingFilenameListener>;
|
||||||
|
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
|
||||||
|
cancel(downloadId: number): Promise<void>;
|
||||||
|
removeFile(downloadId: number): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const chromeDownloadsApi: DownloadsApi = {
|
||||||
|
get onCreated() {
|
||||||
|
return chrome.downloads.onCreated;
|
||||||
|
},
|
||||||
|
get onChanged() {
|
||||||
|
return chrome.downloads.onChanged;
|
||||||
|
},
|
||||||
|
get onDeterminingFilename() {
|
||||||
|
return chrome.downloads.onDeterminingFilename;
|
||||||
|
},
|
||||||
|
search: (query) => chrome.downloads.search(query),
|
||||||
|
cancel: (id) => chrome.downloads.cancel(id),
|
||||||
|
removeFile: (id) => chrome.downloads.removeFile(id),
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface DownloadCaptureOptions {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
target: CdpTarget;
|
||||||
|
expectedFrameId?: string;
|
||||||
|
downloads: DownloadsApi;
|
||||||
|
browserRelativeDir: string;
|
||||||
|
maxByteSize?: number;
|
||||||
|
timeoutMs: number;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
trigger(): Promise<ClickResult | RpcError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadCaptureResult {
|
||||||
|
click: ClickResult;
|
||||||
|
item: chrome.downloads.DownloadItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DownloadIntent {
|
||||||
|
url: string;
|
||||||
|
suggestedFilename: string;
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DownloadCandidate {
|
||||||
|
item: chrome.downloads.DownloadItem;
|
||||||
|
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
|
||||||
|
suggested: boolean;
|
||||||
|
graceTimer: ReturnType<typeof setTimeout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeBasename(filename: string): string {
|
||||||
|
const basename = filename.split(/[\\/]/).pop()?.trim();
|
||||||
|
return basename && basename !== "." && basename !== ".." ? basename : "download";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean {
|
||||||
|
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean {
|
||||||
|
const urlMatches = item.url === intent.url || item.finalUrl === intent.url;
|
||||||
|
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
|
||||||
|
}
|
||||||
|
|
||||||
|
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
|
||||||
|
if (item.fileSize >= 0) return item.fileSize;
|
||||||
|
if (item.totalBytes >= 0) return item.totalBytes;
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function captureError(
|
||||||
|
message: string,
|
||||||
|
effectState: TransferEffectState,
|
||||||
|
phase: string,
|
||||||
|
cleanupFailed = false,
|
||||||
|
): RpcError {
|
||||||
|
return transferError("cdp_failed", "download_capture_failed", message, {
|
||||||
|
effectState,
|
||||||
|
phase,
|
||||||
|
...(cleanupFailed ? { cleanupState: "failed" } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureBrowserDownload(
|
||||||
|
options: DownloadCaptureOptions,
|
||||||
|
): Promise<DownloadCaptureResult | RpcError> {
|
||||||
|
let click: ClickResult | undefined;
|
||||||
|
let intent: DownloadIntent | undefined;
|
||||||
|
let capturedId: number | undefined;
|
||||||
|
let capturedItem: chrome.downloads.DownloadItem | undefined;
|
||||||
|
let settled = false;
|
||||||
|
let succeeded = false;
|
||||||
|
let failureResult: RpcError | undefined;
|
||||||
|
let uniquenessTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
let operationTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
let sizePoll: ReturnType<typeof setInterval> | undefined;
|
||||||
|
const candidates = new Map<number, DownloadCandidate>();
|
||||||
|
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
|
||||||
|
|
||||||
|
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
|
||||||
|
let rejectCompletion!: (error: Error) => void;
|
||||||
|
const completion = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
|
||||||
|
resolveCompletion = resolve;
|
||||||
|
rejectCompletion = reject;
|
||||||
|
});
|
||||||
|
const fail = (error: Error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
rejectCompletion(error);
|
||||||
|
};
|
||||||
|
const complete = (item: chrome.downloads.DownloadItem) => {
|
||||||
|
if (settled) return;
|
||||||
|
const size = knownSize(item);
|
||||||
|
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||||
|
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
settled = true;
|
||||||
|
capturedItem = item;
|
||||||
|
resolveCompletion(item);
|
||||||
|
};
|
||||||
|
const suggestDefault = (candidate: DownloadCandidate) => {
|
||||||
|
if (candidate.suggested) return;
|
||||||
|
candidate.suggested = true;
|
||||||
|
candidate.suggest();
|
||||||
|
};
|
||||||
|
const matchingCandidates = (): DownloadCandidate[] => {
|
||||||
|
const currentIntent = intent;
|
||||||
|
return currentIntent
|
||||||
|
? [...candidates.values()].filter(
|
||||||
|
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
};
|
||||||
|
|
||||||
|
const claimUnique = () => {
|
||||||
|
uniquenessTimer = undefined;
|
||||||
|
if (settled || capturedId !== undefined || !intent) return;
|
||||||
|
const matches = matchingCandidates();
|
||||||
|
if (matches.length !== 1) {
|
||||||
|
if (matches.length > 1) {
|
||||||
|
for (const candidate of matches) suggestDefault(candidate);
|
||||||
|
fail(new Error("download attribution is ambiguous"));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const candidate = matches[0];
|
||||||
|
candidate.suggested = true;
|
||||||
|
clearTimeout(candidate.graceTimer);
|
||||||
|
capturedId = candidate.item.id;
|
||||||
|
capturedItem = candidate.item;
|
||||||
|
candidate.suggest({
|
||||||
|
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
|
||||||
|
conflictAction: "overwrite",
|
||||||
|
});
|
||||||
|
const size = knownSize(candidate.item);
|
||||||
|
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||||
|
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const created = createdItems.get(candidate.item.id);
|
||||||
|
if (created?.state === "interrupted") {
|
||||||
|
fail(new Error(created.error ?? "download interrupted"));
|
||||||
|
} else if (created?.state === "complete") {
|
||||||
|
complete(created);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const reconcile = () => {
|
||||||
|
if (settled || capturedId !== undefined || !intent) return;
|
||||||
|
const matches = matchingCandidates();
|
||||||
|
if (matches.length > 1) {
|
||||||
|
for (const candidate of matches) suggestDefault(candidate);
|
||||||
|
fail(new Error("download attribution is ambiguous"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (matches.length === 1 && !uniquenessTimer) {
|
||||||
|
uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const determiningListener: DeterminingFilenameListener = (item, suggest) => {
|
||||||
|
const candidate: DownloadCandidate = {
|
||||||
|
item,
|
||||||
|
suggest,
|
||||||
|
suggested: false,
|
||||||
|
graceTimer: setTimeout(() => {
|
||||||
|
suggestDefault(candidate);
|
||||||
|
candidates.delete(item.id);
|
||||||
|
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
|
||||||
|
fail(new Error("download correlation grace elapsed before unique attribution"));
|
||||||
|
}
|
||||||
|
}, CORRELATION_GRACE_MS),
|
||||||
|
};
|
||||||
|
candidates.set(item.id, candidate);
|
||||||
|
reconcile();
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
const createdListener = (item: chrome.downloads.DownloadItem) => {
|
||||||
|
createdItems.set(item.id, item);
|
||||||
|
if (capturedId !== item.id) return;
|
||||||
|
capturedItem = item;
|
||||||
|
if (item.state === "interrupted") {
|
||||||
|
fail(new Error(item.error ?? "download interrupted"));
|
||||||
|
} else if (item.state === "complete") {
|
||||||
|
complete(item);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const changedListener = async (delta: chrome.downloads.DownloadDelta) => {
|
||||||
|
if (capturedId === undefined || delta.id !== capturedId || settled) return;
|
||||||
|
if (delta.state?.current === "interrupted" || delta.error?.current) {
|
||||||
|
fail(new Error(delta.error?.current ?? "download interrupted"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (delta.state?.current === "complete") {
|
||||||
|
try {
|
||||||
|
const [item] = await options.downloads.search({ id: delta.id });
|
||||||
|
if (item) complete(item);
|
||||||
|
else fail(new Error("completed download disappeared"));
|
||||||
|
} catch (err) {
|
||||||
|
fail(err instanceof Error ? err : new Error(String(err)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
|
||||||
|
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||||
|
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
|
||||||
|
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
|
||||||
|
if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return;
|
||||||
|
if (options.expectedFrameId && event.frameId !== options.expectedFrameId) {
|
||||||
|
fail(new Error("download originated from a different frame"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (intent) {
|
||||||
|
fail(new Error("download trigger produced more than one browser download intent"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
intent = {
|
||||||
|
url: event.url,
|
||||||
|
suggestedFilename: event.suggestedFilename,
|
||||||
|
...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}),
|
||||||
|
};
|
||||||
|
reconcile();
|
||||||
|
});
|
||||||
|
if (!cdpSubscription) {
|
||||||
|
return captureError("CDP download intent subscription unavailable", "none", "arm");
|
||||||
|
}
|
||||||
|
|
||||||
|
options.downloads.onDeterminingFilename.addListener(determiningListener);
|
||||||
|
options.downloads.onCreated.addListener(createdListener);
|
||||||
|
options.downloads.onChanged.addListener(changedListener);
|
||||||
|
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
operationTimer = setTimeout(
|
||||||
|
() => fail(new Error("download did not complete before timeout")),
|
||||||
|
options.timeoutMs,
|
||||||
|
);
|
||||||
|
sizePoll = setInterval(() => {
|
||||||
|
if (capturedId === undefined || settled || options.maxByteSize === undefined) return;
|
||||||
|
void options.downloads
|
||||||
|
.search({ id: capturedId })
|
||||||
|
.then(([item]) => {
|
||||||
|
if (!item || settled) return;
|
||||||
|
capturedItem = item;
|
||||||
|
if (item.bytesReceived > (options.maxByteSize as number)) {
|
||||||
|
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((err) => fail(err instanceof Error ? err : new Error(String(err))));
|
||||||
|
}, SIZE_POLL_MS);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const triggered = await options.trigger();
|
||||||
|
if (isRpcError(triggered)) {
|
||||||
|
void completion.catch(() => undefined);
|
||||||
|
const effect: TransferEffectState =
|
||||||
|
capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
|
||||||
|
failureResult = {
|
||||||
|
...triggered,
|
||||||
|
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
|
||||||
|
};
|
||||||
|
return failureResult;
|
||||||
|
}
|
||||||
|
click = triggered;
|
||||||
|
const item = await completion;
|
||||||
|
succeeded = true;
|
||||||
|
return { click, item };
|
||||||
|
} catch (err) {
|
||||||
|
const effect: TransferEffectState =
|
||||||
|
capturedId !== undefined ? "committed" : click ? "unknown" : "none";
|
||||||
|
failureResult = captureError(
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
effect,
|
||||||
|
capturedId !== undefined ? "download" : "attribution",
|
||||||
|
);
|
||||||
|
return failureResult;
|
||||||
|
} finally {
|
||||||
|
settled = true;
|
||||||
|
if (operationTimer) clearTimeout(operationTimer);
|
||||||
|
if (uniquenessTimer) clearTimeout(uniquenessTimer);
|
||||||
|
if (sizePoll) clearInterval(sizePoll);
|
||||||
|
options.signal?.removeEventListener("abort", onAbort);
|
||||||
|
options.downloads.onDeterminingFilename.removeListener(determiningListener);
|
||||||
|
options.downloads.onCreated.removeListener(createdListener);
|
||||||
|
options.downloads.onChanged.removeListener(changedListener);
|
||||||
|
cdpSubscription.dispose();
|
||||||
|
for (const candidate of candidates.values()) {
|
||||||
|
clearTimeout(candidate.graceTimer);
|
||||||
|
if (candidate.item.id !== capturedId) suggestDefault(candidate);
|
||||||
|
}
|
||||||
|
if (!succeeded && capturedId !== undefined) {
|
||||||
|
const item = capturedItem ?? createdItems.get(capturedId);
|
||||||
|
try {
|
||||||
|
if (item?.state === "complete") {
|
||||||
|
await options.downloads.removeFile(capturedId);
|
||||||
|
} else {
|
||||||
|
await options.downloads.cancel(capturedId);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
if (failureResult?.data) failureResult.data.cleanup_state = "failed";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
// Download orchestration: validate session/tab ownership, then delegate one
|
||||||
|
// browser-global chrome.downloads transaction to download-capture.ts.
|
||||||
|
|
||||||
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
|
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
|
||||||
|
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
|
||||||
|
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||||
|
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
|
||||||
|
|
||||||
|
let downloadActive = false;
|
||||||
|
|
||||||
|
export type { DownloadsApi } from "./download-capture";
|
||||||
|
|
||||||
|
export interface DownloadDeps extends InteractionDeps {
|
||||||
|
downloads?: DownloadsApi;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleDownload(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: DownloadParams,
|
||||||
|
deps: DownloadDeps,
|
||||||
|
): Promise<DownloadResult | RpcError> {
|
||||||
|
if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" };
|
||||||
|
downloadActive = true;
|
||||||
|
try {
|
||||||
|
const ctx = lookupSession(manager, params, "download");
|
||||||
|
if (isRpcError(ctx)) return ctx;
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "download");
|
||||||
|
if (denied) return denied;
|
||||||
|
if (!params.browser_relative_dir) {
|
||||||
|
return { code: "invalid_params", message: "download requires a daemon capability directory" };
|
||||||
|
}
|
||||||
|
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download");
|
||||||
|
if (isRpcError(address)) return address;
|
||||||
|
|
||||||
|
const capture = await captureBrowserDownload({
|
||||||
|
cdp: deps.cdp,
|
||||||
|
target: address.cdpTarget,
|
||||||
|
downloads: deps.downloads ?? chromeDownloadsApi,
|
||||||
|
browserRelativeDir: params.browser_relative_dir,
|
||||||
|
maxByteSize: params.max_byte_size,
|
||||||
|
timeoutMs: params.timeout_ms ?? 120_000,
|
||||||
|
signal: deps.signal,
|
||||||
|
expectedFrameId: address.frameId,
|
||||||
|
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||||
|
});
|
||||||
|
if (isRpcError(capture)) return capture;
|
||||||
|
const { click, item } = capture;
|
||||||
|
return {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: click.used_ref,
|
||||||
|
used_selector: click.used_selector,
|
||||||
|
suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download",
|
||||||
|
byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes,
|
||||||
|
mime: item.mime || undefined,
|
||||||
|
danger: item.danger,
|
||||||
|
browser_path: item.filename,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
downloadActive = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,9 +1,15 @@
|
|||||||
// Shared CDP helpers for scrolling nodes into view and computing
|
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
// belongs to frame-geometry.ts.
|
||||||
// centre) and observation tools (element screenshot clip).
|
|
||||||
|
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
import {
|
||||||
|
parseCdpQuad,
|
||||||
|
polygonBounds,
|
||||||
|
type Region,
|
||||||
|
rectPolygon,
|
||||||
|
type ViewportRect,
|
||||||
|
} from "./geometry";
|
||||||
import { type CdpRunner, isRpcError } from "./shared";
|
import { type CdpRunner, isRpcError } from "./shared";
|
||||||
|
|
||||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||||
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
|
|||||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
|
||||||
export interface ViewportRect {
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
|
||||||
* quad / box-model polygon. Exported for unit tests.
|
|
||||||
*/
|
|
||||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
|
||||||
if (quad.length !== 8) return null;
|
|
||||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
|
||||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
|
||||||
const minX = Math.min(...xs);
|
|
||||||
const maxX = Math.max(...xs);
|
|
||||||
const minY = Math.min(...ys);
|
|
||||||
const maxY = Math.max(...ys);
|
|
||||||
const width = maxX - minX;
|
|
||||||
const height = maxY - minY;
|
|
||||||
if (width <= 0 || height <= 0) return null;
|
|
||||||
return { x: minX, y: minY, width, height };
|
|
||||||
}
|
|
||||||
|
|
||||||
function rectToQuad(rect: ViewportRect): number[] {
|
|
||||||
return [
|
|
||||||
rect.x,
|
|
||||||
rect.y,
|
|
||||||
rect.x + rect.width,
|
|
||||||
rect.y,
|
|
||||||
rect.x + rect.width,
|
|
||||||
rect.y + rect.height,
|
|
||||||
rect.x,
|
|
||||||
rect.y + rect.height,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||||
* `Runtime.callFunctionOn` against the live JS object.
|
* `Runtime.callFunctionOn` against the live JS object.
|
||||||
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
return (quads ?? []).flatMap((raw) => {
|
||||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
const polygon = parseCdpQuad(raw);
|
||||||
* visible descendant client rects for zero-size overflow containers.
|
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||||
*/
|
});
|
||||||
export async function nodeBoundingRect(
|
}
|
||||||
|
|
||||||
|
async function resolveVisibleContentRegion(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
): Promise<ViewportRect | RpcError> {
|
): Promise<Region | RpcError> {
|
||||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
|
||||||
if (isRpcError(polygon)) return polygon;
|
|
||||||
// visibleContentPolygon only returns quads with positive area.
|
|
||||||
return quadBoundingRect(polygon)!;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
|
||||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
|
||||||
*/
|
|
||||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
|
||||||
const x = (quad[0] + quad[4]) / 2;
|
|
||||||
const y = (quad[1] + quad[5]) / 2;
|
|
||||||
return { x, y };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
|
||||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
|
||||||
return quadCentre(box);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function visibleContentPolygon(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<number[] | RpcError> {
|
|
||||||
try {
|
try {
|
||||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
const visible = validCdpRegion(quads.quads);
|
||||||
if (quad) return quad;
|
if (visible.length > 0) return visible;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||||
}
|
}
|
||||||
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
|
|||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const content = box.model?.content;
|
const content = box.model?.content;
|
||||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||||
|
if (fallback.length > 0) return fallback;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||||
}
|
}
|
||||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||||
if (descendantRect) return rectToQuad(descendantRect);
|
if (descendantRect) {
|
||||||
|
return [
|
||||||
|
rectPolygon({
|
||||||
|
x: descendantRect.x,
|
||||||
|
y: descendantRect.y,
|
||||||
|
w: descendantRect.width,
|
||||||
|
h: descendantRect.height,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
}
|
||||||
return elementNotVisibleError();
|
return elementNotVisibleError();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Resolve every visible content region in the node's target-local viewport. */
|
||||||
|
export async function nodeContentRegion(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<Region | RpcError> {
|
||||||
|
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
async function descendantBoundingRect(
|
async function descendantBoundingRect(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
|||||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||||
return { x, y, width, height };
|
return { x, y, width, height };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the click point for `backendNodeId`. Prefers
|
|
||||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
|
||||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
|
||||||
* Returns `permission_denied` with "element not visible" when every
|
|
||||||
* geometry path comes back empty / fails.
|
|
||||||
*/
|
|
||||||
export async function nodeCentre(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<{ x: number; y: number } | RpcError> {
|
|
||||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
|
||||||
if (isRpcError(polygon)) return polygon;
|
|
||||||
return quadCentre(polygon);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,302 @@
|
|||||||
|
// `tool.emulate` — apply or clear CDP Emulation-domain overrides
|
||||||
|
// (viewport metrics, user agent, touch) on a session tab, so agents can
|
||||||
|
// debug mobile page behaviour. Overrides are per-target: new tabs do
|
||||||
|
// not inherit them, and `off` restores the tab's real environment.
|
||||||
|
//
|
||||||
|
// Device presets are resolved CLI-side; this handler only executes the
|
||||||
|
// concrete override parameters it receives. Each request is merged
|
||||||
|
// field by field onto the tab's remembered emulation state and the
|
||||||
|
// merged state is applied as a whole, so a partial request does not
|
||||||
|
// reset fields set by an earlier one.
|
||||||
|
|
||||||
|
import type { DeviceMetricsOverride, UserAgentOverride } from "@/browser-driver/chromium-cdp";
|
||||||
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
|
import type {
|
||||||
|
EmulateOverrides,
|
||||||
|
EmulateParams,
|
||||||
|
EmulateResult,
|
||||||
|
RpcError,
|
||||||
|
UserAgentMetadata,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
type ChromeTabsApi,
|
||||||
|
chromeTabsApi,
|
||||||
|
enforceAgentWindow,
|
||||||
|
isRpcError,
|
||||||
|
lookupSession,
|
||||||
|
resolveTargetTab,
|
||||||
|
} from "./shared";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Emulation-domain surface the handler needs. Backed by `ChromiumCdp`
|
||||||
|
* in production; tests inject a fake. `trackSessionTab` is optional so
|
||||||
|
* test doubles need not implement it.
|
||||||
|
*/
|
||||||
|
export interface EmulateCdpRunner {
|
||||||
|
setDeviceMetricsOverride(tabId: number, metrics: DeviceMetricsOverride): Promise<void>;
|
||||||
|
clearDeviceMetricsOverride(tabId: number): Promise<void>;
|
||||||
|
setUserAgentOverride(tabId: number, override: UserAgentOverride): Promise<void>;
|
||||||
|
setTouchEmulationEnabled(tabId: number, enabled: boolean, maxTouchPoints?: number): Promise<void>;
|
||||||
|
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EmulateDeps {
|
||||||
|
cdp: EmulateCdpRunner;
|
||||||
|
tabsApi: ChromeTabsApi;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultEmulateDeps(): EmulateDeps {
|
||||||
|
return {
|
||||||
|
cdp: new ChromiumCdp(),
|
||||||
|
tabsApi: chromeTabsApi,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Result note explaining the per-target scope of CDP emulation
|
||||||
|
* overrides, echoed on every successful call so agents know to
|
||||||
|
* re-apply emulation after opening a new tab.
|
||||||
|
*/
|
||||||
|
export const EMULATE_SCOPE_NOTE =
|
||||||
|
"emulation overrides are per-tab (CDP target): new tabs do not inherit them — re-run `bsk emulate` on each new tab";
|
||||||
|
|
||||||
|
/** Convert the wire (snake_case) UA metadata to CDP's camelCase shape. */
|
||||||
|
export function toCdpUserAgentMetadata(meta: UserAgentMetadata): Record<string, unknown> {
|
||||||
|
const out: Record<string, unknown> = {};
|
||||||
|
if (meta.brands !== undefined) out.brands = meta.brands;
|
||||||
|
if (meta.full_version !== undefined) out.fullVersion = meta.full_version;
|
||||||
|
if (meta.platform !== undefined) out.platform = meta.platform;
|
||||||
|
if (meta.platform_version !== undefined) out.platformVersion = meta.platform_version;
|
||||||
|
if (meta.architecture !== undefined) out.architecture = meta.architecture;
|
||||||
|
if (meta.model !== undefined) out.model = meta.model;
|
||||||
|
if (meta.mobile !== undefined) out.mobile = meta.mobile;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function invalidParams(message: string): RpcError {
|
||||||
|
return { code: "invalid_params", message };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-tab record of the emulation state applied so far. New requests
|
||||||
|
* are merged onto it field by field (see `mergeEmulateOverrides`) and
|
||||||
|
* the merged state is applied as a whole, so e.g. a later
|
||||||
|
* `--width/--height` does not silently reset the dpr/mobile of an
|
||||||
|
* earlier `--device` preset.
|
||||||
|
*
|
||||||
|
* The record is best-effort: an entry left behind by a closed tab is
|
||||||
|
* simply overwritten by the next emulate call that targets the same tab
|
||||||
|
* id, and the whole map is lost when the extension service worker
|
||||||
|
* reloads — after a reload the next emulate is equivalent to a full
|
||||||
|
* (re)set of just the fields it carries.
|
||||||
|
*/
|
||||||
|
const tabEmulationStates = new Map<number, EmulateOverrides>();
|
||||||
|
|
||||||
|
/** Test hook: drop every per-tab emulation state record. */
|
||||||
|
export function resetEmulateStatesForTests(): void {
|
||||||
|
tabEmulationStates.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge one validated request onto the tab's stored state: fields
|
||||||
|
* present in `overrides` overwrite the stored value, absent fields keep
|
||||||
|
* it. Returns a fresh object; `stored` is not mutated.
|
||||||
|
*/
|
||||||
|
export function mergeEmulateOverrides(
|
||||||
|
stored: EmulateOverrides | undefined,
|
||||||
|
overrides: EmulateOverrides,
|
||||||
|
): EmulateOverrides {
|
||||||
|
const merged: EmulateOverrides = { ...stored };
|
||||||
|
if (overrides.width !== undefined) merged.width = overrides.width;
|
||||||
|
if (overrides.height !== undefined) merged.height = overrides.height;
|
||||||
|
if (overrides.device_scale_factor !== undefined) {
|
||||||
|
merged.device_scale_factor = overrides.device_scale_factor;
|
||||||
|
}
|
||||||
|
if (overrides.mobile !== undefined) merged.mobile = overrides.mobile;
|
||||||
|
if (overrides.user_agent !== undefined) merged.user_agent = overrides.user_agent;
|
||||||
|
if (overrides.accept_language !== undefined) {
|
||||||
|
merged.accept_language = overrides.accept_language;
|
||||||
|
}
|
||||||
|
if (overrides.user_agent_metadata !== undefined) {
|
||||||
|
merged.user_agent_metadata = overrides.user_agent_metadata;
|
||||||
|
}
|
||||||
|
if (overrides.touch !== undefined) merged.touch = overrides.touch;
|
||||||
|
if (overrides.max_touch_points !== undefined) {
|
||||||
|
merged.max_touch_points = overrides.max_touch_points;
|
||||||
|
}
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CDP calls run in sequence without rollback: on a mid-way failure the
|
||||||
|
* overrides already applied stay in effect, so the error says how to
|
||||||
|
* reset them.
|
||||||
|
*/
|
||||||
|
function cdpFailed(err: unknown): RpcError {
|
||||||
|
const cause = err instanceof Error ? err.message : String(err);
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: `${cause} (overrides applied before the failure were not rolled back — use --off to reset)`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cross-field validation for one overrides set. The CLI performs the
|
||||||
|
* same checks; the extension re-validates because the wire format is
|
||||||
|
* public.
|
||||||
|
*/
|
||||||
|
export function validateOverrides(overrides: EmulateOverrides): RpcError | null {
|
||||||
|
const hasWidth = overrides.width !== undefined;
|
||||||
|
const hasHeight = overrides.height !== undefined;
|
||||||
|
if (hasWidth !== hasHeight) {
|
||||||
|
return invalidParams("emulate overrides require width and height together");
|
||||||
|
}
|
||||||
|
if (hasWidth) {
|
||||||
|
if (!Number.isSafeInteger(overrides.width) || (overrides.width as number) <= 0) {
|
||||||
|
return invalidParams("width must be a positive integer");
|
||||||
|
}
|
||||||
|
if (!Number.isSafeInteger(overrides.height) || (overrides.height as number) <= 0) {
|
||||||
|
return invalidParams("height must be a positive integer");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
(overrides.device_scale_factor !== undefined || overrides.mobile !== undefined) &&
|
||||||
|
!hasWidth
|
||||||
|
) {
|
||||||
|
return invalidParams("device_scale_factor and mobile require width and height");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
overrides.device_scale_factor !== undefined &&
|
||||||
|
(!Number.isFinite(overrides.device_scale_factor) || overrides.device_scale_factor <= 0)
|
||||||
|
) {
|
||||||
|
return invalidParams("device_scale_factor must be a positive finite number");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
(overrides.accept_language !== undefined || overrides.user_agent_metadata !== undefined) &&
|
||||||
|
overrides.user_agent === undefined
|
||||||
|
) {
|
||||||
|
return invalidParams("accept_language and user_agent_metadata require user_agent");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
overrides.max_touch_points !== undefined &&
|
||||||
|
(!Number.isSafeInteger(overrides.max_touch_points) || overrides.max_touch_points < 1)
|
||||||
|
) {
|
||||||
|
return invalidParams("max_touch_points must be a positive integer");
|
||||||
|
}
|
||||||
|
const anyOverride =
|
||||||
|
hasWidth ||
|
||||||
|
overrides.user_agent !== undefined ||
|
||||||
|
overrides.touch !== undefined ||
|
||||||
|
overrides.max_touch_points !== undefined;
|
||||||
|
if (!anyOverride) {
|
||||||
|
return invalidParams(
|
||||||
|
"emulate requires at least one override (viewport, user agent, or touch) or off: true",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handler for `tool.emulate` (called by the daemon over WS).
|
||||||
|
*
|
||||||
|
* Merges the requested overrides onto the tab's remembered state and
|
||||||
|
* applies the merged state in a fixed order — viewport metrics, then
|
||||||
|
* UA, then touch — or clears everything (state included) when `off` is
|
||||||
|
* set. Raw CDP failures surface as `cdp_failed` (§4.5).
|
||||||
|
*/
|
||||||
|
export async function handleEmulate(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: EmulateParams,
|
||||||
|
deps: EmulateDeps = defaultEmulateDeps(),
|
||||||
|
): Promise<EmulateResult | RpcError> {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "emulate");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params?.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "emulate");
|
||||||
|
if (denied) return denied;
|
||||||
|
|
||||||
|
if (params.off === true) {
|
||||||
|
if (params.overrides !== undefined) {
|
||||||
|
return invalidParams("emulate off cannot be combined with overrides");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
await deps.cdp.clearDeviceMetricsOverride(target.tabId);
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after clearing device metrics" };
|
||||||
|
}
|
||||||
|
await deps.cdp.setTouchEmulationEnabled(target.tabId, false);
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after clearing touch emulation" };
|
||||||
|
}
|
||||||
|
// An empty userAgent string clears the override (CDP convention).
|
||||||
|
await deps.cdp.setUserAgentOverride(target.tabId, { userAgent: "" });
|
||||||
|
} catch (err) {
|
||||||
|
return cdpFailed(err);
|
||||||
|
}
|
||||||
|
tabEmulationStates.delete(target.tabId);
|
||||||
|
return { tab_id: target.tabId, cleared: true, note: EMULATE_SCOPE_NOTE };
|
||||||
|
}
|
||||||
|
|
||||||
|
const overrides = params.overrides;
|
||||||
|
if (!overrides) {
|
||||||
|
return invalidParams("emulate requires overrides or off: true");
|
||||||
|
}
|
||||||
|
const invalid = validateOverrides(overrides);
|
||||||
|
if (invalid) return invalid;
|
||||||
|
|
||||||
|
// Fields absent from this request keep their previously applied
|
||||||
|
// values; the merged state is what gets applied (and echoed back).
|
||||||
|
const merged = mergeEmulateOverrides(tabEmulationStates.get(target.tabId), overrides);
|
||||||
|
try {
|
||||||
|
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
if (merged.width !== undefined && merged.height !== undefined) {
|
||||||
|
await deps.cdp.setDeviceMetricsOverride(target.tabId, {
|
||||||
|
width: merged.width,
|
||||||
|
height: merged.height,
|
||||||
|
deviceScaleFactor: merged.device_scale_factor ?? 0,
|
||||||
|
mobile: merged.mobile ?? false,
|
||||||
|
});
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after applying device metrics" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (merged.user_agent !== undefined) {
|
||||||
|
await deps.cdp.setUserAgentOverride(target.tabId, {
|
||||||
|
userAgent: merged.user_agent,
|
||||||
|
...(merged.accept_language !== undefined ? { acceptLanguage: merged.accept_language } : {}),
|
||||||
|
...(merged.user_agent_metadata !== undefined
|
||||||
|
? { userAgentMetadata: toCdpUserAgentMetadata(merged.user_agent_metadata) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
if (deps.signal?.aborted) {
|
||||||
|
return { code: "cancelled", message: "emulate aborted after applying user agent" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (merged.touch !== undefined || merged.max_touch_points !== undefined) {
|
||||||
|
await deps.cdp.setTouchEmulationEnabled(
|
||||||
|
target.tabId,
|
||||||
|
merged.touch ?? true,
|
||||||
|
merged.max_touch_points,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
return cdpFailed(err);
|
||||||
|
}
|
||||||
|
// Record the merged state only once it was fully applied.
|
||||||
|
tabEmulationStates.set(target.tabId, merged);
|
||||||
|
return {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
cleared: false,
|
||||||
|
applied: merged,
|
||||||
|
note: EMULATE_SCOPE_NOTE,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -2,10 +2,23 @@
|
|||||||
// rendering. Extension handlers attach reasons here; human-facing copy
|
// rendering. Extension handlers attach reasons here; human-facing copy
|
||||||
// lives in bsk-cli `render_error.rs`.
|
// lives in bsk-cli `render_error.rs`.
|
||||||
|
|
||||||
import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types";
|
import type {
|
||||||
|
ErrorCode,
|
||||||
|
RpcError,
|
||||||
|
RpcErrorData,
|
||||||
|
RpcErrorReason,
|
||||||
|
TransferCleanupState,
|
||||||
|
TransferEffectState,
|
||||||
|
} from "@/transport/types";
|
||||||
|
|
||||||
export type { RpcErrorData, RpcErrorReason };
|
export type { RpcErrorData, RpcErrorReason };
|
||||||
|
|
||||||
|
export interface TransferErrorOptions {
|
||||||
|
effectState: TransferEffectState;
|
||||||
|
phase: string;
|
||||||
|
cleanupState?: TransferCleanupState;
|
||||||
|
}
|
||||||
|
|
||||||
export function rpcError(
|
export function rpcError(
|
||||||
code: ErrorCode,
|
code: ErrorCode,
|
||||||
reason: RpcErrorReason,
|
reason: RpcErrorReason,
|
||||||
@@ -15,3 +28,16 @@ export function rpcError(
|
|||||||
const data: RpcErrorData = { reason, ...extra };
|
const data: RpcErrorData = { reason, ...extra };
|
||||||
return { code, message, data };
|
return { code, message, data };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function transferError(
|
||||||
|
code: ErrorCode,
|
||||||
|
reason: RpcErrorReason,
|
||||||
|
message: string,
|
||||||
|
options: TransferErrorOptions,
|
||||||
|
): RpcError {
|
||||||
|
return rpcError(code, reason, message, {
|
||||||
|
effect_state: options.effectState,
|
||||||
|
phase: options.phase,
|
||||||
|
...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,484 @@
|
|||||||
|
// One upload transaction with an explicit browser-side commit boundary.
|
||||||
|
// Chrome interception prevents a native chooser from escaping automation;
|
||||||
|
// chooser events and a frame-scoped DOM probe are independent input-location
|
||||||
|
// signals, so event delivery is not required for standard file inputs.
|
||||||
|
|
||||||
|
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||||
|
import { transferError } from "./errors";
|
||||||
|
import type { ResolvedActionTarget } from "./interaction";
|
||||||
|
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||||
|
|
||||||
|
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||||
|
const ACTIVATION_GRACE_MS = 1_000;
|
||||||
|
const PROBE_INTERVAL_MS = 20;
|
||||||
|
|
||||||
|
type UploadPhase =
|
||||||
|
| "arm_interception"
|
||||||
|
| "arm_input_probe"
|
||||||
|
| "trigger"
|
||||||
|
| "resolve_input"
|
||||||
|
| "set_files"
|
||||||
|
| "cleanup";
|
||||||
|
|
||||||
|
interface RuntimeReply {
|
||||||
|
result?: { value?: unknown; objectId?: string };
|
||||||
|
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ChooserEvent {
|
||||||
|
frameId?: string;
|
||||||
|
backendNodeId?: number;
|
||||||
|
mode?: "selectSingle" | "selectMultiple";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FileInputTransactionOptions {
|
||||||
|
cdp: CdpRunner;
|
||||||
|
actionTarget: ResolvedActionTarget;
|
||||||
|
files: string[];
|
||||||
|
timeoutMs: number;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
trigger(): Promise<ClickResult | RpcError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FileInputTransactionResult {
|
||||||
|
click: ClickResult;
|
||||||
|
multiple: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
class BoundedWaitError extends Error {
|
||||||
|
constructor(
|
||||||
|
readonly kind: "timeout" | "aborted",
|
||||||
|
message: string,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function remainingMs(deadline: number): number {
|
||||||
|
return Math.max(0, deadline - Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitBounded<T>(
|
||||||
|
promise: Promise<T>,
|
||||||
|
deadline: number,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
timeoutMessage: string,
|
||||||
|
): Promise<T> {
|
||||||
|
const remaining = remainingMs(deadline);
|
||||||
|
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||||
|
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||||
|
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
let onAbort: (() => void) | undefined;
|
||||||
|
const boundary = new Promise<never>((_resolve, reject) => {
|
||||||
|
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||||
|
if (signal) {
|
||||||
|
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return await Promise.race([promise, boundary]);
|
||||||
|
} finally {
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||||
|
if (!reply.exceptionDetails) return null;
|
||||||
|
return new Error(
|
||||||
|
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function transferFailure(
|
||||||
|
code: RpcError["code"],
|
||||||
|
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
|
||||||
|
message: string,
|
||||||
|
effectState: TransferEffectState,
|
||||||
|
phase: UploadPhase,
|
||||||
|
): RpcError {
|
||||||
|
return transferError(code, reason, message, { effectState, phase });
|
||||||
|
}
|
||||||
|
|
||||||
|
function enrichFailure(
|
||||||
|
error: RpcError,
|
||||||
|
effectState: TransferEffectState,
|
||||||
|
phase: UploadPhase,
|
||||||
|
): RpcError {
|
||||||
|
return {
|
||||||
|
...error,
|
||||||
|
data: { ...error.data, effect_state: effectState, phase },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameTarget(
|
||||||
|
source: { tabId?: number; sessionId?: string },
|
||||||
|
target: { tabId: number; sessionId?: string },
|
||||||
|
): boolean {
|
||||||
|
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
await waitBounded(
|
||||||
|
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||||
|
Date.now() + ms + 1,
|
||||||
|
signal,
|
||||||
|
"upload activation probe timed out",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function callOnTrigger<T = RuntimeReply>(
|
||||||
|
options: FileInputTransactionOptions,
|
||||||
|
objectId: string,
|
||||||
|
functionDeclaration: string,
|
||||||
|
args: unknown[],
|
||||||
|
returnByValue: boolean,
|
||||||
|
): Promise<T> {
|
||||||
|
return sendToCdpTarget<T>(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration,
|
||||||
|
arguments: args.map((value) => ({ value })),
|
||||||
|
returnByValue,
|
||||||
|
awaitPromise: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function uploadThroughActivatedFileInput(
|
||||||
|
options: FileInputTransactionOptions,
|
||||||
|
): Promise<FileInputTransactionResult | RpcError> {
|
||||||
|
const deadline = Date.now() + options.timeoutMs;
|
||||||
|
const target = options.actionTarget.cdpTarget;
|
||||||
|
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||||
|
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||||
|
const chooserEvents: ChooserEvent[] = [];
|
||||||
|
let interceptionArmed = false;
|
||||||
|
let probeArmed = false;
|
||||||
|
let triggerObjectId: string | undefined;
|
||||||
|
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||||
|
"protocol_error",
|
||||||
|
"file_input_probe_failed",
|
||||||
|
"upload transaction ended without an outcome",
|
||||||
|
"none",
|
||||||
|
"cleanup",
|
||||||
|
);
|
||||||
|
|
||||||
|
const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||||
|
if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return;
|
||||||
|
const event = raw as ChooserEvent;
|
||||||
|
chooserEvents.push(event);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
try {
|
||||||
|
await waitBounded(
|
||||||
|
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||||
|
enabled: true,
|
||||||
|
}),
|
||||||
|
deadline,
|
||||||
|
options.signal,
|
||||||
|
"arming native file chooser interception timed out",
|
||||||
|
);
|
||||||
|
interceptionArmed = true;
|
||||||
|
} catch (err) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||||
|
"file_input_probe_failed",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
"none",
|
||||||
|
"arm_interception",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resolved = await waitBounded(
|
||||||
|
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||||
|
options.cdp,
|
||||||
|
target,
|
||||||
|
"DOM.resolveNode",
|
||||||
|
{
|
||||||
|
backendNodeId: options.actionTarget.backendNodeId,
|
||||||
|
objectGroup,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
deadline,
|
||||||
|
options.signal,
|
||||||
|
"resolving upload trigger timed out",
|
||||||
|
);
|
||||||
|
triggerObjectId = resolved.object?.objectId;
|
||||||
|
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||||
|
|
||||||
|
const armed = await waitBounded(
|
||||||
|
callOnTrigger<RuntimeReply>(
|
||||||
|
options,
|
||||||
|
triggerObjectId,
|
||||||
|
`function(key) {
|
||||||
|
const doc = this.ownerDocument;
|
||||||
|
const owner = doc.defaultView;
|
||||||
|
if (!owner) return false;
|
||||||
|
const state = { inputs: [], listener: null };
|
||||||
|
Object.defineProperty(owner, key, { value: state, configurable: true });
|
||||||
|
state.listener = event => {
|
||||||
|
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
||||||
|
const candidate = path[0] || event.target;
|
||||||
|
if (candidate && candidate.nodeType === 1 &&
|
||||||
|
candidate.localName === "input" && candidate.type === "file") {
|
||||||
|
if (!state.inputs.includes(candidate)) state.inputs.push(candidate);
|
||||||
|
event.preventDefault();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
doc.addEventListener("click", state.listener, true);
|
||||||
|
return true;
|
||||||
|
}`,
|
||||||
|
[stateKey],
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
deadline,
|
||||||
|
options.signal,
|
||||||
|
"arming frame-scoped file input probe timed out",
|
||||||
|
);
|
||||||
|
const armError = runtimeError(armed, "failed to arm file input probe");
|
||||||
|
if (armError || armed.result?.value !== true) {
|
||||||
|
throw armError ?? new Error("file input probe did not arm");
|
||||||
|
}
|
||||||
|
probeArmed = true;
|
||||||
|
} catch (err) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||||
|
"file_input_probe_failed",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
"none",
|
||||||
|
"arm_input_probe",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
let click: ClickResult | RpcError;
|
||||||
|
try {
|
||||||
|
click = await waitBounded(
|
||||||
|
options.trigger(),
|
||||||
|
deadline,
|
||||||
|
options.signal,
|
||||||
|
"upload trigger timed out",
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||||
|
"file_input_probe_failed",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
"none",
|
||||||
|
"trigger",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
if (isRpcError(click)) {
|
||||||
|
outcome = enrichFailure(click, "none", "trigger");
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS);
|
||||||
|
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||||
|
while (Date.now() < activationDeadline) {
|
||||||
|
if (chooserEvents.length > 0) break;
|
||||||
|
const reply = await callOnTrigger<RuntimeReply>(
|
||||||
|
options,
|
||||||
|
triggerObjectId,
|
||||||
|
`function(key) {
|
||||||
|
const state = this.ownerDocument.defaultView?.[key];
|
||||||
|
return state
|
||||||
|
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||||
|
: { count: 0, multiple: false };
|
||||||
|
}`,
|
||||||
|
[stateKey],
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||||
|
if (summaryError) throw summaryError;
|
||||||
|
const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined;
|
||||||
|
summary = {
|
||||||
|
count: typeof value?.count === "number" ? value.count : 0,
|
||||||
|
multiple: value?.multiple === true,
|
||||||
|
};
|
||||||
|
if (summary.count > 0) break;
|
||||||
|
await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (chooserEvents.length > 1) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
"unsupported",
|
||||||
|
"file_input_not_activated",
|
||||||
|
"upload trigger activated more than one file chooser",
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
const chooser = chooserEvents[0];
|
||||||
|
let backendNodeId: number | undefined;
|
||||||
|
let multiple = summary.multiple;
|
||||||
|
if (chooser) {
|
||||||
|
if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
"unsupported",
|
||||||
|
"file_input_not_activated",
|
||||||
|
"upload trigger activated a file chooser in a different frame",
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
if (typeof chooser.backendNodeId !== "number") {
|
||||||
|
outcome = transferFailure(
|
||||||
|
"unsupported",
|
||||||
|
"file_input_not_activated",
|
||||||
|
"upload trigger invoked a non-input file picker",
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
backendNodeId = chooser.backendNodeId;
|
||||||
|
multiple = chooser.mode === "selectMultiple";
|
||||||
|
} else {
|
||||||
|
if (summary.count !== 1) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
"unsupported",
|
||||||
|
"file_input_not_activated",
|
||||||
|
summary.count === 0
|
||||||
|
? "upload trigger did not activate an input[type=file]"
|
||||||
|
: "upload trigger activated more than one input[type=file]",
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
const input = await callOnTrigger<RuntimeReply>(
|
||||||
|
options,
|
||||||
|
triggerObjectId,
|
||||||
|
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||||
|
[stateKey],
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||||
|
if (inputError) throw inputError;
|
||||||
|
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||||
|
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||||
|
options.cdp,
|
||||||
|
target,
|
||||||
|
"DOM.describeNode",
|
||||||
|
{ objectId: input.result.objectId },
|
||||||
|
);
|
||||||
|
backendNodeId = described.node?.backendNodeId;
|
||||||
|
if (typeof backendNodeId !== "number") {
|
||||||
|
throw new Error("DOM.describeNode returned no file input backendNodeId");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!multiple && options.files.length !== 1) {
|
||||||
|
outcome = enrichFailure(
|
||||||
|
{ code: "invalid_params", message: "file input accepts exactly one file" },
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await waitBounded(
|
||||||
|
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||||
|
files: options.files,
|
||||||
|
backendNodeId,
|
||||||
|
}),
|
||||||
|
deadline,
|
||||||
|
options.signal,
|
||||||
|
"setting file input files timed out",
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||||
|
"set_file_input_failed",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
"unknown",
|
||||||
|
"set_files",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
outcome = { click, multiple };
|
||||||
|
return outcome;
|
||||||
|
} catch (err) {
|
||||||
|
outcome = transferFailure(
|
||||||
|
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
|
||||||
|
"file_input_probe_failed",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
"none",
|
||||||
|
"resolve_input",
|
||||||
|
);
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
chooserSubscription?.dispose();
|
||||||
|
let cleanupFailed = false;
|
||||||
|
if (probeArmed && triggerObjectId) {
|
||||||
|
try {
|
||||||
|
await waitBounded(
|
||||||
|
callOnTrigger(
|
||||||
|
options,
|
||||||
|
triggerObjectId,
|
||||||
|
`function(key) {
|
||||||
|
const owner = this.ownerDocument.defaultView;
|
||||||
|
const state = owner?.[key];
|
||||||
|
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||||
|
if (owner) delete owner[key];
|
||||||
|
}`,
|
||||||
|
[stateKey],
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||||
|
undefined,
|
||||||
|
"cleaning file input probe timed out",
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
cleanupFailed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (interceptionArmed) {
|
||||||
|
try {
|
||||||
|
await waitBounded(
|
||||||
|
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||||
|
enabled: false,
|
||||||
|
cancel: true,
|
||||||
|
}),
|
||||||
|
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||||
|
undefined,
|
||||||
|
"disabling file chooser interception timed out",
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
cleanupFailed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await waitBounded(
|
||||||
|
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||||
|
Date.now() + CLEANUP_TIMEOUT_MS,
|
||||||
|
undefined,
|
||||||
|
"releasing upload object group timed out",
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
cleanupFailed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const effect = isRpcError(outcome)
|
||||||
|
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||||
|
: "committed";
|
||||||
|
if (effect === "unknown" || cleanupFailed) {
|
||||||
|
await options.cdp.detach?.(target.tabId);
|
||||||
|
}
|
||||||
|
if (cleanupFailed && isRpcError(outcome)) {
|
||||||
|
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||||
|
import {
|
||||||
|
clipPolygon,
|
||||||
|
type GeometryProjection,
|
||||||
|
type Point,
|
||||||
|
type Polygon,
|
||||||
|
type ProjectiveEdge,
|
||||||
|
parseCdpQuad,
|
||||||
|
polygonArea,
|
||||||
|
polygonCentroid,
|
||||||
|
projectRegionToViewport,
|
||||||
|
type Quad,
|
||||||
|
type Region,
|
||||||
|
rectPolygon,
|
||||||
|
regionBounds,
|
||||||
|
type Size,
|
||||||
|
type ViewportRect,
|
||||||
|
} from "./geometry";
|
||||||
|
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||||
|
|
||||||
|
export interface NodeAddress {
|
||||||
|
target: CdpTarget;
|
||||||
|
backendNodeId: number;
|
||||||
|
frameId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedNodeGeometry {
|
||||||
|
topVisibleRegions: Region;
|
||||||
|
topBounds: ViewportRect;
|
||||||
|
actionPoint: Point;
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryError(message: string): RpcError {
|
||||||
|
return { code: "cdp_failed", message };
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||||
|
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||||
|
}
|
||||||
|
|
||||||
|
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = frame;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const path: CdpFrame[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let current = byId.get(frameId);
|
||||||
|
if (!current) return null;
|
||||||
|
while (current.parentFrameId) {
|
||||||
|
if (seen.has(current.frameId)) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
path.push(current);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent) return null;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||||
|
const metrics = await sendToCdpTarget<{
|
||||||
|
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||||
|
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
const width = viewport.clientWidth ?? 0;
|
||||||
|
const height = viewport.clientHeight ?? 0;
|
||||||
|
return width > 0 && height > 0 ? { width, height } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ownerContentQuad(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
parent: CdpFrame,
|
||||||
|
ownerBackendNodeId: number,
|
||||||
|
): Promise<Quad | null> {
|
||||||
|
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||||
|
cdp,
|
||||||
|
parent.target,
|
||||||
|
"DOM.getBoxModel",
|
||||||
|
{ backendNodeId: ownerBackendNodeId },
|
||||||
|
);
|
||||||
|
return parseCdpQuad(result.model?.content);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sameTargetFrameClips(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
byId: Map<string, CdpFrame>,
|
||||||
|
frame: CdpFrame,
|
||||||
|
targetRoot: CdpFrame,
|
||||||
|
): Promise<Polygon[] | null> {
|
||||||
|
const clips: Polygon[] = [];
|
||||||
|
let current = frame;
|
||||||
|
const seen = new Set<string>();
|
||||||
|
while (current.frameId !== targetRoot.frameId) {
|
||||||
|
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||||
|
seen.add(current.frameId);
|
||||||
|
const parent = byId.get(current.parentFrameId);
|
||||||
|
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||||
|
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||||
|
if (!clip) return null;
|
||||||
|
clips.push(clip);
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
return clips;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveFrameProjection(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
graph: CdpFrameGraph,
|
||||||
|
frameId: string,
|
||||||
|
): Promise<GeometryProjection | null> {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const frame = byId.get(frameId);
|
||||||
|
if (!frame) return null;
|
||||||
|
let targetRoot = targetRootFrame(byId, frame);
|
||||||
|
if (!targetRoot) return null;
|
||||||
|
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||||
|
if (!sourceViewport) return null;
|
||||||
|
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||||
|
if (!sourceClips) return null;
|
||||||
|
|
||||||
|
const edges: ProjectiveEdge[] = [];
|
||||||
|
while (targetRoot.parentFrameId) {
|
||||||
|
const parent = byId.get(targetRoot.parentFrameId);
|
||||||
|
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||||
|
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||||
|
if (!destinationQuad) return null;
|
||||||
|
const source =
|
||||||
|
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||||
|
if (!source) return null;
|
||||||
|
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||||
|
if (!parentTargetRoot) return null;
|
||||||
|
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||||
|
if (!destinationClips) return null;
|
||||||
|
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||||
|
targetRoot = parentTargetRoot;
|
||||||
|
}
|
||||||
|
|
||||||
|
const topViewport =
|
||||||
|
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||||
|
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||||
|
if (!cdp.getFrameGraph) return null;
|
||||||
|
try {
|
||||||
|
return await cdp.getFrameGraph(tabId);
|
||||||
|
} catch (error) {
|
||||||
|
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrollFrameOwners(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
graph: CdpFrameGraph,
|
||||||
|
frameId: string,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const byId = frameMap(graph);
|
||||||
|
const ancestry = frameAncestry(graph, frameId);
|
||||||
|
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||||
|
for (const child of [...ancestry].reverse()) {
|
||||||
|
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||||
|
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||||
|
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||||
|
}
|
||||||
|
const error = await scrollNodeIntoView(
|
||||||
|
cdpRunnerForTarget(cdp, parent.target),
|
||||||
|
tabId,
|
||||||
|
child.ownerBackendNodeId,
|
||||||
|
);
|
||||||
|
if (error) return error;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrollElementWithFrameGraph(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
frameId: string | undefined,
|
||||||
|
graph: CdpFrameGraph | null,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
if (frameId) {
|
||||||
|
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||||
|
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||||
|
if (error) return error;
|
||||||
|
} else if (target.sessionId) {
|
||||||
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
|
}
|
||||||
|
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function scrollElementAndFramesIntoView(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
target: CdpTarget,
|
||||||
|
backendNodeId: number,
|
||||||
|
frameId?: string,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||||
|
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||||
|
}
|
||||||
|
|
||||||
|
function largestRegion(regions: Region): Polygon | null {
|
||||||
|
let largest: { polygon: Polygon; area: number } | null = null;
|
||||||
|
for (const polygon of regions) {
|
||||||
|
const area = polygonArea(polygon);
|
||||||
|
if (area <= 0) continue;
|
||||||
|
if (!largest || area > largest.area) largest = { polygon, area };
|
||||||
|
}
|
||||||
|
return largest?.polygon ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveNodeGeometry(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
address: NodeAddress,
|
||||||
|
options: { scrollIntoView?: boolean } = {},
|
||||||
|
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||||
|
try {
|
||||||
|
if (address.target.sessionId && !address.frameId) {
|
||||||
|
return geometryError("an OOPIF node address requires frameId");
|
||||||
|
}
|
||||||
|
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||||
|
if (address.frameId && !graph) {
|
||||||
|
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.scrollIntoView) {
|
||||||
|
const scrollError = await scrollElementWithFrameGraph(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
address.target,
|
||||||
|
address.backendNodeId,
|
||||||
|
address.frameId,
|
||||||
|
graph,
|
||||||
|
);
|
||||||
|
if (scrollError) return scrollError;
|
||||||
|
}
|
||||||
|
|
||||||
|
const localRegion = await nodeContentRegion(
|
||||||
|
cdpRunnerForTarget(cdp, address.target),
|
||||||
|
tabId,
|
||||||
|
address.backendNodeId,
|
||||||
|
);
|
||||||
|
if (isRpcError(localRegion)) return localRegion;
|
||||||
|
|
||||||
|
let topVisibleRegions: Region;
|
||||||
|
if (address.frameId && graph) {
|
||||||
|
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||||
|
if (!projection)
|
||||||
|
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||||
|
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||||
|
} else {
|
||||||
|
const viewport = await targetViewport(cdp, address.target);
|
||||||
|
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||||
|
topVisibleRegions = localRegion
|
||||||
|
.map((polygon) =>
|
||||||
|
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
const topBounds = regionBounds(topVisibleRegions);
|
||||||
|
const actionRegion = largestRegion(topVisibleRegions);
|
||||||
|
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||||
|
if (!topBounds || !actionPoint) {
|
||||||
|
return { code: "permission_denied", message: "element not visible" };
|
||||||
|
}
|
||||||
|
return { topVisibleRegions, topBounds, actionPoint };
|
||||||
|
} catch (error) {
|
||||||
|
return geometryError(error instanceof Error ? error.message : String(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
export interface Point {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Polygon = Point[];
|
||||||
|
export type Region = Polygon[];
|
||||||
|
export type Quad = [Point, Point, Point, Point];
|
||||||
|
|
||||||
|
export interface Size {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ViewportRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProjectiveEdge {
|
||||||
|
sourceViewport: Size;
|
||||||
|
destinationQuad: Quad;
|
||||||
|
destinationClips?: Polygon[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GeometryProjection {
|
||||||
|
sourceClips: Polygon[];
|
||||||
|
edges: ProjectiveEdge[];
|
||||||
|
topViewport: Size;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||||
|
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||||
|
return [
|
||||||
|
{ x: raw[0], y: raw[1] },
|
||||||
|
{ x: raw[2], y: raw[3] },
|
||||||
|
{ x: raw[4], y: raw[5] },
|
||||||
|
{ x: raw[6], y: raw[7] },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||||
|
return [
|
||||||
|
{ x: rect.x, y: rect.y },
|
||||||
|
{ x: rect.x + rect.w, y: rect.y },
|
||||||
|
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||||
|
{ x: rect.x, y: rect.y + rect.h },
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function viewportPolygon(viewport: Size): Quad {
|
||||||
|
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||||
|
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||||
|
const [p0, p1, p2, p3] = quad;
|
||||||
|
const dx1 = p1.x - p2.x;
|
||||||
|
const dx2 = p3.x - p2.x;
|
||||||
|
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||||
|
const dy1 = p1.y - p2.y;
|
||||||
|
const dy2 = p3.y - p2.y;
|
||||||
|
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||||
|
|
||||||
|
let a: number;
|
||||||
|
let b: number;
|
||||||
|
let c: number;
|
||||||
|
let d: number;
|
||||||
|
let e: number;
|
||||||
|
let f: number;
|
||||||
|
let g = 0;
|
||||||
|
let h = 0;
|
||||||
|
|
||||||
|
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||||
|
a = p1.x - p0.x;
|
||||||
|
b = p3.x - p0.x;
|
||||||
|
c = p0.x;
|
||||||
|
d = p1.y - p0.y;
|
||||||
|
e = p3.y - p0.y;
|
||||||
|
f = p0.y;
|
||||||
|
} else {
|
||||||
|
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||||
|
if (Math.abs(denominator) < 1e-9) {
|
||||||
|
return {
|
||||||
|
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||||
|
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||||
|
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||||
|
a = p1.x - p0.x + g * p1.x;
|
||||||
|
b = p3.x - p0.x + h * p3.x;
|
||||||
|
c = p0.x;
|
||||||
|
d = p1.y - p0.y + g * p1.y;
|
||||||
|
e = p3.y - p0.y + h * p3.y;
|
||||||
|
f = p0.y;
|
||||||
|
}
|
||||||
|
|
||||||
|
const scale = g * point.x + h * point.y + 1;
|
||||||
|
return {
|
||||||
|
x: (a * point.x + b * point.y + c) / scale,
|
||||||
|
y: (d * point.x + e * point.y + f) / scale,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||||
|
const { width, height } = edge.sourceViewport;
|
||||||
|
if (width <= 0 || height <= 0) return [];
|
||||||
|
return polygon.map((point) =>
|
||||||
|
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function polygonSignedArea(points: Polygon): number {
|
||||||
|
let area = 0;
|
||||||
|
for (let index = 0; index < points.length; index += 1) {
|
||||||
|
const current = points[index];
|
||||||
|
const next = points[(index + 1) % points.length];
|
||||||
|
area += current.x * next.y - next.x * current.y;
|
||||||
|
}
|
||||||
|
return area / 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||||
|
const subjectX = end.x - start.x;
|
||||||
|
const subjectY = end.y - start.y;
|
||||||
|
const clipX = clipEnd.x - clipStart.x;
|
||||||
|
const clipY = clipEnd.y - clipStart.y;
|
||||||
|
const denominator = subjectX * clipY - subjectY * clipX;
|
||||||
|
if (Math.abs(denominator) < 1e-9) return end;
|
||||||
|
const offsetX = clipStart.x - start.x;
|
||||||
|
const offsetY = clipStart.y - start.y;
|
||||||
|
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||||
|
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||||
|
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||||
|
if (subject.length < 3 || clip.length < 3) return [];
|
||||||
|
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||||
|
let output = [...subject];
|
||||||
|
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||||
|
const clipStart = clip[edge];
|
||||||
|
const clipEnd = clip[(edge + 1) % clip.length];
|
||||||
|
const input = output;
|
||||||
|
output = [];
|
||||||
|
const inside = (point: Point) =>
|
||||||
|
orientation *
|
||||||
|
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||||
|
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||||
|
-1e-9;
|
||||||
|
let start = input[input.length - 1];
|
||||||
|
for (const end of input) {
|
||||||
|
const startInside = inside(start);
|
||||||
|
const endInside = inside(end);
|
||||||
|
if (endInside) {
|
||||||
|
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||||
|
output.push(end);
|
||||||
|
} else if (startInside) {
|
||||||
|
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||||
|
}
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||||
|
if (points.length < 3) return null;
|
||||||
|
const xs = points.map((point) => point.x);
|
||||||
|
const ys = points.map((point) => point.y);
|
||||||
|
const x = Math.min(...xs);
|
||||||
|
const y = Math.min(...ys);
|
||||||
|
const right = Math.max(...xs);
|
||||||
|
const bottom = Math.max(...ys);
|
||||||
|
if (right <= x || bottom <= y) return null;
|
||||||
|
return { x, y, width: right - x, height: bottom - y };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function regionBounds(region: Region): ViewportRect | null {
|
||||||
|
return polygonBounds(region.flat());
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonArea(points: Polygon): number {
|
||||||
|
return Math.abs(polygonSignedArea(points));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function polygonCentroid(points: Polygon): Point | null {
|
||||||
|
if (points.length < 3) return null;
|
||||||
|
let twiceArea = 0;
|
||||||
|
let x = 0;
|
||||||
|
let y = 0;
|
||||||
|
for (let index = 0; index < points.length; index += 1) {
|
||||||
|
const current = points[index];
|
||||||
|
const next = points[(index + 1) % points.length];
|
||||||
|
const cross = current.x * next.y - next.x * current.y;
|
||||||
|
twiceArea += cross;
|
||||||
|
x += (current.x + next.x) * cross;
|
||||||
|
y += (current.y + next.y) * cross;
|
||||||
|
}
|
||||||
|
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||||
|
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectAndClipRegion(
|
||||||
|
region: Region,
|
||||||
|
edges: ProjectiveEdge[],
|
||||||
|
topViewport: Size,
|
||||||
|
): Region {
|
||||||
|
let projected = region;
|
||||||
|
for (const edge of edges) {
|
||||||
|
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||||
|
projected = projected
|
||||||
|
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||||
|
.filter((polygon) => polygon.length >= 3)
|
||||||
|
.map((polygon) =>
|
||||||
|
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||||
|
(current, clip) => clipPolygon(current, clip),
|
||||||
|
projectPolygon(polygon, edge),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
const topClip = viewportPolygon(topViewport);
|
||||||
|
return projected
|
||||||
|
.map((polygon) => clipPolygon(polygon, topClip))
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||||
|
const clipped = region
|
||||||
|
.map((polygon) =>
|
||||||
|
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||||
|
)
|
||||||
|
.filter((polygon) => polygon.length >= 3);
|
||||||
|
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function projectRectToViewport(
|
||||||
|
rect: { x: number; y: number; w: number; h: number } | null,
|
||||||
|
projection: GeometryProjection,
|
||||||
|
): ViewportRect | null {
|
||||||
|
if (!rect) return null;
|
||||||
|
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function childFrameProjection(
|
||||||
|
parent: GeometryProjection,
|
||||||
|
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||||
|
): GeometryProjection {
|
||||||
|
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||||
|
return {
|
||||||
|
sourceClips: [],
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||||
|
destinationQuad,
|
||||||
|
destinationClips: parent.sourceClips,
|
||||||
|
},
|
||||||
|
...parent.edges,
|
||||||
|
],
|
||||||
|
topViewport: parent.topViewport,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,27 +1,32 @@
|
|||||||
// `tool.request_help` — pause and ask the human to act on a tab.
|
// `tool.request_help` — pause automation and let the human control a tab.
|
||||||
//
|
|
||||||
// Brings the target tab to the foreground, resolves `ref` targets to a
|
|
||||||
// temporary attribute selector via CDP, asks the content script to enter
|
|
||||||
// help mode (HelpRequestOverlay + hidden ControlOverlay), and blocks
|
|
||||||
// until the user clicks Continue / Cancel, the wait times out, or the
|
|
||||||
// daemon cancels the RPC.
|
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
HELP_ACK,
|
||||||
HELP_CANCEL,
|
HELP_CANCEL,
|
||||||
|
HELP_FINISH,
|
||||||
|
HELP_QUERY,
|
||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
HELP_RESPONSE,
|
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
|
type HelpFinishMessage,
|
||||||
|
type HelpHighlightRect,
|
||||||
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
type HelpResponseMessage,
|
isHelpAckMessage,
|
||||||
|
isHelpFinishMessage,
|
||||||
|
isHelpQueryMessage,
|
||||||
|
isHelpResponseMessage,
|
||||||
} from "@/lib/help-bridge";
|
} from "@/lib/help-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
|
HelpCompletionCondition,
|
||||||
|
HelpCompletionCriteria,
|
||||||
HelpTarget,
|
HelpTarget,
|
||||||
RequestHelpParams,
|
RequestHelpParams,
|
||||||
RequestHelpResult,
|
RequestHelpResult,
|
||||||
ResolvedTarget,
|
ResolvedTarget,
|
||||||
RpcError,
|
RpcError,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import { resolveNodeGeometry } from "./frame-geometry";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
@@ -34,53 +39,59 @@ import {
|
|||||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||||
|
|
||||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||||
/** Attribute the overlay highlights for resolved `ref` targets. */
|
const HELP_SEND_RETRIES = 3;
|
||||||
const HELP_ATTR = "data-bsk-help";
|
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||||
|
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||||
|
const HELP_REARM_MAX_ATTEMPTS = 12;
|
||||||
|
const HELP_REARM_RETRY_DELAY_MS = 400;
|
||||||
|
const DEFAULT_COMPLETION_STABLE_MS = 1_000;
|
||||||
|
const COMPLETION_POLL_MS = 500;
|
||||||
|
|
||||||
export interface RequestHelpNotifications {
|
export interface RequestHelpNotifications {
|
||||||
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
create(id: string, options: chrome.notifications.NotificationOptions<true>): Promise<string>;
|
||||||
clear(id: string): Promise<boolean>;
|
clear(id: string): Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TabNavigationUnsubscribe = () => void;
|
|
||||||
|
|
||||||
export interface RequestHelpDeps {
|
export interface RequestHelpDeps {
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
windows: {
|
windows: {
|
||||||
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
update(windowId: number, info: { focused?: boolean }): Promise<chrome.windows.Window>;
|
||||||
};
|
};
|
||||||
/** Activate a tab inside its window. */
|
|
||||||
activateTab(tabId: number): Promise<void>;
|
activateTab(tabId: number): Promise<void>;
|
||||||
/** Send the help-request to the tab's content script and await reply. */
|
|
||||||
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
sendToTab(tabId: number, msg: HelpRequestMessage | HelpCancelMessage): Promise<unknown>;
|
||||||
/**
|
|
||||||
* Watch for navigation on `tabId` (full page load or SPA URL change).
|
|
||||||
* Invokes `onNavigated` once; returns unsubscribe.
|
|
||||||
*/
|
|
||||||
watchTabNavigation: (tabId: number, onNavigated: () => void) => TabNavigationUnsubscribe;
|
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
/** Pass `null` to skip OS notifications (tests). */
|
|
||||||
notifications: RequestHelpNotifications | null;
|
notifications: RequestHelpNotifications | null;
|
||||||
notificationCopy?: { title: string; body: string };
|
notificationCopy?: { title: string; body: string };
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
autoAttachLifecycle?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Default navigation watcher: full reload (`loading`) and SPA (`url`). */
|
interface ActiveHelpRequest {
|
||||||
export function defaultWatchTabNavigation(
|
ctx: SessionContext;
|
||||||
tabId: number,
|
requestId: string;
|
||||||
onNavigated: () => void,
|
primaryTabId: number;
|
||||||
): TabNavigationUnsubscribe {
|
overlayTabIds: Set<number>;
|
||||||
const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => {
|
prompt: string;
|
||||||
if (updatedTabId !== tabId) return;
|
title?: string;
|
||||||
if (changeInfo.url !== undefined || changeInfo.status === "loading") {
|
targets: HelpTarget[];
|
||||||
chrome.tabs.onUpdated.removeListener(listener);
|
selectors: string[];
|
||||||
onNavigated();
|
rects: HelpHighlightRect[];
|
||||||
}
|
timeoutMs: number;
|
||||||
};
|
notificationId: string;
|
||||||
chrome.tabs.onUpdated.addListener(listener);
|
resolvedTargets?: ResolvedTarget[];
|
||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
completionCriteria?: HelpCompletionCriteria;
|
||||||
|
deps: RequestHelpDeps;
|
||||||
|
settled: boolean;
|
||||||
|
resolve: (value: RequestHelpResult | RpcError) => void;
|
||||||
|
timeoutTimer: ReturnType<typeof setTimeout>;
|
||||||
|
completionTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
completionMatchedSince: number | null;
|
||||||
|
abortHandler: (() => void) | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activeHelpRequests = new Map<string, ActiveHelpRequest>();
|
||||||
|
const helpRearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||||
|
|
||||||
let defaultDeps: RequestHelpDeps | null = null;
|
let defaultDeps: RequestHelpDeps | null = null;
|
||||||
function getDefaultDeps(): RequestHelpDeps {
|
function getDefaultDeps(): RequestHelpDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
@@ -91,7 +102,6 @@ function getDefaultDeps(): RequestHelpDeps {
|
|||||||
await chrome.tabs.update(tabId, { active: true });
|
await chrome.tabs.update(tabId, { active: true });
|
||||||
},
|
},
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
watchTabNavigation: defaultWatchTabNavigation,
|
|
||||||
notifications: {
|
notifications: {
|
||||||
create: (id, opts) =>
|
create: (id, opts) =>
|
||||||
new Promise((resolve, reject) =>
|
new Promise((resolve, reject) =>
|
||||||
@@ -113,39 +123,51 @@ function makeRequestId(tabId: number): string {
|
|||||||
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
return `${tabId}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
function sleep(ms: number): Promise<void> {
|
||||||
* Tag a `ref` target's DOM node with `data-bsk-help="<i>"` via CDP so the
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
* content overlay can highlight it with a plain attribute selector.
|
}
|
||||||
* Returns the selector on success, or null when the ref can't resolve.
|
|
||||||
*/
|
function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequestMessage {
|
||||||
async function tagRefTarget(
|
const isPrimaryTab = tabId === help.primaryTabId;
|
||||||
cdp: CdpRunner,
|
return {
|
||||||
|
type: HELP_REQUEST,
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
displayMode: isPrimaryTab ? "full" : "compact",
|
||||||
|
selectors: isPrimaryTab ? help.selectors : [],
|
||||||
|
rects: isPrimaryTab ? help.rects : [],
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendHelpRequestWithAck(
|
||||||
tabId: number,
|
tabId: number,
|
||||||
backendNodeId: number,
|
msg: HelpRequestMessage,
|
||||||
index: number,
|
deps: RequestHelpDeps,
|
||||||
): Promise<string | null> {
|
): Promise<HelpFinishMessage | null> {
|
||||||
let objectId: string | undefined;
|
let lastError: unknown;
|
||||||
try {
|
for (let attempt = 0; attempt < HELP_SEND_RETRIES; attempt += 1) {
|
||||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
try {
|
||||||
backendNodeId,
|
const response = await deps.sendToTab(tabId, msg);
|
||||||
});
|
if (isHelpAckMessage(response)) return null;
|
||||||
objectId = resolved.object?.objectId;
|
if (isHelpResponseMessage(response)) {
|
||||||
if (!objectId) return null;
|
return {
|
||||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
type: HELP_FINISH,
|
||||||
objectId,
|
requestId: msg.requestId,
|
||||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
outcome: response.outcome,
|
||||||
});
|
...(response.note ? { note: response.note } : {}),
|
||||||
return `[${HELP_ATTR}="${index}"]`;
|
};
|
||||||
} catch {
|
}
|
||||||
return null;
|
lastError = new Error("content script did not ack HELP_REQUEST");
|
||||||
} finally {
|
} catch (err) {
|
||||||
if (objectId) {
|
lastError = err;
|
||||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
|
||||||
}
|
}
|
||||||
|
if (attempt + 1 < HELP_SEND_RETRIES) await sleep(HELP_SEND_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
|
throw lastError ?? new Error("failed to show help overlay");
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns true when `selector` matches a live element in the page. */
|
|
||||||
async function selectorExists(
|
async function selectorExists(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -163,25 +185,471 @@ async function selectorExists(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove every help attribute we added (best-effort cleanup). */
|
async function resolveHelpTargets(
|
||||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
ctx: ReturnType<SessionManager["get"]>,
|
||||||
if (!cdp) return;
|
tabId: number,
|
||||||
|
targets: HelpTarget[],
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
options: { scrollIntoView: boolean },
|
||||||
|
): Promise<{
|
||||||
|
selectors: string[];
|
||||||
|
rects: HelpHighlightRect[];
|
||||||
|
resolvedTargets?: ResolvedTarget[];
|
||||||
|
}> {
|
||||||
|
const selectors: string[] = [];
|
||||||
|
const rects: HelpHighlightRect[] = [];
|
||||||
|
const resolved: ResolvedTarget[] = [];
|
||||||
|
let rootNodeId: number | undefined;
|
||||||
|
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||||
|
try {
|
||||||
|
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||||
|
depth: 0,
|
||||||
|
});
|
||||||
|
rootNodeId = doc.root?.nodeId;
|
||||||
|
} catch {
|
||||||
|
rootNodeId = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < targets.length; i += 1) {
|
||||||
|
const tgt = targets[i];
|
||||||
|
if (tgt.selector) {
|
||||||
|
selectors.push(tgt.selector);
|
||||||
|
const matched =
|
||||||
|
deps.cdp && rootNodeId !== undefined
|
||||||
|
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
||||||
|
: deps.cdp
|
||||||
|
? false
|
||||||
|
: true;
|
||||||
|
resolved.push({ matched, selector: tgt.selector });
|
||||||
|
} else if (tgt.ref) {
|
||||||
|
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||||
|
let rect: HelpHighlightRect | null = null;
|
||||||
|
if (looked && deps.cdp) {
|
||||||
|
const target = {
|
||||||
|
tabId,
|
||||||
|
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||||
|
};
|
||||||
|
const geometry = await resolveNodeGeometry(
|
||||||
|
deps.cdp,
|
||||||
|
tabId,
|
||||||
|
{
|
||||||
|
target,
|
||||||
|
backendNodeId: looked.backendNodeId,
|
||||||
|
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||||
|
},
|
||||||
|
{ scrollIntoView: options.scrollIntoView },
|
||||||
|
);
|
||||||
|
if (!isRpcError(geometry)) {
|
||||||
|
rect = {
|
||||||
|
top: geometry.topBounds.y,
|
||||||
|
left: geometry.topBounds.x,
|
||||||
|
width: geometry.topBounds.width,
|
||||||
|
height: geometry.topBounds.height,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (rect) rects.push(rect);
|
||||||
|
resolved.push({
|
||||||
|
matched: rect !== null,
|
||||||
|
ref: tgt.ref,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshHelpTargets(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||||
|
): Promise<void> {
|
||||||
|
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||||
|
help.ctx,
|
||||||
|
help.primaryTabId,
|
||||||
|
help.targets,
|
||||||
|
help.deps,
|
||||||
|
options,
|
||||||
|
);
|
||||||
|
help.selectors = selectors;
|
||||||
|
help.rects = rects;
|
||||||
|
help.resolvedTargets = resolvedTargets;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||||
|
if (help.deps.notifications) {
|
||||||
|
await help.deps.notifications.clear(help.notificationId).catch(() => {});
|
||||||
|
}
|
||||||
|
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||||
|
await Promise.all(
|
||||||
|
[...tabsToCancel].map((tabId) =>
|
||||||
|
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishHelp(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
value: RequestHelpResult | RpcError,
|
||||||
|
notifyContent = true,
|
||||||
|
waitForCleanup = false,
|
||||||
|
): Promise<void> {
|
||||||
|
if (help.settled) return;
|
||||||
|
help.settled = true;
|
||||||
|
activeHelpRequests.delete(help.requestId);
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
if (help.abortHandler) help.deps.signal?.removeEventListener("abort", help.abortHandler);
|
||||||
|
clearRearmTimer(help.primaryTabId);
|
||||||
|
for (const tabId of help.overlayTabIds) clearRearmTimer(tabId);
|
||||||
|
if (notifyContent) {
|
||||||
|
const cleanup = cleanupHelp(help);
|
||||||
|
if (waitForCleanup) await cleanup;
|
||||||
|
else void cleanup;
|
||||||
|
}
|
||||||
|
help.resolve(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findHelpByKnownTabId(tabId: number): ActiveHelpRequest | null {
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
if (!help.settled && (help.primaryTabId === tabId || help.overlayTabIds.has(tabId))) {
|
||||||
|
return help;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findHelpForTab(
|
||||||
|
tabId: number,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<ActiveHelpRequest | null> {
|
||||||
|
const direct = findHelpByKnownTabId(tabId);
|
||||||
|
if (direct) return direct;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
const tab = await deps.tabsApi.get(tabId);
|
||||||
depth: 0,
|
const windowId = tab.windowId;
|
||||||
});
|
if (typeof windowId !== "number") return null;
|
||||||
const rootId = doc.root?.nodeId;
|
for (const help of activeHelpRequests.values()) {
|
||||||
if (rootId === undefined) return;
|
if (!help.settled && help.ctx.agentWindowId === windowId) return help;
|
||||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
|
||||||
nodeId: rootId,
|
|
||||||
selector: `[${HELP_ATTR}]`,
|
|
||||||
});
|
|
||||||
for (const nodeId of nodeIds ?? []) {
|
|
||||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort
|
return null;
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearRearmTimer(tabId: number): void {
|
||||||
|
const timer = helpRearmTimers.get(tabId);
|
||||||
|
if (!timer) return;
|
||||||
|
clearTimeout(timer);
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendCurrentHelpOverlay(help: ActiveHelpRequest): Promise<HelpFinishMessage | null> {
|
||||||
|
const legacyFinish = await sendHelpRequestWithAck(
|
||||||
|
help.primaryTabId,
|
||||||
|
helpRequestMessage(help, help.primaryTabId),
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.overlayTabIds.add(help.primaryTabId);
|
||||||
|
return legacyFinish;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshAndSendHelpOverlay(
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<HelpFinishMessage | null> {
|
||||||
|
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||||
|
const legacyFinish = await sendHelpRequestWithAck(
|
||||||
|
tabId,
|
||||||
|
helpRequestMessage(help, tabId),
|
||||||
|
help.deps,
|
||||||
|
);
|
||||||
|
help.overlayTabIds.add(tabId);
|
||||||
|
return legacyFinish;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rearmHelp(help: ActiveHelpRequest, tabId: number): Promise<boolean> {
|
||||||
|
for (let attempt = 0; attempt < HELP_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const legacyFinish = await refreshAndSendHelpOverlay(help, tabId);
|
||||||
|
if (legacyFinish && !help.settled) {
|
||||||
|
await finishHelp(
|
||||||
|
help,
|
||||||
|
{
|
||||||
|
outcome: legacyFinish.outcome,
|
||||||
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
if (attempt + 1 < HELP_REARM_MAX_ATTEMPTS) await sleep(HELP_REARM_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleRearmForTab(tabId: number, deps: RequestHelpDeps): void {
|
||||||
|
const existing = helpRearmTimers.get(tabId);
|
||||||
|
if (existing) clearTimeout(existing);
|
||||||
|
helpRearmTimers.set(
|
||||||
|
tabId,
|
||||||
|
setTimeout(() => {
|
||||||
|
helpRearmTimers.delete(tabId);
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (!help) return;
|
||||||
|
await rearmHelp(help, tabId);
|
||||||
|
})();
|
||||||
|
}, HELP_REARM_DEBOUNCE_MS),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let detachHelpObservation: (() => void) | null = null;
|
||||||
|
|
||||||
|
export function ensureHelpLifecycleListeners(deps: RequestHelpDeps = getDefaultDeps()): void {
|
||||||
|
if (detachHelpObservation) return;
|
||||||
|
const detachRuntime = attachHelpRuntimeListener(deps);
|
||||||
|
const detachTabs = attachHelpTabListener(deps);
|
||||||
|
const detachNav = attachHelpNavigationListener(deps);
|
||||||
|
detachHelpObservation = () => {
|
||||||
|
detachRuntime();
|
||||||
|
detachTabs();
|
||||||
|
detachNav();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resetHelpLifecycleForTests(): void {
|
||||||
|
detachHelpObservation?.();
|
||||||
|
detachHelpObservation = null;
|
||||||
|
for (const timer of helpRearmTimers.values()) clearTimeout(timer);
|
||||||
|
helpRearmTimers.clear();
|
||||||
|
for (const help of activeHelpRequests.values()) {
|
||||||
|
clearTimeout(help.timeoutTimer);
|
||||||
|
if (help.completionTimer) clearInterval(help.completionTimer);
|
||||||
|
}
|
||||||
|
activeHelpRequests.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const listener = (
|
||||||
|
message: unknown,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: HelpQueryResponse) => void,
|
||||||
|
) => {
|
||||||
|
if (isHelpFinishMessage(message)) {
|
||||||
|
void finishHelpFromContent(message, sender, deps);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isHelpQueryMessage(message)) {
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId === undefined) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
void (async () => {
|
||||||
|
const help = await findHelpForTab(tabId, deps);
|
||||||
|
if (!help) {
|
||||||
|
sendResponse({ active: false });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (tabId === help.primaryTabId) {
|
||||||
|
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||||
|
}
|
||||||
|
help.overlayTabIds.add(tabId);
|
||||||
|
sendResponse({
|
||||||
|
active: true,
|
||||||
|
request: {
|
||||||
|
requestId: help.requestId,
|
||||||
|
prompt: help.prompt,
|
||||||
|
...(help.title ? { title: help.title } : {}),
|
||||||
|
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||||
|
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||||
|
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||||
|
timeoutMs: help.timeoutMs,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
chrome.runtime.onMessage.addListener(listener);
|
||||||
|
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishHelpFromContent(
|
||||||
|
message: HelpFinishMessage,
|
||||||
|
sender: chrome.runtime.MessageSender,
|
||||||
|
deps: RequestHelpDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const help = activeHelpRequests.get(message.requestId);
|
||||||
|
if (!help || help.settled) return;
|
||||||
|
const tabId = sender.tab?.id;
|
||||||
|
if (tabId !== undefined) {
|
||||||
|
const match = await findHelpForTab(tabId, deps);
|
||||||
|
if (match !== help) return;
|
||||||
|
}
|
||||||
|
await finishHelp(
|
||||||
|
help,
|
||||||
|
{
|
||||||
|
outcome: message.outcome,
|
||||||
|
...(message.note ? { note: message.note } : {}),
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpTabListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||||
|
if (typeof tab.id !== "number") return;
|
||||||
|
scheduleRearmForTab(tab.id, deps);
|
||||||
|
};
|
||||||
|
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||||
|
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||||
|
};
|
||||||
|
chrome.tabs.onCreated?.addListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.addListener(onActivated);
|
||||||
|
return () => {
|
||||||
|
chrome.tabs.onCreated?.removeListener(onCreated);
|
||||||
|
chrome.tabs.onActivated?.removeListener(onActivated);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachHelpNavigationListener(deps: RequestHelpDeps): () => void {
|
||||||
|
const onMainFrameComplete = (tabId: number) => scheduleRearmForTab(tabId, deps);
|
||||||
|
|
||||||
|
if (chrome.webNavigation?.onCompleted) {
|
||||||
|
const completedListener = (
|
||||||
|
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||||
|
) => {
|
||||||
|
if (details.frameId !== 0) return;
|
||||||
|
onMainFrameComplete(details.tabId);
|
||||||
|
};
|
||||||
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
|
return () => chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||||
|
}
|
||||||
|
|
||||||
|
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||||
|
if (info.status !== "complete") return;
|
||||||
|
onMainFrameComplete(tabId);
|
||||||
|
};
|
||||||
|
chrome.tabs.onUpdated?.addListener(listener);
|
||||||
|
return () => chrome.tabs.onUpdated?.removeListener(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateCompletionCondition(
|
||||||
|
condition: HelpCompletionCondition,
|
||||||
|
help: ActiveHelpRequest,
|
||||||
|
): Promise<boolean> {
|
||||||
|
if (condition.url_contains || condition.url_matches) {
|
||||||
|
let url = "";
|
||||||
|
try {
|
||||||
|
const tab = await help.deps.tabsApi.get(help.primaryTabId);
|
||||||
|
url = tab.url ?? "";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (condition.url_contains && !url.includes(condition.url_contains)) return false;
|
||||||
|
if (condition.url_matches) {
|
||||||
|
try {
|
||||||
|
if (!new RegExp(condition.url_matches).test(url)) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
condition.selector_exists ||
|
||||||
|
condition.selector_missing ||
|
||||||
|
condition.text_exists ||
|
||||||
|
condition.text_missing
|
||||||
|
) {
|
||||||
|
if (!help.deps.cdp) return false;
|
||||||
|
const expression = `(() => {
|
||||||
|
const selectorExists = ${JSON.stringify(condition.selector_exists ?? null)};
|
||||||
|
const selectorMissing = ${JSON.stringify(condition.selector_missing ?? null)};
|
||||||
|
const textExists = ${JSON.stringify(condition.text_exists ?? null)};
|
||||||
|
const textMissing = ${JSON.stringify(condition.text_missing ?? null)};
|
||||||
|
if (selectorExists && !document.querySelector(selectorExists)) return false;
|
||||||
|
if (selectorMissing && document.querySelector(selectorMissing)) return false;
|
||||||
|
const text = document.body ? document.body.innerText || "" : "";
|
||||||
|
if (textExists && !text.includes(textExists)) return false;
|
||||||
|
if (textMissing && text.includes(textMissing)) return false;
|
||||||
|
return true;
|
||||||
|
})()`;
|
||||||
|
try {
|
||||||
|
const result = await help.deps.cdp.send<{ result?: { value?: boolean } }>(
|
||||||
|
help.primaryTabId,
|
||||||
|
"Runtime.evaluate",
|
||||||
|
{ expression, returnByValue: true },
|
||||||
|
);
|
||||||
|
if (result.result?.value !== true) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function completionCriteriaMatches(help: ActiveHelpRequest): Promise<boolean> {
|
||||||
|
const criteria = help.completionCriteria;
|
||||||
|
if (!criteria) return false;
|
||||||
|
const any = criteria.any ?? [];
|
||||||
|
const all = criteria.all ?? [];
|
||||||
|
const hasAny = any.length > 0;
|
||||||
|
const hasAll = all.length > 0;
|
||||||
|
if (!hasAny && !hasAll) return false;
|
||||||
|
|
||||||
|
if (hasAll) {
|
||||||
|
for (const condition of all) {
|
||||||
|
if (!(await evaluateCompletionCondition(condition, help))) return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (hasAny) {
|
||||||
|
for (const condition of any) {
|
||||||
|
if (await evaluateCompletionCondition(condition, help)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function startCompletionPolling(help: ActiveHelpRequest): void {
|
||||||
|
if (!help.completionCriteria) return;
|
||||||
|
const stableMs = help.completionCriteria.stable_for_ms ?? DEFAULT_COMPLETION_STABLE_MS;
|
||||||
|
const check = async () => {
|
||||||
|
if (help.settled) return;
|
||||||
|
const matched = await completionCriteriaMatches(help);
|
||||||
|
const now = Date.now();
|
||||||
|
if (!matched) {
|
||||||
|
help.completionMatchedSince = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
help.completionMatchedSince ??= now;
|
||||||
|
if (now - help.completionMatchedSince < stableMs) return;
|
||||||
|
void finishHelp(help, {
|
||||||
|
outcome: "completed",
|
||||||
|
completed_by: "system",
|
||||||
|
tab_id: help.primaryTabId,
|
||||||
|
resolved_targets: help.resolvedTargets,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
help.completionTimer = setInterval(() => void check(), COMPLETION_POLL_MS);
|
||||||
|
void check();
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleRequestHelp(
|
export async function handleRequestHelp(
|
||||||
@@ -195,59 +663,24 @@ export async function handleRequestHelp(
|
|||||||
if (!params.prompt || typeof params.prompt !== "string") {
|
if (!params.prompt || typeof params.prompt !== "string") {
|
||||||
return { code: "invalid_params", message: "request_help requires a prompt" };
|
return { code: "invalid_params", message: "request_help requires a prompt" };
|
||||||
}
|
}
|
||||||
if (deps.signal?.aborted) {
|
if (deps.signal?.aborted) return { code: "cancelled", message: "request_help aborted" };
|
||||||
return { code: "cancelled", message: "request_help aborted" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
const denied = enforceAgentWindow(ctx, target, "request_help");
|
const denied = enforceAgentWindow(ctx, target, "request_help");
|
||||||
if (denied) return denied;
|
if (denied) return denied;
|
||||||
|
|
||||||
|
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||||
const tabId = target.tabId;
|
const tabId = target.tabId;
|
||||||
|
const initialTargets = params.targets ?? [];
|
||||||
|
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||||
|
ctx,
|
||||||
|
tabId,
|
||||||
|
initialTargets,
|
||||||
|
deps,
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
|
|
||||||
// Resolve targets → selectors. `selector` passes through; `ref` is
|
|
||||||
// tagged with an attribute via CDP and reported as its attr selector.
|
|
||||||
const targets = params.targets ?? [];
|
|
||||||
const selectors: string[] = [];
|
|
||||||
const resolved: ResolvedTarget[] = [];
|
|
||||||
let rootNodeId: number | undefined;
|
|
||||||
if (deps.cdp && targets.length > 0) {
|
|
||||||
try {
|
|
||||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
|
||||||
depth: 0,
|
|
||||||
});
|
|
||||||
rootNodeId = doc.root?.nodeId;
|
|
||||||
} catch {
|
|
||||||
rootNodeId = undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (let i = 0; i < targets.length; i++) {
|
|
||||||
const tgt = targets[i] as HelpTarget;
|
|
||||||
if (tgt.selector) {
|
|
||||||
selectors.push(tgt.selector);
|
|
||||||
let matched: boolean;
|
|
||||||
if (deps.cdp) {
|
|
||||||
matched =
|
|
||||||
rootNodeId !== undefined
|
|
||||||
? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector)
|
|
||||||
: false;
|
|
||||||
} else {
|
|
||||||
matched = true;
|
|
||||||
}
|
|
||||||
resolved.push({ matched, selector: tgt.selector });
|
|
||||||
} else if (tgt.ref) {
|
|
||||||
const looked = lookupSnapshotRef(ctx, tgt.ref, tabId);
|
|
||||||
const backendNodeId = looked?.backendNodeId ?? null;
|
|
||||||
let sel: string | null = null;
|
|
||||||
if (backendNodeId !== null && deps.cdp) {
|
|
||||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
|
||||||
}
|
|
||||||
if (sel) selectors.push(sel);
|
|
||||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bring the tab to the foreground.
|
|
||||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||||
await deps.activateTab(tabId).catch(() => {});
|
await deps.activateTab(tabId).catch(() => {});
|
||||||
|
|
||||||
@@ -255,7 +688,6 @@ export async function handleRequestHelp(
|
|||||||
const notificationId = `bsk-help:${requestId}`;
|
const notificationId = `bsk-help:${requestId}`;
|
||||||
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
const timeoutMs = params.timeout_ms ?? DEFAULT_HELP_TIMEOUT_MS;
|
||||||
|
|
||||||
// OS notification (best-effort).
|
|
||||||
if (deps.notifications) {
|
if (deps.notifications) {
|
||||||
const copy = deps.notificationCopy ?? {
|
const copy = deps.notificationCopy ?? {
|
||||||
title: "BrowserSkill: Agent needs your help",
|
title: "BrowserSkill: Agent needs your help",
|
||||||
@@ -272,85 +704,66 @@ export async function handleRequestHelp(
|
|||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
const cleanup = async () => {
|
return new Promise<RequestHelpResult | RpcError>((resolve) => {
|
||||||
await clearRefTags(deps.cdp, tabId);
|
const timeoutTimer = setTimeout(() => {
|
||||||
if (deps.notifications) await deps.notifications.clear(notificationId).catch(() => {});
|
void finishHelp(help, {
|
||||||
// Retract the overlay in case we resolved via timeout / abort.
|
outcome: "timed_out",
|
||||||
await deps
|
tab_id: help.primaryTabId,
|
||||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId } satisfies HelpCancelMessage)
|
resolved_targets: help.resolvedTargets,
|
||||||
.catch(() => {});
|
});
|
||||||
};
|
}, timeoutMs);
|
||||||
|
|
||||||
const request: HelpRequestMessage = {
|
const help: ActiveHelpRequest = {
|
||||||
type: HELP_REQUEST,
|
ctx,
|
||||||
requestId,
|
requestId,
|
||||||
prompt: params.prompt,
|
primaryTabId: tabId,
|
||||||
...(params.title ? { title: params.title } : {}),
|
overlayTabIds: new Set(),
|
||||||
selectors,
|
prompt: params.prompt,
|
||||||
timeoutMs,
|
...(params.title ? { title: params.title } : {}),
|
||||||
};
|
targets: initialTargets,
|
||||||
|
selectors,
|
||||||
const resolvedTargets = resolved.length > 0 ? resolved : undefined;
|
rects,
|
||||||
|
timeoutMs,
|
||||||
return new Promise<RequestHelpResult | RpcError>((resolveOuter) => {
|
notificationId,
|
||||||
let settled = false;
|
resolvedTargets,
|
||||||
const unwatchNav = deps.watchTabNavigation(tabId, () => {
|
completionCriteria: params.completion_criteria,
|
||||||
finish({ outcome: "navigated", tab_id: tabId, resolved_targets: resolvedTargets });
|
deps,
|
||||||
});
|
settled: false,
|
||||||
|
resolve,
|
||||||
const finish = (value: RequestHelpResult | RpcError) => {
|
timeoutTimer,
|
||||||
if (settled) return;
|
completionTimer: null,
|
||||||
settled = true;
|
completionMatchedSince: null,
|
||||||
if (timer !== null) clearTimeout(timer);
|
abortHandler: null,
|
||||||
unwatchNav();
|
|
||||||
deps.signal?.removeEventListener("abort", onAbort);
|
|
||||||
void cleanup();
|
|
||||||
resolveOuter(value);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const onAbort = () => finish({ code: "cancelled", message: "request_help aborted" });
|
const onAbort = () =>
|
||||||
|
void finishHelp(help, { code: "cancelled", message: "request_help aborted" });
|
||||||
const timer = setTimeout(() => {
|
help.abortHandler = onAbort;
|
||||||
finish({ outcome: "timed_out", tab_id: tabId, resolved_targets: resolvedTargets });
|
activeHelpRequests.set(requestId, help);
|
||||||
}, timeoutMs) as unknown as number;
|
if (deps.signal?.aborted) {
|
||||||
|
onAbort();
|
||||||
if (deps.signal) {
|
return;
|
||||||
if (deps.signal.aborted) {
|
|
||||||
onAbort();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
deps.signal.addEventListener("abort", onAbort, { once: true });
|
|
||||||
}
|
}
|
||||||
|
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
startCompletionPolling(help);
|
||||||
|
|
||||||
deps
|
void sendCurrentHelpOverlay(help)
|
||||||
.sendToTab(tabId, request)
|
.then(async (legacyFinish) => {
|
||||||
.then((reply) => {
|
if (!legacyFinish || help.settled) return;
|
||||||
const res = reply as HelpResponseMessage | undefined;
|
await finishHelp(
|
||||||
if (
|
help,
|
||||||
!res ||
|
{
|
||||||
res.type !== HELP_RESPONSE ||
|
outcome: legacyFinish.outcome,
|
||||||
(res.outcome !== "continued" && res.outcome !== "cancelled")
|
...(legacyFinish.note ? { note: legacyFinish.note } : {}),
|
||||||
) {
|
tab_id: help.primaryTabId,
|
||||||
finish({
|
resolved_targets: help.resolvedTargets,
|
||||||
code: "protocol_error",
|
},
|
||||||
message: `invalid help response from tab ${tabId}`,
|
true,
|
||||||
});
|
true,
|
||||||
return;
|
);
|
||||||
}
|
|
||||||
finish({
|
|
||||||
outcome: res.outcome,
|
|
||||||
...(res.note ? { note: res.note } : {}),
|
|
||||||
tab_id: tabId,
|
|
||||||
resolved_targets: resolvedTargets,
|
|
||||||
});
|
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch(() => {
|
||||||
finish({
|
scheduleRearmForTab(tabId, deps);
|
||||||
code: "protocol_error",
|
|
||||||
message: `failed to show help overlay on tab ${tabId}: ${
|
|
||||||
err instanceof Error ? err.message : String(err)
|
|
||||||
}`,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,20 +3,23 @@
|
|||||||
//
|
//
|
||||||
// All interaction tools:
|
// All interaction tools:
|
||||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
// or `selector` (DOM.querySelector + describeNode).
|
||||||
// 3. Scroll the node into view, then dispatch the appropriate
|
// 3. Scroll the node and its frame owners into view, project its live
|
||||||
// `Input.*` CDP events.
|
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||||
// commands.
|
// commands.
|
||||||
|
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
ClickResult,
|
ClickResult,
|
||||||
FillParams,
|
FillParams,
|
||||||
FillResult,
|
FillResult,
|
||||||
|
HoverParams,
|
||||||
|
HoverResult,
|
||||||
KeyModifier,
|
KeyModifier,
|
||||||
MouseButton,
|
MouseButton,
|
||||||
PressParams,
|
PressParams,
|
||||||
@@ -26,21 +29,18 @@ import type {
|
|||||||
SelectResult,
|
SelectResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import {
|
import { backendNodeToObject } from "./element-geometry";
|
||||||
backendNodeToObject,
|
|
||||||
boxCentre,
|
|
||||||
nodeCentre,
|
|
||||||
quadCentre,
|
|
||||||
scrollNodeIntoView,
|
|
||||||
} from "./element-geometry";
|
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
|
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
|
cdpRunnerForTarget,
|
||||||
chromeTabsApi,
|
chromeTabsApi,
|
||||||
enforceAgentWindow,
|
enforceAgentWindow,
|
||||||
isRpcError,
|
isRpcError,
|
||||||
lookupSession,
|
lookupSession,
|
||||||
|
type ResolvedTargetTab,
|
||||||
resolveTargetTab,
|
resolveTargetTab,
|
||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
@@ -53,9 +53,21 @@ export interface InteractionDeps {
|
|||||||
defaultTimeoutMs?: number;
|
defaultTimeoutMs?: number;
|
||||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||||
|
keepOverlayBypassAfterHover?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResolvedActionTarget {
|
||||||
|
tab: ResolvedTargetTab;
|
||||||
|
backendNodeId: number;
|
||||||
|
cdpTarget: CdpTarget;
|
||||||
|
frameId?: string;
|
||||||
|
usedRef?: string;
|
||||||
|
usedSelector?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||||
|
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||||
|
|
||||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||||
@@ -100,18 +112,48 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAbortLikeError(err: unknown): boolean {
|
||||||
|
return err instanceof DOMException && err.name === "AbortError";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
cleanup();
|
||||||
|
reject(new DOMException("aborted", "AbortError"));
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||||
* lookup matched.
|
* lookup matched.
|
||||||
*/
|
*/
|
||||||
async function resolveBackendNode(
|
export async function resolveBackendNode(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
target: { tabId: number },
|
target: { tabId: number },
|
||||||
params: { ref?: string; selector?: string },
|
params: { ref?: string; selector?: string },
|
||||||
toolName: string,
|
toolName: string,
|
||||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
): Promise<
|
||||||
|
| {
|
||||||
|
backendNodeId: number;
|
||||||
|
cdpTarget: CdpTarget;
|
||||||
|
frameId?: string;
|
||||||
|
usedRef?: string;
|
||||||
|
usedSelector?: string;
|
||||||
|
}
|
||||||
|
| RpcError
|
||||||
|
> {
|
||||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||||
if (hasRef && hasSelector) {
|
if (hasRef && hasSelector) {
|
||||||
@@ -129,7 +171,15 @@ async function resolveBackendNode(
|
|||||||
if (hasRef) {
|
if (hasRef) {
|
||||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||||
if (isRpcError(resolved)) return resolved;
|
if (isRpcError(resolved)) return resolved;
|
||||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
return {
|
||||||
|
backendNodeId: resolved.backendNodeId,
|
||||||
|
cdpTarget: {
|
||||||
|
tabId: target.tabId,
|
||||||
|
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||||
|
},
|
||||||
|
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||||
|
usedRef: resolved.refKey,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
// selector path
|
// selector path
|
||||||
try {
|
try {
|
||||||
@@ -166,7 +216,7 @@ async function resolveBackendNode(
|
|||||||
message: "DOM.describeNode returned no backendNodeId",
|
message: "DOM.describeNode returned no backendNodeId",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { backendNodeId, usedSelector: params.selector };
|
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -175,6 +225,17 @@ async function resolveBackendNode(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function resolveActionTarget(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
ctx: SessionContext,
|
||||||
|
target: ResolvedTargetTab,
|
||||||
|
params: { ref?: string; selector?: string },
|
||||||
|
toolName: string,
|
||||||
|
): Promise<ResolvedActionTarget | RpcError> {
|
||||||
|
const node = await resolveBackendNode(cdp, ctx, target, params, toolName);
|
||||||
|
return isRpcError(node) ? node : { tab: target, ...node };
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// tool.click
|
// tool.click
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -193,28 +254,37 @@ export async function handleClick(
|
|||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
const denied = enforceAgentWindow(ctx, target, "click");
|
const denied = enforceAgentWindow(ctx, target, "click");
|
||||||
if (denied) return denied;
|
if (denied) return denied;
|
||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
|
||||||
|
if (isRpcError(resolved)) return resolved;
|
||||||
|
return clickResolvedTarget(ctx, resolved, params, deps);
|
||||||
|
}
|
||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
export async function clickResolvedTarget(
|
||||||
if (isRpcError(node)) return node;
|
ctx: SessionContext,
|
||||||
|
resolved: ResolvedActionTarget,
|
||||||
|
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||||
|
deps: InteractionDeps,
|
||||||
|
): Promise<ClickResult | RpcError> {
|
||||||
|
const { tab: target } = resolved;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
const geometry = await resolveNodeGeometry(
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
deps.cdp,
|
||||||
if (scrollErr) return scrollErr;
|
target.tabId,
|
||||||
} catch (err) {
|
{
|
||||||
return {
|
target: resolved.cdpTarget,
|
||||||
code: "cdp_failed",
|
backendNodeId: resolved.backendNodeId,
|
||||||
message: err instanceof Error ? err.message : String(err),
|
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||||
};
|
},
|
||||||
}
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
if (isRpcError(geometry)) return geometry;
|
||||||
if (isRpcError(centre)) return centre;
|
const centre = geometry.actionPoint;
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
@@ -286,7 +356,104 @@ export async function handleClick(
|
|||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: resolved.usedRef,
|
||||||
|
used_selector: resolved.usedSelector,
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tool.hover
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function handleHover(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: HoverParams,
|
||||||
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
|
): Promise<HoverResult | RpcError> {
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
const aborted = throwIfAborted(deps.signal);
|
||||||
|
if (aborted) return aborted;
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||||
|
if (denied) return denied;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||||
|
if (isRpcError(node)) return node;
|
||||||
|
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const geometry = await resolveNodeGeometry(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
{
|
||||||
|
target: node.cdpTarget,
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||||
|
},
|
||||||
|
{ scrollIntoView: true },
|
||||||
|
);
|
||||||
|
if (isRpcError(geometry)) return geometry;
|
||||||
|
const centre = geometry.actionPoint;
|
||||||
|
|
||||||
|
if (throwIfAborted(deps.signal)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const modifiers = modifiersBitfield(params.modifiers);
|
||||||
|
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||||
|
let automationBypassEnabled = false;
|
||||||
|
let hoverCompleted = false;
|
||||||
|
if (overlayBlocking && deps.bypassOverlay) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(target.tabId, true);
|
||||||
|
automationBypassEnabled = true;
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: centre.x,
|
||||||
|
y: centre.y,
|
||||||
|
modifiers,
|
||||||
|
});
|
||||||
|
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||||
|
if (settleMs > 0) {
|
||||||
|
await wait(settleMs, deps.signal);
|
||||||
|
}
|
||||||
|
hoverCompleted = true;
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortLikeError(err)) {
|
||||||
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
if (
|
||||||
|
automationBypassEnabled &&
|
||||||
|
deps.bypassOverlay &&
|
||||||
|
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
await deps.bypassOverlay(target.tabId, false);
|
await deps.bypassOverlay(target.tabId, false);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -462,10 +629,11 @@ export async function handleFill(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{
|
{
|
||||||
@@ -479,12 +647,18 @@ export async function handleFill(
|
|||||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -496,7 +670,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
@@ -505,7 +679,7 @@ export async function handleFill(
|
|||||||
if (clearBefore) {
|
if (clearBefore) {
|
||||||
// Clear input/textarea value or wipe contenteditable innerText,
|
// Clear input/textarea value or wipe contenteditable innerText,
|
||||||
// then fire `input` so frameworks observe the empty state.
|
// then fire `input` so frameworks observe the empty state.
|
||||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
if (this.isContentEditable) { this.textContent = ''; }
|
if (this.isContentEditable) { this.textContent = ''; }
|
||||||
@@ -532,7 +706,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
@@ -730,14 +904,21 @@ export async function handlePress(
|
|||||||
if (params.ref || params.selector) {
|
if (params.ref || params.selector) {
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "press aborted" };
|
return { code: "cancelled", message: "press aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -842,10 +1023,11 @@ export async function handleSelect(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{ backendNodeId: node.backendNodeId },
|
{ backendNodeId: node.backendNodeId },
|
||||||
@@ -868,12 +1050,18 @@ export async function handleSelect(
|
|||||||
"single-select <select> requires exactly one value",
|
"single-select <select> requires exactly one value",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
node.cdpTarget,
|
||||||
|
node.backendNodeId,
|
||||||
|
node.frameId,
|
||||||
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -885,12 +1073,12 @@ export async function handleSelect(
|
|||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const evaluated = await deps.cdp.send<{
|
const evaluated = await nodeCdp.send<{
|
||||||
result?: { value?: SelectMutationResult | null };
|
result?: { value?: SelectMutationResult | null };
|
||||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
@@ -949,9 +1137,6 @@ export async function handleSelect(
|
|||||||
|
|
||||||
export const __testing__ = {
|
export const __testing__ = {
|
||||||
DEFAULT_TIMEOUT_MS,
|
DEFAULT_TIMEOUT_MS,
|
||||||
quadCentre,
|
|
||||||
boxCentre,
|
|
||||||
resolveBackendNode,
|
resolveBackendNode,
|
||||||
nodeCentre,
|
|
||||||
isFillable,
|
isFillable,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -325,14 +325,14 @@ function startLifecycleWait(
|
|||||||
if (source.tabId !== expectedTabId) return;
|
if (source.tabId !== expectedTabId) return;
|
||||||
|
|
||||||
if (targetName === "commit" && method === "Page.frameNavigated") {
|
if (targetName === "commit" && method === "Page.frameNavigated") {
|
||||||
const p = params as { frame?: { id?: string; parentId?: string } };
|
const p = params as { frame?: { id?: string; parentId?: string; loaderId?: string } };
|
||||||
const expectedFrameId = currentFrameId();
|
const expectedFrameId = currentFrameId();
|
||||||
if (expectedFrameId.length > 0) {
|
if (expectedFrameId.length > 0) {
|
||||||
if (p.frame?.id !== expectedFrameId) return;
|
if (p.frame?.id !== expectedFrameId) return;
|
||||||
} else if (p.frame?.parentId) {
|
} else if (p.frame?.parentId) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!acceptLifecycleEvent("commit", p.frame?.id)) return;
|
if (!acceptLifecycleEvent("commit", p.frame?.id, p.frame?.loaderId)) return;
|
||||||
finish({ reached: "match", lastLifecycle: "commit" });
|
finish({ reached: "match", lastLifecycle: "commit" });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,17 +37,22 @@ export async function handleNetwork(
|
|||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: NetworkParams,
|
params: NetworkParams,
|
||||||
deps: NetworkDeps = defaultNetworkDeps(),
|
deps: NetworkDeps = defaultNetworkDeps(),
|
||||||
|
signal?: AbortSignal,
|
||||||
): Promise<NetworkResult | RpcError> {
|
): Promise<NetworkResult | RpcError> {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
const ctxOrErr = lookupSession(manager, params, "network");
|
const ctxOrErr = lookupSession(manager, params, "network");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
const bounds = parseBufferedReadBounds(params);
|
const bounds = parseBufferedReadBounds(params);
|
||||||
if (isRpcError(bounds)) return bounds;
|
if (isRpcError(bounds)) return bounds;
|
||||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||||
if (isRpcError(target)) return target;
|
if (isRpcError(target)) return target;
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureNetworkCapture(target.tabId);
|
await deps.cdp.ensureNetworkCapture(target.tabId);
|
||||||
|
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||||
return deps.cdp.networkEntriesSince(
|
return deps.cdp.networkEntriesSince(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
bounds.since,
|
bounds.since,
|
||||||
|
|||||||
+1350
-217
File diff suppressed because it is too large
Load Diff
@@ -49,6 +49,7 @@ interface ActiveRecording {
|
|||||||
purpose?: string;
|
purpose?: string;
|
||||||
steps: DraftTraceStep[];
|
steps: DraftTraceStep[];
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
|
startedAtMs: number;
|
||||||
finishPromise: Promise<Trace>;
|
finishPromise: Promise<Trace>;
|
||||||
resolveFinish: (trace: Trace) => void;
|
resolveFinish: (trace: Trace) => void;
|
||||||
rejectFinish: (err: Error) => void;
|
rejectFinish: (err: Error) => void;
|
||||||
@@ -373,7 +374,11 @@ async function rearmRecording(
|
|||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||||
// already hides the control chrome while activeRecord is set.
|
// already hides the control chrome while activeRecord is set.
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
|
const startMsg: RecordStartMessage = {
|
||||||
|
type: RECORD_START,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
};
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||||
recording.tabId = targetTabId;
|
recording.tabId = targetTabId;
|
||||||
@@ -501,7 +506,11 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await rearmRecording(recording, tabId, deps);
|
await rearmRecording(recording, tabId, deps);
|
||||||
sendResponse({ active: true, requestId: recording.requestId });
|
sendResponse({
|
||||||
|
active: true,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
});
|
||||||
})();
|
})();
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
@@ -572,6 +581,7 @@ export async function handleRecordStart(
|
|||||||
rejectFinish = reject;
|
rejectFinish = reject;
|
||||||
});
|
});
|
||||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||||
|
const startedAtMs = Date.now();
|
||||||
recordings.set(params.session_id, {
|
recordings.set(params.session_id, {
|
||||||
requestId,
|
requestId,
|
||||||
tabId: target.tabId,
|
tabId: target.tabId,
|
||||||
@@ -579,7 +589,8 @@ export async function handleRecordStart(
|
|||||||
startUrl: navigateUrl,
|
startUrl: navigateUrl,
|
||||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||||
steps: [],
|
steps: [],
|
||||||
startedAt: new Date().toISOString(),
|
startedAt: new Date(startedAtMs).toISOString(),
|
||||||
|
startedAtMs,
|
||||||
finishPromise,
|
finishPromise,
|
||||||
resolveFinish,
|
resolveFinish,
|
||||||
rejectFinish,
|
rejectFinish,
|
||||||
@@ -707,7 +718,7 @@ export async function handleRecordStart(
|
|||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
}
|
}
|
||||||
|
|
||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
|
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||||
|
|||||||
@@ -1,17 +1,68 @@
|
|||||||
import type { SessionManager } from "@/session-manager/manager";
|
import { type SessionManager, SessionStartCleanupError } from "@/session-manager/manager";
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { rpcError } from "./errors";
|
||||||
import { clearRecordingForSession } from "./record";
|
import { clearRecordingForSession } from "./record";
|
||||||
|
import { isRpcError } from "./shared";
|
||||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||||
|
|
||||||
|
/** Valid range for Agent Window dimensions in CSS pixels. */
|
||||||
|
export const WINDOW_SIZE_MIN = 100;
|
||||||
|
export const WINDOW_SIZE_MAX = 7680;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate an optional width/height pair. Both must be given (or
|
||||||
|
* neither), be integers, and fall within 100..=7680. Returns the size
|
||||||
|
* to pass down, or an `invalid_params` RpcError.
|
||||||
|
*/
|
||||||
|
export function validateWindowSize(
|
||||||
|
width: unknown,
|
||||||
|
height: unknown,
|
||||||
|
): { width: number; height: number } | undefined | RpcError {
|
||||||
|
if (width === undefined && height === undefined) return undefined;
|
||||||
|
if (width === undefined || height === undefined) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "width and height must be given together",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
for (const [name, value] of [
|
||||||
|
["width", width],
|
||||||
|
["height", height],
|
||||||
|
] as const) {
|
||||||
|
if (
|
||||||
|
typeof value !== "number" ||
|
||||||
|
!Number.isInteger(value) ||
|
||||||
|
value < WINDOW_SIZE_MIN ||
|
||||||
|
value > WINDOW_SIZE_MAX
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: `${name} must be an integer in 100..=7680`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { width: width as number, height: height as number };
|
||||||
|
}
|
||||||
|
|
||||||
export interface SessionStartParams {
|
export interface SessionStartParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
browser_instance_id?: string;
|
browser_instance_id?: string;
|
||||||
|
/** Optional Agent Window outer width in CSS pixels (100..=7680). */
|
||||||
|
width?: number;
|
||||||
|
/** Optional Agent Window outer height in CSS pixels (100..=7680). */
|
||||||
|
height?: number;
|
||||||
|
/** Defaults to true so existing clients preserve visible Agent Windows. */
|
||||||
|
focused?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SessionStartResult {
|
export interface SessionStartResult {
|
||||||
agent_window_id?: number;
|
agent_window_id?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SessionStartDeps {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SessionStopParams {
|
export interface SessionStopParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
}
|
}
|
||||||
@@ -45,6 +96,7 @@ export interface SessionStopDeps {
|
|||||||
export async function handleSessionStart(
|
export async function handleSessionStart(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SessionStartParams,
|
params: SessionStartParams,
|
||||||
|
deps: SessionStartDeps = {},
|
||||||
): Promise<SessionStartResult | RpcError> {
|
): Promise<SessionStartResult | RpcError> {
|
||||||
if (!params?.session_id) {
|
if (!params?.session_id) {
|
||||||
return {
|
return {
|
||||||
@@ -52,10 +104,29 @@ export async function handleSessionStart(
|
|||||||
message: "session.start requires session_id",
|
message: "session.start requires session_id",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
const sizeOrErr = validateWindowSize(params.width, params.height);
|
||||||
|
if (isRpcError(sizeOrErr)) return sizeOrErr;
|
||||||
try {
|
try {
|
||||||
const ctx = await manager.start(params.session_id);
|
const ctx = await manager.start(params.session_id, {
|
||||||
|
size: sizeOrErr,
|
||||||
|
focused: params.focused,
|
||||||
|
signal: deps.signal,
|
||||||
|
});
|
||||||
return { agent_window_id: ctx.agentWindowId };
|
return { agent_window_id: ctx.agentWindowId };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (err instanceof SessionStartCleanupError) {
|
||||||
|
return rpcError("protocol_error", "cleanup_failed", err.message, {
|
||||||
|
resource_type: "agent_window",
|
||||||
|
resource_id: err.windowId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
typeof err === "object" &&
|
||||||
|
err !== null &&
|
||||||
|
(err as { name?: string }).name === "AbortError"
|
||||||
|
) {
|
||||||
|
return { code: "cancelled", message: "session_start aborted" };
|
||||||
|
}
|
||||||
// chrome.windows.create / SessionManager failures are not CDP
|
// chrome.windows.create / SessionManager failures are not CDP
|
||||||
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
||||||
// them as protocol_error so the CLI maps to the right exit code
|
// them as protocol_error so the CLI maps to the right exit code
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
// exactly the same sandbox + visibility rules as the M6 observation
|
// exactly the same sandbox + visibility rules as the M6 observation
|
||||||
// handlers (review parity).
|
// handlers (review parity).
|
||||||
|
|
||||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||||
|
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import { normaliseRef } from "@/session-manager/ref-store";
|
import { normaliseRef } from "@/session-manager/ref-store";
|
||||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||||
@@ -32,6 +33,13 @@ export const chromeTabsApi: ChromeTabsApi = {
|
|||||||
query: (q) => chrome.tabs.query(q),
|
query: (q) => chrome.tabs.query(q),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface ResolvedTargetTab {
|
||||||
|
tabId: number;
|
||||||
|
windowId: number;
|
||||||
|
active: boolean;
|
||||||
|
url?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export type { DialogCursor };
|
export type { DialogCursor };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -44,8 +52,12 @@ export type { DialogCursor };
|
|||||||
*/
|
*/
|
||||||
export interface CdpRunner {
|
export interface CdpRunner {
|
||||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||||
|
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||||
|
detach?(tabId: number): Promise<void>;
|
||||||
|
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||||
|
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
};
|
};
|
||||||
dialogCursor?(tabId: number): DialogCursor;
|
dialogCursor?(tabId: number): DialogCursor;
|
||||||
@@ -60,6 +72,25 @@ export interface CdpRunner {
|
|||||||
): ConsoleResult;
|
): ConsoleResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function sendToCdpTarget<T = unknown>(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
target: CdpTarget,
|
||||||
|
method: string,
|
||||||
|
params?: object,
|
||||||
|
): Promise<T> {
|
||||||
|
if (target.sessionId && cdp.sendToTarget) {
|
||||||
|
return cdp.sendToTarget<T>(target, method, params);
|
||||||
|
}
|
||||||
|
return cdp.send<T>(target.tabId, method, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||||
|
return {
|
||||||
|
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||||
|
sendToCdpTarget<T>(cdp, target, method, params),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface BufferedReadBounds {
|
export interface BufferedReadBounds {
|
||||||
since: number | undefined;
|
since: number | undefined;
|
||||||
limit: number;
|
limit: number;
|
||||||
@@ -146,7 +177,7 @@ export async function resolveTargetTab(
|
|||||||
ctx: SessionContext,
|
ctx: SessionContext,
|
||||||
tabId: number | undefined,
|
tabId: number | undefined,
|
||||||
api: ChromeTabsApi,
|
api: ChromeTabsApi,
|
||||||
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
|
): Promise<ResolvedTargetTab | RpcError> {
|
||||||
if (tabId !== undefined) {
|
if (tabId !== undefined) {
|
||||||
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
|
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
|
||||||
return {
|
return {
|
||||||
@@ -176,7 +207,7 @@ export async function resolveTargetTab(
|
|||||||
message: `tab ${tabId} not found in session scope`,
|
message: `tab ${tabId} not found in session scope`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
|
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
|
||||||
}
|
}
|
||||||
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
|
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
|
||||||
const first = tabs.find((t) => typeof t.id === "number");
|
const first = tabs.find((t) => typeof t.id === "number");
|
||||||
@@ -186,7 +217,12 @@ export async function resolveTargetTab(
|
|||||||
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
|
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
|
return {
|
||||||
|
tabId: first.id,
|
||||||
|
windowId: ctx.agentWindowId,
|
||||||
|
active: first.active === true,
|
||||||
|
url: first.url,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isRpcError(v: unknown): v is RpcError {
|
export function isRpcError(v: unknown): v is RpcError {
|
||||||
@@ -202,6 +238,91 @@ export function isRpcError(v: unknown): v is RpcError {
|
|||||||
/** Re-export so M6/M7 tools keep a stable import path. */
|
/** Re-export so M6/M7 tools keep a stable import path. */
|
||||||
export { normaliseRef };
|
export { normaliseRef };
|
||||||
|
|
||||||
|
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
|
||||||
|
|
||||||
|
const CDP_BLOCKED_PROTOCOLS = new Set([
|
||||||
|
"chrome:",
|
||||||
|
"chrome-extension:",
|
||||||
|
"devtools:",
|
||||||
|
"edge:",
|
||||||
|
"brave:",
|
||||||
|
"vivaldi:",
|
||||||
|
"opera:",
|
||||||
|
]);
|
||||||
|
|
||||||
|
export function cdpBlockedUrlReason(url: string | undefined): string | null {
|
||||||
|
if (!url) return null;
|
||||||
|
let parsed: URL;
|
||||||
|
try {
|
||||||
|
parsed = new URL(url);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
|
||||||
|
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Page CDP tools cannot inspect browser/extension internal pages. Check this
|
||||||
|
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
|
||||||
|
* leak as ambiguous page-read failures.
|
||||||
|
*/
|
||||||
|
export function enforceCdpAccessibleTarget(
|
||||||
|
target: ResolvedTargetTab,
|
||||||
|
toolName: string,
|
||||||
|
): RpcError | null {
|
||||||
|
const reason = cdpBlockedUrlReason(target.url);
|
||||||
|
if (!reason) return null;
|
||||||
|
return rpcError(
|
||||||
|
"permission_denied",
|
||||||
|
"restricted_tab_url",
|
||||||
|
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvedTargetFromChromeTab(
|
||||||
|
tab: chrome.tabs.Tab,
|
||||||
|
fallbackWindowId: number,
|
||||||
|
): ResolvedTargetTab | null {
|
||||||
|
if (typeof tab.id !== "number") return null;
|
||||||
|
return {
|
||||||
|
tabId: tab.id,
|
||||||
|
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
|
||||||
|
active: tab.active === true,
|
||||||
|
url: tab.url,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
|
||||||
|
* rejected when browser security blocks the URL. For default targeting, skip a
|
||||||
|
* restricted Agent Window active tab if another CDP-accessible tab exists in
|
||||||
|
* the same Agent Window.
|
||||||
|
*/
|
||||||
|
export async function resolveCdpAccessibleTargetTab(
|
||||||
|
manager: SessionManager,
|
||||||
|
ctx: SessionContext,
|
||||||
|
tabId: number | undefined,
|
||||||
|
api: ChromeTabsApi,
|
||||||
|
toolName: string,
|
||||||
|
): Promise<ResolvedTargetTab | RpcError> {
|
||||||
|
const target = await resolveTargetTab(manager, ctx, tabId, api);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
|
||||||
|
const restricted = enforceCdpAccessibleTarget(target, toolName);
|
||||||
|
if (!restricted) return target;
|
||||||
|
if (tabId !== undefined) return restricted;
|
||||||
|
|
||||||
|
const tabs = await api.query({ windowId: ctx.agentWindowId });
|
||||||
|
for (const tab of tabs) {
|
||||||
|
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
|
||||||
|
if (!candidate) continue;
|
||||||
|
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
|
||||||
|
}
|
||||||
|
return restricted;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
|
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
|
||||||
* MUST refuse to touch a tab outside the session's Agent Window
|
* MUST refuse to touch a tab outside the session's Agent Window
|
||||||
@@ -224,3 +345,17 @@ export function enforceAgentWindow(
|
|||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unified target-scope policy by tool effect. Passive reads may inspect user
|
||||||
|
* tabs; any tool that dispatches page input must stay inside the Agent Window.
|
||||||
|
*/
|
||||||
|
export function enforceToolTargetScope(
|
||||||
|
ctx: SessionContext,
|
||||||
|
target: { tabId: number; windowId: number },
|
||||||
|
effect: ToolEffect,
|
||||||
|
toolName: string,
|
||||||
|
): RpcError | null {
|
||||||
|
if (effect === "passive_read") return null;
|
||||||
|
return enforceAgentWindow(ctx, target, toolName);
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user