mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd85b80c78 | ||
|
|
464a8da86b | ||
|
|
9b6a525439 | ||
|
|
61a0e0d218 | ||
|
|
67b7227abf | ||
|
|
02d1d69379 | ||
|
|
025751ef03 | ||
|
|
13a61b2688 | ||
|
|
1508839bf9 |
@@ -1,105 +0,0 @@
|
|||||||
name: Release dsh plugin
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- dsh-plugin-v*
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: release-dsh-plugin-${{ github.ref }}
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
env:
|
|
||||||
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
|
||||||
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
resolve:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
version: ${{ steps.version.outputs.version }}
|
|
||||||
tag: ${{ steps.version.outputs.tag }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Resolve plugin version
|
|
||||||
id: version
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
|
||||||
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
|
||||||
else
|
|
||||||
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
|
||||||
fi
|
|
||||||
VERSION="${VERSION#v}"
|
|
||||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
|
||||||
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
guard:
|
|
||||||
needs: resolve
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- name: Verify tag matches package.json name and version
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
|
||||||
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
|
||||||
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
|
||||||
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
|
||||||
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
|
||||||
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
|
||||||
|
|
||||||
publish:
|
|
||||||
needs: [resolve, guard]
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
|
||||||
environment: npm-publish
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
|
|
||||||
- uses: pnpm/action-setup@v6
|
|
||||||
|
|
||||||
- uses: actions/setup-node@v6
|
|
||||||
with:
|
|
||||||
node-version: 22
|
|
||||||
cache: pnpm
|
|
||||||
registry-url: https://registry.npmjs.org
|
|
||||||
scope: "@wxg-prc-cpg"
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: pnpm install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Typecheck plugin
|
|
||||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
|
||||||
|
|
||||||
- name: Test plugin
|
|
||||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
|
||||||
|
|
||||||
- name: Publish to npm
|
|
||||||
working-directory: ${{ env.PACKAGE_DIR }}
|
|
||||||
env:
|
|
||||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
|
||||||
echo "::error::Environment secret NPM_TOKEN is not configured"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
pnpm publish --access public --no-git-checks
|
|
||||||
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
|
||||||
@@ -325,180 +325,3 @@ jobs:
|
|||||||
|
|
||||||
echo "Chrome Web Store submission created successfully:"
|
echo "Chrome Web Store submission created successfully:"
|
||||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||||
|
|
||||||
publish-edge-addons:
|
|
||||||
needs: [resolve, build-extension]
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
environment: edge-addons
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
steps:
|
|
||||||
- name: Download build artifact
|
|
||||||
uses: actions/download-artifact@v8
|
|
||||||
with:
|
|
||||||
name: extension-zip
|
|
||||||
path: dist
|
|
||||||
|
|
||||||
- name: Validate Microsoft Edge Add-ons configuration
|
|
||||||
shell: bash
|
|
||||||
env:
|
|
||||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
|
||||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
|
||||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
if [ -z "$API_KEY" ]; then
|
|
||||||
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ -z "$CLIENT_ID" ]; then
|
|
||||||
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ -z "$PRODUCT_ID" ]; then
|
|
||||||
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
|
||||||
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
|
||||||
if [ -z "$ARCHIVE" ]; then
|
|
||||||
echo "::error::No extension zip found in the downloaded artifact"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Upload and publish Microsoft Edge Add-ons extension
|
|
||||||
shell: bash
|
|
||||||
env:
|
|
||||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
|
||||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
|
||||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
|
||||||
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
|
||||||
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
|
||||||
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
|
||||||
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
|
||||||
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
|
||||||
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
|
||||||
|
|
||||||
operation_id_from_headers() {
|
|
||||||
local headers="$1"
|
|
||||||
local location
|
|
||||||
location="$(awk '
|
|
||||||
tolower($1) == "location:" {
|
|
||||||
sub(/\r$/, "")
|
|
||||||
sub(/^[^:]*:[[:space:]]*/, "")
|
|
||||||
print
|
|
||||||
exit
|
|
||||||
}
|
|
||||||
' "$headers")"
|
|
||||||
location="${location%%\?*}"
|
|
||||||
printf '%s' "${location##*/}"
|
|
||||||
}
|
|
||||||
|
|
||||||
poll_operation() {
|
|
||||||
local operation_name="$1"
|
|
||||||
local status_url="$2"
|
|
||||||
local status="InProgress"
|
|
||||||
local http_status
|
|
||||||
|
|
||||||
for attempt in $(seq 1 30); do
|
|
||||||
sleep 5
|
|
||||||
http_status="$(curl --silent --show-error \
|
|
||||||
--output "$STATUS_RESPONSE" \
|
|
||||||
--write-out '%{http_code}' \
|
|
||||||
--header "Authorization: ApiKey $API_KEY" \
|
|
||||||
--header "X-ClientID: $CLIENT_ID" \
|
|
||||||
"$status_url")"
|
|
||||||
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
|
||||||
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
|
||||||
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
|
||||||
echo "$operation_name status check $attempt/30: $status"
|
|
||||||
case "$status" in
|
|
||||||
Succeeded)
|
|
||||||
return 0
|
|
||||||
;;
|
|
||||||
InProgress)
|
|
||||||
;;
|
|
||||||
Failed)
|
|
||||||
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
|
||||||
jq . "$STATUS_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
|
||||||
jq . "$STATUS_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
|
||||||
jq . "$STATUS_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
HTTP_STATUS="$(curl --silent --show-error \
|
|
||||||
--output "$UPLOAD_RESPONSE" \
|
|
||||||
--dump-header "$UPLOAD_HEADERS" \
|
|
||||||
--write-out '%{http_code}' \
|
|
||||||
--request POST \
|
|
||||||
--header "Authorization: ApiKey $API_KEY" \
|
|
||||||
--header "X-ClientID: $CLIENT_ID" \
|
|
||||||
--header "Content-Type: application/zip" \
|
|
||||||
--data-binary "@$ARCHIVE" \
|
|
||||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
|
||||||
if [ "$HTTP_STATUS" != "202" ]; then
|
|
||||||
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
|
||||||
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
|
||||||
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
|
||||||
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
|
||||||
cat "$UPLOAD_HEADERS"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
poll_operation \
|
|
||||||
"package upload" \
|
|
||||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
|
||||||
|
|
||||||
HTTP_STATUS="$(curl --silent --show-error \
|
|
||||||
--output "$PUBLISH_RESPONSE" \
|
|
||||||
--dump-header "$PUBLISH_HEADERS" \
|
|
||||||
--write-out '%{http_code}' \
|
|
||||||
--request POST \
|
|
||||||
--header "Authorization: ApiKey $API_KEY" \
|
|
||||||
--header "X-ClientID: $CLIENT_ID" \
|
|
||||||
--header "Content-Type: application/json" \
|
|
||||||
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
|
||||||
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
|
||||||
if [ "$HTTP_STATUS" != "202" ]; then
|
|
||||||
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
|
||||||
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
|
||||||
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
|
||||||
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
|
||||||
cat "$PUBLISH_HEADERS"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
poll_operation \
|
|
||||||
"submission" \
|
|
||||||
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
|
||||||
|
|
||||||
echo "Microsoft Edge Add-ons submission created successfully:"
|
|
||||||
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
|
||||||
|
|||||||
@@ -11,10 +11,6 @@ dist/
|
|||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
|
|
||||||
# Local browser evaluation artifacts
|
|
||||||
evals/browser/results/
|
|
||||||
evals/browser/agents.local.json
|
|
||||||
|
|
||||||
# macOS
|
# macOS
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,3 @@
|
|||||||
**/.output/**
|
**/.output/**
|
||||||
**/target/**
|
**/target/**
|
||||||
**/node_modules/**
|
**/node_modules/**
|
||||||
# Generated client CSS (tailwind compile output); built by
|
|
||||||
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
|
||||||
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
|
||||||
|
|||||||
+5
-9
@@ -34,17 +34,13 @@ where only `extension connected` fails is expected; go to Step 3.
|
|||||||
## 3. Open the extension install page
|
## 3. Open the extension install page
|
||||||
|
|
||||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||||
has not installed or enabled the browser extension yet. Open the install page
|
has not installed or enabled the browser extension yet. Open the
|
||||||
matching the user's browser —
|
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||||
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
in the user's browser, then tell the user:
|
||||||
for Chrome and other Chromium browsers,
|
|
||||||
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
|
||||||
for Microsoft Edge — then tell the user:
|
|
||||||
|
|
||||||
> Install **BrowserSkill** from the
|
> Install **BrowserSkill** from the
|
||||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
||||||
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
> open the popup, and wait until it turns green. Reply when done.
|
||||||
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
|
||||||
|
|
||||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||||
once more. All `ok`/`na` → tell the user it's ready.
|
once more. All `ok`/`na` → tell the user it's ready.
|
||||||
|
|||||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk"
|
name = "bsk"
|
||||||
version = "0.1.11"
|
version = "0.1.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"bsk-protocol",
|
"bsk-protocol",
|
||||||
@@ -168,7 +168,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bsk-protocol"
|
name = "bsk-protocol"
|
||||||
version = "0.1.11"
|
version = "0.1.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"semver",
|
"semver",
|
||||||
|
|||||||
+1
-1
@@ -7,7 +7,7 @@ members = [
|
|||||||
default-members = ["crates/bsk-cli"]
|
default-members = ["crates/bsk-cli"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.1.11"
|
version = "0.1.10"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||||
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
||||||
browser.
|
browser.
|
||||||
|
|
||||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||||
@@ -67,8 +67,7 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
|
||||||
|
|
||||||
#### 1. Install the `bsk` CLI
|
#### 1. Install the `bsk` CLI
|
||||||
|
|
||||||
@@ -92,14 +91,7 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. Install the browser extension
|
#### 2. Install the browser extension
|
||||||
|
|
||||||
Install BrowserSkill from your browser's store:
|
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||||
|
|
||||||
| Browser | Store listing |
|
|
||||||
| --- | --- |
|
|
||||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
|
||||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
|
||||||
|
|
||||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
|
||||||
|
|
||||||
#### 3. Install the skill
|
#### 3. Install the skill
|
||||||
|
|
||||||
@@ -131,8 +123,7 @@ internal variants and install paths.
|
|||||||
|
|
||||||
Other shell-capable agent harnesses are supported too. Copy
|
Other shell-capable agent harnesses are supported too. Copy
|
||||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||||
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
`browser-skill/SKILL.md` to install the skill manually.
|
||||||
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -142,26 +133,6 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
## DeepSeek Harness plugin
|
|
||||||
|
|
||||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
|
||||||
BrowserSkill ships a first-class dsh plugin on npm as
|
|
||||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
|
||||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
|
||||||
overlay of each Agent Window.
|
|
||||||
|
|
||||||
Add it to a dsh profile, then start that profile:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
|
||||||
dsh --profile web
|
|
||||||
```
|
|
||||||
|
|
||||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
|
||||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
|
||||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
|
||||||
configuration, and the observation overlay.
|
|
||||||
|
|
||||||
## How It Works
|
## How It Works
|
||||||
|
|
||||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||||
@@ -195,9 +166,7 @@ flowchart TB
|
|||||||
|
|
||||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||||
browser task; the local daemon routes that request to the extension; the
|
browser task; the local daemon routes that request to the extension; the
|
||||||
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
extension runs it in an Agent Window.
|
||||||
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
|
||||||
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
|
||||||
|
|
||||||
## For Developers
|
## For Developers
|
||||||
|
|
||||||
@@ -207,8 +176,6 @@ The repository is a Cargo + pnpm workspace:
|
|||||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||||
- `apps/extension` — browser extension
|
- `apps/extension` — browser extension
|
||||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
|
||||||
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+7
-33
@@ -12,7 +12,7 @@
|
|||||||
<a href="README.md">English</a> · 中文
|
<a href="README.md">English</a> · 中文
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
||||||
|
|
||||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||||
|
|
||||||
@@ -54,8 +54,7 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
|
|
||||||
<br>
|
<br>
|
||||||
|
|
||||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
||||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
|
||||||
|
|
||||||
#### 1. 安装 `bsk` CLI
|
#### 1. 安装 `bsk` CLI
|
||||||
|
|
||||||
@@ -65,11 +64,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
|||||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
||||||
|
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
||||||
```powershell
|
|
||||||
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
|
||||||
```
|
|
||||||
|
|
||||||
验证二进制:
|
验证二进制:
|
||||||
|
|
||||||
@@ -79,14 +75,7 @@ bsk --version
|
|||||||
|
|
||||||
#### 2. 安装浏览器扩展
|
#### 2. 安装浏览器扩展
|
||||||
|
|
||||||
在对应浏览器的商店安装 BrowserSkill:
|
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
||||||
|
|
||||||
| 浏览器 | 商店页面 |
|
|
||||||
| --- | --- |
|
|
||||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
|
||||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
|
||||||
|
|
||||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
|
||||||
|
|
||||||
#### 3. 安装 skill
|
#### 3. 安装 skill
|
||||||
|
|
||||||
@@ -113,7 +102,7 @@ bsk install-skill
|
|||||||
|
|
||||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||||
|
|
||||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
@@ -123,19 +112,6 @@ bsk install-skill
|
|||||||
/browser-skill open example.com and summarize what is on the page.
|
/browser-skill open example.com and summarize what is on the page.
|
||||||
```
|
```
|
||||||
|
|
||||||
## DeepSeek Harness 插件
|
|
||||||
|
|
||||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
|
||||||
|
|
||||||
把它装进某个 dsh profile,然后启动该 profile:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
|
||||||
dsh --profile web
|
|
||||||
```
|
|
||||||
|
|
||||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
|
||||||
|
|
||||||
## 工作原理
|
## 工作原理
|
||||||
|
|
||||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||||
@@ -167,7 +143,7 @@ flowchart TB
|
|||||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||||
```
|
```
|
||||||
|
|
||||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
||||||
|
|
||||||
## 面向开发者
|
## 面向开发者
|
||||||
|
|
||||||
@@ -177,8 +153,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
|||||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||||
- `apps/extension` — 浏览器扩展
|
- `apps/extension` — 浏览器扩展
|
||||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
|
||||||
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
|
||||||
|
|
||||||
## 许可证
|
## 许可证
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@browser-skill/extension",
|
"name": "@browser-skill/extension",
|
||||||
"version": "0.1.7",
|
"version": "0.1.6",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||||
|
|
||||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||||
const listeners = new Set<(...args: TArgs) => void>();
|
const listeners = new Set<(...args: TArgs) => void>();
|
||||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeApi() {
|
function fakeApi() {
|
||||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||||
const api: CdpDebuggerApi = {
|
const api: CdpDebuggerApi = {
|
||||||
attach: vi.fn(async () => {}),
|
attach: vi.fn(async () => {}),
|
||||||
@@ -29,135 +29,6 @@ function fakeApi() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("ChromiumCdp", () => {
|
describe("ChromiumCdp", () => {
|
||||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
|
||||||
const { api, onEvent } = fakeApi();
|
|
||||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
|
||||||
async (
|
|
||||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
|
||||||
method: string,
|
|
||||||
params?: { frameId?: string },
|
|
||||||
) => {
|
|
||||||
if (method === "DOM.getFrameOwner") {
|
|
||||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
|
||||||
}
|
|
||||||
if (method !== "Page.getFrameTree") return {};
|
|
||||||
if (target.sessionId === "right-session") {
|
|
||||||
return {
|
|
||||||
frameTree: {
|
|
||||||
frame: { id: "right", url: "https://right.test" },
|
|
||||||
childFrames: [
|
|
||||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (target.sessionId === "nested-session") {
|
|
||||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
frameTree: {
|
|
||||||
frame: { id: "main", url: "https://app.test" },
|
|
||||||
childFrames: [
|
|
||||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
|
||||||
{
|
|
||||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
|
||||||
childFrames: [
|
|
||||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const cdp = new ChromiumCdp(api);
|
|
||||||
await cdp.ensureAttached(4);
|
|
||||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
|
||||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
|
||||||
sessionId: "nested-session",
|
|
||||||
});
|
|
||||||
|
|
||||||
const graph = await cdp.getFrameGraph(4);
|
|
||||||
|
|
||||||
expect(graph.frames).toHaveLength(4);
|
|
||||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
|
||||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
|
||||||
"right-session",
|
|
||||||
);
|
|
||||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
|
||||||
"nested-session",
|
|
||||||
);
|
|
||||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
|
||||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
|
||||||
{ tabId: 4, sessionId: "right-session" },
|
|
||||||
"DOM.getFrameOwner",
|
|
||||||
{ frameId: "nested" },
|
|
||||||
);
|
|
||||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
|
||||||
{ tabId: 4, sessionId: "nested-session" },
|
|
||||||
"Target.setAutoAttach",
|
|
||||||
expect.objectContaining({ flatten: true }),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
|
||||||
const { api, onEvent } = fakeApi();
|
|
||||||
const depth = 6;
|
|
||||||
const scheduled = new Set<string>();
|
|
||||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
|
||||||
frame: {
|
|
||||||
id: index === 0 ? "main" : `frame-${index}`,
|
|
||||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
|
||||||
},
|
|
||||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
|
||||||
});
|
|
||||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
|
||||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
|
||||||
if (method === "Target.setAutoAttach") {
|
|
||||||
const parentIndex = target.sessionId
|
|
||||||
? Number(target.sessionId.replace("session-", ""))
|
|
||||||
: 0;
|
|
||||||
const nextIndex = parentIndex + 1;
|
|
||||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
|
||||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
|
||||||
scheduled.add(key);
|
|
||||||
setTimeout(() => {
|
|
||||||
onEvent.fire(
|
|
||||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
|
||||||
"Target.attachedToTarget",
|
|
||||||
{
|
|
||||||
sessionId: `session-${nextIndex}`,
|
|
||||||
targetInfo: { type: "iframe" },
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}, 5);
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
if (method === "Page.getFrameTree") {
|
|
||||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
|
||||||
const index = Number(target.sessionId.replace("session-", ""));
|
|
||||||
return { frameTree: nestedTree(index) };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getFrameOwner") {
|
|
||||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const cdp = new ChromiumCdp(api);
|
|
||||||
|
|
||||||
const graph = await cdp.getFrameGraph(4);
|
|
||||||
|
|
||||||
expect(graph.frames).toHaveLength(depth + 1);
|
|
||||||
for (let index = 1; index <= depth; index += 1) {
|
|
||||||
expect(
|
|
||||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
|
||||||
).toBe(`session-${index}`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||||
const { api } = fakeApi();
|
const { api } = fakeApi();
|
||||||
let releaseAttach!: () => void;
|
let releaseAttach!: () => void;
|
||||||
|
|||||||
@@ -1,136 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
|
||||||
|
|
||||||
describe("buildFrameGraph", () => {
|
|
||||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
|
||||||
const graph = buildFrameGraph([
|
|
||||||
{
|
|
||||||
target: { tabId: 4 },
|
|
||||||
tree: {
|
|
||||||
frame: { id: "main", url: "https://app.test" },
|
|
||||||
childFrames: [
|
|
||||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
|
||||||
{
|
|
||||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
|
||||||
childFrames: [
|
|
||||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
target: { tabId: 4, sessionId: "right-session" },
|
|
||||||
tree: {
|
|
||||||
frame: { id: "right", url: "https://right.test" },
|
|
||||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
target: { tabId: 4, sessionId: "nested-session" },
|
|
||||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(graph?.frames).toHaveLength(4);
|
|
||||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
|
||||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
|
||||||
parentFrameId: "main",
|
|
||||||
target: { tabId: 4, sessionId: "right-session" },
|
|
||||||
});
|
|
||||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
|
||||||
parentFrameId: "right",
|
|
||||||
target: { tabId: 4, sessionId: "nested-session" },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("propagates each target boundary to its same-process descendants", () => {
|
|
||||||
const sources: CdpFrameTreeSource[] = [
|
|
||||||
{
|
|
||||||
target: { tabId: 4 },
|
|
||||||
tree: {
|
|
||||||
frame: { id: "main" },
|
|
||||||
childFrames: [
|
|
||||||
{
|
|
||||||
frame: { id: "oopif-a", parentId: "main" },
|
|
||||||
childFrames: [
|
|
||||||
{
|
|
||||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
|
||||||
childFrames: [
|
|
||||||
{
|
|
||||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
|
||||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
target: { tabId: 4, sessionId: "session-a" },
|
|
||||||
tree: {
|
|
||||||
frame: { id: "oopif-a" },
|
|
||||||
childFrames: [
|
|
||||||
{
|
|
||||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
|
||||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
target: { tabId: 4, sessionId: "session-c" },
|
|
||||||
tree: {
|
|
||||||
frame: { id: "oopif-c" },
|
|
||||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const graph = buildFrameGraph(sources);
|
|
||||||
|
|
||||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
|
||||||
["main", undefined],
|
|
||||||
["oopif-a", "session-a"],
|
|
||||||
["same-process-b", "session-a"],
|
|
||||||
["oopif-c", "session-c"],
|
|
||||||
["same-process-d", "session-c"],
|
|
||||||
]);
|
|
||||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
|
||||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
|
||||||
"same-process-b",
|
|
||||||
);
|
|
||||||
|
|
||||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
|
||||||
expect(reordered).toEqual(graph);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
|
||||||
const depth = 5_000;
|
|
||||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
|
||||||
let parent = root;
|
|
||||||
for (let index = 1; index <= depth; index += 1) {
|
|
||||||
const child: CdpFrameTreeNode = {
|
|
||||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
|
||||||
};
|
|
||||||
parent.childFrames = [child];
|
|
||||||
parent = child;
|
|
||||||
}
|
|
||||||
|
|
||||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
|
||||||
|
|
||||||
expect(graph?.frames).toHaveLength(depth + 1);
|
|
||||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
|
||||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
|
||||||
root.childFrames = [root];
|
|
||||||
|
|
||||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -31,15 +31,6 @@ import type {
|
|||||||
NetworkEntryKind,
|
NetworkEntryKind,
|
||||||
NetworkResult,
|
NetworkResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import {
|
|
||||||
buildFrameGraph,
|
|
||||||
type CdpFrameGraph,
|
|
||||||
type CdpFrameTreeNode,
|
|
||||||
type CdpFrameTreeSource,
|
|
||||||
type CdpTarget,
|
|
||||||
} from "./frame-graph";
|
|
||||||
|
|
||||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||||
@@ -47,15 +38,21 @@ export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
|||||||
* fake without monkey-patching the real `chrome` global.
|
* fake without monkey-patching the real `chrome` global.
|
||||||
*/
|
*/
|
||||||
export interface CdpDebuggerApi {
|
export interface CdpDebuggerApi {
|
||||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||||
detach(target: CdpDebuggee): Promise<void>;
|
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
sendCommand(
|
||||||
|
target: chrome.debugger.Debuggee,
|
||||||
|
method: string,
|
||||||
|
commandParams?: object,
|
||||||
|
): Promise<unknown>;
|
||||||
/**
|
/**
|
||||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||||
* …). The first callback argument is the source debuggee; the second
|
* …). The first callback argument is the source debuggee; the second
|
||||||
* is the CDP method name; the third is the payload.
|
* is the CDP method name; the third is the payload.
|
||||||
*/
|
*/
|
||||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
onEvent: chrome.events.Event<
|
||||||
|
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||||
|
>;
|
||||||
/**
|
/**
|
||||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||||
* the tab navigated to a chrome:// URL or the user clicked
|
* the tab navigated to a chrome:// URL or the user clicked
|
||||||
@@ -113,8 +110,6 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
|||||||
const MAX_NETWORK_BUFFER = 200;
|
const MAX_NETWORK_BUFFER = 200;
|
||||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||||
const MAX_NETWORK_REQUEST_META = 1024;
|
const MAX_NETWORK_REQUEST_META = 1024;
|
||||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
|
||||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
|
||||||
|
|
||||||
interface ParsedDialogOpening {
|
interface ParsedDialogOpening {
|
||||||
type: JavaScriptDialogType;
|
type: JavaScriptDialogType;
|
||||||
@@ -136,27 +131,6 @@ interface NetworkRequestMeta {
|
|||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FrameDiscoveryState {
|
|
||||||
sessions: Set<string>;
|
|
||||||
pending: Set<Promise<void>>;
|
|
||||||
generation: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
|
||||||
if (promises.length === 0) return true;
|
|
||||||
const remaining = deadline - Date.now();
|
|
||||||
if (remaining <= 0) return false;
|
|
||||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
||||||
const timedOut = await Promise.race([
|
|
||||||
Promise.allSettled(promises).then(() => false),
|
|
||||||
new Promise<boolean>((resolve) => {
|
|
||||||
timer = setTimeout(() => resolve(true), remaining);
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
if (timer) clearTimeout(timer);
|
|
||||||
return !timedOut;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||||
* tabId" cache and exposes typed `send<T>()`.
|
* tabId" cache and exposes typed `send<T>()`.
|
||||||
@@ -175,12 +149,10 @@ export class ChromiumCdp {
|
|||||||
private readonly networkSequences = new Map<number, number>();
|
private readonly networkSequences = new Map<number, number>();
|
||||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
|
||||||
private detachSubscription: { dispose(): void } | null = null;
|
private detachSubscription: { dispose(): void } | null = null;
|
||||||
private dialogSubscription: { dispose(): void } | null = null;
|
private dialogSubscription: { dispose(): void } | null = null;
|
||||||
private consoleSubscription: { dispose(): void } | null = null;
|
private consoleSubscription: { dispose(): void } | null = null;
|
||||||
private networkSubscription: { dispose(): void } | null = null;
|
private networkSubscription: { dispose(): void } | null = null;
|
||||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
|
||||||
|
|
||||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||||
this.api = api;
|
this.api = api;
|
||||||
@@ -188,7 +160,6 @@ export class ChromiumCdp {
|
|||||||
this.bindDialogHandler();
|
this.bindDialogHandler();
|
||||||
this.bindConsoleHandler();
|
this.bindConsoleHandler();
|
||||||
this.bindNetworkHandler();
|
this.bindNetworkHandler();
|
||||||
this.bindFrameTargetHandler();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Attach to `tabId` if we haven't already in this driver. */
|
/** Attach to `tabId` if we haven't already in this driver. */
|
||||||
@@ -206,9 +177,6 @@ export class ChromiumCdp {
|
|||||||
await this.enableConsoleDomains(tabId);
|
await this.enableConsoleDomains(tabId);
|
||||||
await this.enableNetworkDomainBestEffort(tabId);
|
await this.enableNetworkDomainBestEffort(tabId);
|
||||||
this.attachedTabs.add(tabId);
|
this.attachedTabs.add(tabId);
|
||||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
|
||||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// A CDP domain enable failed after the raw attach succeeded
|
// A CDP domain enable failed after the raw attach succeeded
|
||||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||||
@@ -254,73 +222,6 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
|
||||||
if (!this.attachedTabs.has(target.tabId)) {
|
|
||||||
await this.ensureAttached(target.tabId);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
|
||||||
} catch (err) {
|
|
||||||
throw normalizeError(err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
|
||||||
await this.ensureAttached(tabId);
|
|
||||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
|
||||||
await this.drainFrameAttachTasks(tabId);
|
|
||||||
|
|
||||||
const sources: CdpFrameTreeSource[] = [];
|
|
||||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
|
||||||
{ tabId },
|
|
||||||
"Page.getFrameTree",
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
|
||||||
|
|
||||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
|
||||||
const childTrees = await Promise.all(
|
|
||||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
|
||||||
const target = { tabId, sessionId };
|
|
||||||
try {
|
|
||||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
|
||||||
target,
|
|
||||||
"Page.getFrameTree",
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
for (const source of childTrees) {
|
|
||||||
if (source) sources.push(source);
|
|
||||||
}
|
|
||||||
|
|
||||||
const graph = buildFrameGraph(sources);
|
|
||||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
|
||||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
|
||||||
await Promise.all(
|
|
||||||
graph.frames.map(async (frame) => {
|
|
||||||
if (!frame.parentFrameId) return;
|
|
||||||
const parent = frameById.get(frame.parentFrameId);
|
|
||||||
if (!parent) return;
|
|
||||||
try {
|
|
||||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
|
||||||
parent.target,
|
|
||||||
"DOM.getFrameOwner",
|
|
||||||
{ frameId: frame.frameId },
|
|
||||||
);
|
|
||||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
|
||||||
} catch {
|
|
||||||
// The frame may have navigated between tree capture and owner lookup.
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return graph;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||||
dialogCursor(tabId: number): DialogCursor {
|
dialogCursor(tabId: number): DialogCursor {
|
||||||
return this.dialogSequences.get(tabId) ?? 0;
|
return this.dialogSequences.get(tabId) ?? 0;
|
||||||
@@ -431,7 +332,6 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(tabId);
|
this.clearDialogState(tabId);
|
||||||
this.clearConsoleState(tabId);
|
this.clearConsoleState(tabId);
|
||||||
this.clearNetworkState(tabId);
|
this.clearNetworkState(tabId);
|
||||||
this.clearFrameState(tabId);
|
|
||||||
try {
|
try {
|
||||||
await this.api.detach({ tabId });
|
await this.api.detach({ tabId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -454,7 +354,7 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
} {
|
} {
|
||||||
this.api.onEvent.addListener(handler);
|
this.api.onEvent.addListener(handler);
|
||||||
@@ -478,7 +378,6 @@ export class ChromiumCdp {
|
|||||||
this.networkSequences.clear();
|
this.networkSequences.clear();
|
||||||
this.networkDomainsEnabledTabs.clear();
|
this.networkDomainsEnabledTabs.clear();
|
||||||
this.networkRequestMeta.clear();
|
this.networkRequestMeta.clear();
|
||||||
this.frameDiscovery.clear();
|
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
tabs.map(async (tabId) => {
|
tabs.map(async (tabId) => {
|
||||||
try {
|
try {
|
||||||
@@ -529,104 +428,9 @@ export class ChromiumCdp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
|
||||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
|
||||||
autoAttach: true,
|
|
||||||
waitForDebuggerOnStart: false,
|
|
||||||
flatten: true,
|
|
||||||
filter: [{ type: "iframe", exclude: false }],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private bindFrameTargetHandler(): void {
|
|
||||||
if (this.frameTargetSubscription) return;
|
|
||||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
|
||||||
const tabId = source.tabId;
|
|
||||||
if (typeof tabId !== "number") return;
|
|
||||||
const raw = (params ?? {}) as Record<string, unknown>;
|
|
||||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
|
||||||
if (!sessionId) return;
|
|
||||||
|
|
||||||
if (method === "Target.detachedFromTarget") {
|
|
||||||
const state = this.frameDiscovery.get(tabId);
|
|
||||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (method !== "Target.attachedToTarget") return;
|
|
||||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
|
||||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
|
||||||
|
|
||||||
const state = this.frameDiscoveryState(tabId);
|
|
||||||
if (state.sessions.has(sessionId)) return;
|
|
||||||
state.sessions.add(sessionId);
|
|
||||||
state.generation += 1;
|
|
||||||
|
|
||||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
|
||||||
state.pending.add(task);
|
|
||||||
void task.finally(() => {
|
|
||||||
state.pending.delete(task);
|
|
||||||
});
|
|
||||||
};
|
|
||||||
this.api.onEvent.addListener(listener);
|
|
||||||
this.frameTargetSubscription = {
|
|
||||||
dispose: () => this.api.onEvent.removeListener(listener),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
|
||||||
try {
|
|
||||||
await this.enableFrameDiscovery(target);
|
|
||||||
} catch (err) {
|
|
||||||
const state = this.frameDiscovery.get(target.tabId);
|
|
||||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
|
||||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
|
||||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
|
||||||
while (Date.now() < deadline) {
|
|
||||||
const state = this.frameDiscovery.get(tabId);
|
|
||||||
const generation = state?.generation ?? 0;
|
|
||||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
|
||||||
|
|
||||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
|
||||||
// its command promise settles. Wait for a short quiet window, then finish
|
|
||||||
// only if neither the state object nor its generation changed.
|
|
||||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
|
||||||
if (quietTime <= 0) break;
|
|
||||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
|
||||||
const current = this.frameDiscovery.get(tabId);
|
|
||||||
if (
|
|
||||||
current === state &&
|
|
||||||
(current?.generation ?? 0) === generation &&
|
|
||||||
!current?.pending.size
|
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
|
||||||
}
|
|
||||||
|
|
||||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
|
||||||
const existing = this.frameDiscovery.get(tabId);
|
|
||||||
if (existing) return existing;
|
|
||||||
const created: FrameDiscoveryState = {
|
|
||||||
sessions: new Set(),
|
|
||||||
pending: new Set(),
|
|
||||||
generation: 0,
|
|
||||||
};
|
|
||||||
this.frameDiscovery.set(tabId, created);
|
|
||||||
return created;
|
|
||||||
}
|
|
||||||
|
|
||||||
private clearFrameState(tabId: number): void {
|
|
||||||
this.frameDiscovery.delete(tabId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private bindDialogHandler(): void {
|
private bindDialogHandler(): void {
|
||||||
if (this.dialogSubscription) return;
|
if (this.dialogSubscription) return;
|
||||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||||
if (method !== "Page.javascriptDialogOpening") return;
|
if (method !== "Page.javascriptDialogOpening") return;
|
||||||
const tabId = source.tabId;
|
const tabId = source.tabId;
|
||||||
if (typeof tabId !== "number") return;
|
if (typeof tabId !== "number") return;
|
||||||
@@ -810,7 +614,6 @@ export class ChromiumCdp {
|
|||||||
this.clearDialogState(source.tabId);
|
this.clearDialogState(source.tabId);
|
||||||
this.clearConsoleState(source.tabId);
|
this.clearConsoleState(source.tabId);
|
||||||
this.clearNetworkState(source.tabId);
|
this.clearNetworkState(source.tabId);
|
||||||
this.clearFrameState(source.tabId);
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
this.api.onDetach.addListener(listener);
|
this.api.onDetach.addListener(listener);
|
||||||
@@ -829,8 +632,6 @@ export class ChromiumCdp {
|
|||||||
this.consoleSubscription = null;
|
this.consoleSubscription = null;
|
||||||
this.networkSubscription?.dispose();
|
this.networkSubscription?.dispose();
|
||||||
this.networkSubscription = null;
|
this.networkSubscription = null;
|
||||||
this.frameTargetSubscription?.dispose();
|
|
||||||
this.frameTargetSubscription = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Detach tabs only when no other live session has claimed them. */
|
/** Detach tabs only when no other live session has claimed them. */
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
export interface CdpTarget {
|
|
||||||
tabId: number;
|
|
||||||
sessionId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CdpFrame {
|
|
||||||
frameId: string;
|
|
||||||
parentFrameId?: string;
|
|
||||||
ownerBackendNodeId?: number;
|
|
||||||
url?: string;
|
|
||||||
target: CdpTarget;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CdpFrameGraph {
|
|
||||||
rootFrameId: string;
|
|
||||||
frames: CdpFrame[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CdpFrameTreeNode {
|
|
||||||
frame: {
|
|
||||||
id: string;
|
|
||||||
parentId?: string;
|
|
||||||
url?: string;
|
|
||||||
};
|
|
||||||
childFrames?: CdpFrameTreeNode[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CdpFrameTreeSource {
|
|
||||||
target: CdpTarget;
|
|
||||||
tree: CdpFrameTreeNode;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function cdpTargetKey(target: CdpTarget): string {
|
|
||||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function mergeFrame(
|
|
||||||
frames: Map<string, CdpFrame>,
|
|
||||||
order: string[],
|
|
||||||
node: CdpFrameTreeNode,
|
|
||||||
target: CdpTarget,
|
|
||||||
): void {
|
|
||||||
const existing = frames.get(node.frame.id);
|
|
||||||
if (!existing) order.push(node.frame.id);
|
|
||||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
|
||||||
const url = node.frame.url || existing?.url;
|
|
||||||
frames.set(node.frame.id, {
|
|
||||||
frameId: node.frame.id,
|
|
||||||
...(parentFrameId ? { parentFrameId } : {}),
|
|
||||||
...(url ? { url } : {}),
|
|
||||||
target,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function walkFrameTree(
|
|
||||||
source: CdpFrameTreeSource,
|
|
||||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
|
||||||
frames: Map<string, CdpFrame>,
|
|
||||||
order: string[],
|
|
||||||
): void {
|
|
||||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
|
||||||
{ node: source.tree, inheritedTarget: source.target },
|
|
||||||
];
|
|
||||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
|
||||||
|
|
||||||
while (stack.length > 0) {
|
|
||||||
const current = stack.pop();
|
|
||||||
if (!current || expanded.has(current.node)) continue;
|
|
||||||
expanded.add(current.node);
|
|
||||||
|
|
||||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
|
||||||
mergeFrame(frames, order, current.node, target);
|
|
||||||
|
|
||||||
const children = current.node.childFrames ?? [];
|
|
||||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
|
||||||
stack.push({ node: children[index], inheritedTarget: target });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
|
||||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
|
||||||
if (!rootSource) return null;
|
|
||||||
|
|
||||||
const frames = new Map<string, CdpFrame>();
|
|
||||||
const order: string[] = [];
|
|
||||||
const targetByRootFrameId = new Map(
|
|
||||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
|
||||||
);
|
|
||||||
// The top-level source is authoritative if malformed input reports the same
|
|
||||||
// frame as both a root target and a child target.
|
|
||||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
|
||||||
|
|
||||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
|
||||||
for (const source of sources) {
|
|
||||||
if (source === rootSource) continue;
|
|
||||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
rootFrameId: rootSource.tree.frame.id,
|
|
||||||
frames: order.flatMap((frameId) => {
|
|
||||||
const frame = frames.get(frameId);
|
|
||||||
return frame ? [frame] : [];
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import {
|
|||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
|
||||||
import logoUrl from "../../assets/logo.png";
|
import logoUrl from "../../assets/logo.png";
|
||||||
|
|
||||||
export interface HelpRequestData {
|
export interface HelpRequestData {
|
||||||
@@ -20,10 +19,6 @@ export interface HelpRequestData {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight. */
|
/** CSS selectors to scroll to + flash-highlight. */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
|
||||||
rects?: HelpHighlightRect[];
|
|
||||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
|
||||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
|
||||||
onContinue: (note: string) => void;
|
onContinue: (note: string) => void;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
}
|
}
|
||||||
@@ -54,7 +49,6 @@ const PANEL_WIDTH = 420;
|
|||||||
const FALLBACK_PANEL_H = 180;
|
const FALLBACK_PANEL_H = 180;
|
||||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||||
const EMPTY_SELECTORS: string[] = [];
|
const EMPTY_SELECTORS: string[] = [];
|
||||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
|
||||||
|
|
||||||
function clamp(value: number, min: number, max: number): number {
|
function clamp(value: number, min: number, max: number): number {
|
||||||
return Math.max(min, Math.min(max, value));
|
return Math.max(min, Math.min(max, value));
|
||||||
@@ -185,8 +179,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const isCompact = request?.displayMode === "compact";
|
const isCompact = request?.displayMode === "compact";
|
||||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
|
||||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
|
||||||
const [note, setNote] = useState("");
|
const [note, setNote] = useState("");
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||||
@@ -209,7 +201,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
// Reset the note and collapse state whenever a new request appears.
|
// Reset the note and collapse state whenever a new request appears.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setNote("");
|
setNote("");
|
||||||
setLiveRects(null);
|
|
||||||
setCollapsed(false);
|
setCollapsed(false);
|
||||||
placementRef.current = null;
|
placementRef.current = null;
|
||||||
setDragPos(null);
|
setDragPos(null);
|
||||||
@@ -218,32 +209,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
dragStartRef.current = null;
|
dragStartRef.current = null;
|
||||||
}, [request?.id]);
|
}, [request?.id]);
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
if (!request || isCompact || !request.refreshRects) return;
|
|
||||||
let disposed = false;
|
|
||||||
let resolving = false;
|
|
||||||
const refresh = async () => {
|
|
||||||
if (resolving) return;
|
|
||||||
resolving = true;
|
|
||||||
try {
|
|
||||||
const rects = await request.refreshRects?.();
|
|
||||||
if (!disposed && rects) setLiveRects(rects);
|
|
||||||
} finally {
|
|
||||||
resolving = false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
const onViewportChange = () => void refresh();
|
|
||||||
window.addEventListener("scroll", onViewportChange, true);
|
|
||||||
window.addEventListener("resize", onViewportChange);
|
|
||||||
const interval = window.setInterval(onViewportChange, 500);
|
|
||||||
return () => {
|
|
||||||
disposed = true;
|
|
||||||
window.removeEventListener("scroll", onViewportChange, true);
|
|
||||||
window.removeEventListener("resize", onViewportChange);
|
|
||||||
window.clearInterval(interval);
|
|
||||||
};
|
|
||||||
}, [request, isCompact]);
|
|
||||||
|
|
||||||
// Scroll the first matched target into view once per distinct request id.
|
// Scroll the first matched target into view once per distinct request id.
|
||||||
useLayoutEffect(() => {
|
useLayoutEffect(() => {
|
||||||
if (!request) return;
|
if (!request) return;
|
||||||
@@ -266,7 +231,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const update = () => {
|
const update = () => {
|
||||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
const nextBoxes = measure(effectiveSelectors);
|
||||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||||
};
|
};
|
||||||
update();
|
update();
|
||||||
@@ -278,7 +243,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
window.removeEventListener("resize", update);
|
window.removeEventListener("resize", update);
|
||||||
window.clearInterval(interval);
|
window.clearInterval(interval);
|
||||||
};
|
};
|
||||||
}, [request, effectiveSelectors, effectiveRects]);
|
}, [request, effectiveSelectors]);
|
||||||
|
|
||||||
const onHeaderPointerDown = useCallback(
|
const onHeaderPointerDown = useCallback(
|
||||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||||
@@ -340,7 +305,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (userMovedRef.current) return;
|
if (userMovedRef.current) return;
|
||||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||||
const anchor = unionBox(measured);
|
const anchor = unionBox(measured);
|
||||||
const banner = bannerRef.current;
|
const banner = bannerRef.current;
|
||||||
if (!anchor || !banner) {
|
if (!anchor || !banner) {
|
||||||
@@ -364,7 +329,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
|||||||
}
|
}
|
||||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||||
|
|
||||||
if (!request) return null;
|
if (!request) return null;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||||
import { createElement } from "react";
|
import { createElement } from "react";
|
||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||||
@@ -38,36 +38,6 @@ describe("HelpRequestOverlay", () => {
|
|||||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
|
||||||
const { container } = renderOverlay(
|
|
||||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
|
||||||
);
|
|
||||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
|
||||||
expect(highlight?.style.top).toBe("30px");
|
|
||||||
expect(highlight?.style.left).toBe("40px");
|
|
||||||
expect(highlight?.style.width).toBe("120px");
|
|
||||||
expect(highlight?.style.height).toBe("50px");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
|
||||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
|
||||||
const { container } = renderOverlay(
|
|
||||||
baseRequest({
|
|
||||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
|
||||||
refreshRects,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
window.dispatchEvent(new Event("scroll"));
|
|
||||||
|
|
||||||
await waitFor(() => {
|
|
||||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
|
||||||
expect(highlight?.style.top).toBe("80px");
|
|
||||||
expect(highlight?.style.left).toBe("90px");
|
|
||||||
});
|
|
||||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps the inactive render stable", () => {
|
it("keeps the inactive render stable", () => {
|
||||||
const { container, rerender } = renderOverlay(null);
|
const { container, rerender } = renderOverlay(null);
|
||||||
|
|
||||||
|
|||||||
@@ -1,64 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import { createHelpRequestData } from "../help-request";
|
|
||||||
|
|
||||||
describe("createHelpRequestData", () => {
|
|
||||||
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
|
||||||
const finish = vi.fn();
|
|
||||||
const query = vi.fn(async () => ({
|
|
||||||
active: true,
|
|
||||||
request: {
|
|
||||||
requestId: "help-1",
|
|
||||||
prompt: "complete the challenge",
|
|
||||||
selectors: [],
|
|
||||||
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
|
||||||
timeoutMs: 1_000,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
const request = createHelpRequestData(
|
|
||||||
{
|
|
||||||
requestId: "help-1",
|
|
||||||
prompt: "complete the challenge",
|
|
||||||
selectors: [],
|
|
||||||
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
|
||||||
timeoutMs: 1_000,
|
|
||||||
},
|
|
||||||
{ finish, query },
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(request.refreshRects?.()).resolves.toEqual([
|
|
||||||
{ top: 80, left: 90, width: 140, height: 60 },
|
|
||||||
]);
|
|
||||||
request.onContinue(" done ");
|
|
||||||
request.onCancel();
|
|
||||||
|
|
||||||
expect(query).toHaveBeenCalledOnce();
|
|
||||||
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
|
||||||
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("ignores rectangle replies for a replaced request", async () => {
|
|
||||||
const request = createHelpRequestData(
|
|
||||||
{
|
|
||||||
requestId: "help-1",
|
|
||||||
prompt: "complete the challenge",
|
|
||||||
selectors: [],
|
|
||||||
timeoutMs: 1_000,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
finish: vi.fn(),
|
|
||||||
query: vi.fn(async () => ({
|
|
||||||
active: true,
|
|
||||||
request: {
|
|
||||||
requestId: "help-2",
|
|
||||||
prompt: "new challenge",
|
|
||||||
selectors: [],
|
|
||||||
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
|
||||||
timeoutMs: 1_000,
|
|
||||||
},
|
|
||||||
})),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||||
import { startRecordCapture } from "../record-capture";
|
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||||
|
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
runtime: {
|
runtime: {
|
||||||
sendMessage: vi.fn((message: { sequence?: number }) =>
|
sendMessage: vi.fn(() => Promise.resolve()),
|
||||||
Promise.resolve({ ok: true, sequence: message.sequence }),
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -48,6 +46,117 @@ function click(el: Element): void {
|
|||||||
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
describe("handleRecordContentMessage stop/cancel", () => {
|
||||||
|
it("ignores STOP when no recording is active", () => {
|
||||||
|
const dispose = vi.fn();
|
||||||
|
const onStop = vi.fn();
|
||||||
|
const setActiveRequestId = vi.fn();
|
||||||
|
const setCapture = vi.fn();
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
|
||||||
|
const needsAsync = handleRecordContentMessage(
|
||||||
|
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||||
|
{
|
||||||
|
activeRequestId: null,
|
||||||
|
capture: { dispose },
|
||||||
|
setActiveRequestId,
|
||||||
|
setCapture,
|
||||||
|
onStart: vi.fn(),
|
||||||
|
onStop,
|
||||||
|
},
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(false);
|
||||||
|
expect(dispose).not.toHaveBeenCalled();
|
||||||
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
|
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||||
|
expect(setCapture).not.toHaveBeenCalled();
|
||||||
|
expect(sendResponse).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores STOP for a mismatched requestId", () => {
|
||||||
|
const dispose = vi.fn();
|
||||||
|
const onStop = vi.fn();
|
||||||
|
|
||||||
|
const needsAsync = handleRecordContentMessage(
|
||||||
|
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||||
|
{
|
||||||
|
activeRequestId: "rec-1",
|
||||||
|
capture: { dispose },
|
||||||
|
setActiveRequestId: vi.fn(),
|
||||||
|
setCapture: vi.fn(),
|
||||||
|
onStart: vi.fn(),
|
||||||
|
onStop,
|
||||||
|
},
|
||||||
|
vi.fn(),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(false);
|
||||||
|
expect(dispose).not.toHaveBeenCalled();
|
||||||
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
|
||||||
|
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
|
||||||
|
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
|
||||||
|
sendMessage.mockReset();
|
||||||
|
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
|
||||||
|
sendMessage.mockResolvedValueOnce(undefined);
|
||||||
|
|
||||||
|
let activeRequestId: string | null = null;
|
||||||
|
let capture: ReturnType<typeof startRecordCapture> | null = null;
|
||||||
|
const onStop = vi.fn();
|
||||||
|
const dispatch = (
|
||||||
|
message:
|
||||||
|
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
|
||||||
|
| { type: typeof RECORD_STOP; requestId: string },
|
||||||
|
sendResponse?: (response: unknown) => void,
|
||||||
|
) =>
|
||||||
|
handleRecordContentMessage(
|
||||||
|
message,
|
||||||
|
{
|
||||||
|
activeRequestId,
|
||||||
|
capture,
|
||||||
|
setActiveRequestId: (id) => {
|
||||||
|
activeRequestId = id;
|
||||||
|
},
|
||||||
|
setCapture: (next) => {
|
||||||
|
capture = next;
|
||||||
|
},
|
||||||
|
onStart: vi.fn(),
|
||||||
|
onStop,
|
||||||
|
},
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
dispatch({ type: RECORD_START, requestId: "rec-retry" });
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "dirty final value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
|
||||||
|
const firstResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(firstResponse).toHaveBeenCalledWith({
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(activeRequestId).toBe("rec-retry");
|
||||||
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const retryResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
|
||||||
|
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
expect(activeRequestId).toBeNull();
|
||||||
|
expect(onStop).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("record-capture semantic", () => {
|
describe("record-capture semantic", () => {
|
||||||
let steps: RecordStepPayload[];
|
let steps: RecordStepPayload[];
|
||||||
|
|
||||||
@@ -124,19 +233,6 @@ describe("record-capture semantic", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not emit page navigation from a child Document capture", () => {
|
|
||||||
const originalUrl = location.href;
|
|
||||||
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
|
|
||||||
captureNavigation: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
history.pushState({}, "", "#inside-frame");
|
|
||||||
expect(steps).toEqual([]);
|
|
||||||
|
|
||||||
capture.dispose();
|
|
||||||
history.replaceState({}, "", originalUrl);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not record clicks on anonymous layout divs", () => {
|
it("does not record clicks on anonymous layout divs", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<div id="chrome">page chrome</div>
|
<div id="chrome">page chrome</div>
|
||||||
|
|||||||
@@ -1,102 +0,0 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
||||||
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
|
|
||||||
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
|
|
||||||
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
|
||||||
import { RecordFrameAgent } from "../recording/frame-agent";
|
|
||||||
|
|
||||||
class PortListeners<T extends (...args: never[]) => unknown> {
|
|
||||||
readonly values = new Set<T>();
|
|
||||||
addListener = (listener: T) => this.values.add(listener);
|
|
||||||
removeListener = (listener: T) => this.values.delete(listener);
|
|
||||||
}
|
|
||||||
|
|
||||||
function portHarness() {
|
|
||||||
const onMessage = new PortListeners<(message: unknown) => void>();
|
|
||||||
const onDisconnect = new PortListeners<() => void>();
|
|
||||||
const outbound: RecordFramePortMessage[] = [];
|
|
||||||
const port = {
|
|
||||||
name: "bsk-record-frame",
|
|
||||||
onMessage,
|
|
||||||
onDisconnect,
|
|
||||||
postMessage(message: RecordFramePortMessage) {
|
|
||||||
outbound.push(message);
|
|
||||||
if (message.type === "ready") {
|
|
||||||
queueMicrotask(() => {
|
|
||||||
for (const listener of onMessage.values) {
|
|
||||||
listener({
|
|
||||||
type: "ready_ack",
|
|
||||||
requestId: message.requestId,
|
|
||||||
producerId: message.producerId,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
},
|
|
||||||
disconnect: vi.fn(),
|
|
||||||
} as unknown as chrome.runtime.Port;
|
|
||||||
return {
|
|
||||||
port,
|
|
||||||
outbound,
|
|
||||||
receive(message: RecordFramePortMessage) {
|
|
||||||
for (const listener of onMessage.values) listener(message);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("RecordFrameAgent", () => {
|
|
||||||
afterEach(() => {
|
|
||||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
|
||||||
document.body.replaceChildren();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
|
|
||||||
const harness = portHarness();
|
|
||||||
const sendMessage = vi
|
|
||||||
.fn<(message: { sequence: number }) => Promise<unknown>>()
|
|
||||||
.mockRejectedValueOnce(new Error("offline"))
|
|
||||||
.mockRejectedValueOnce(new Error("still offline"))
|
|
||||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
|
||||||
vi.stubGlobal("chrome", {
|
|
||||||
runtime: {
|
|
||||||
connect: vi.fn(() => harness.port),
|
|
||||||
sendMessage,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
|
|
||||||
const agent = new RecordFrameAgent();
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
|
|
||||||
).resolves.toEqual({ ok: true });
|
|
||||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
|
||||||
|
|
||||||
const input = document.querySelector("input")!;
|
|
||||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
|
||||||
input.value = "final value";
|
|
||||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
|
||||||
|
|
||||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
|
|
||||||
await vi.waitFor(() =>
|
|
||||||
expect(harness.outbound).toContainEqual({
|
|
||||||
type: "stopped",
|
|
||||||
requestId: "rec-1",
|
|
||||||
commandId: "stop-1",
|
|
||||||
ok: false,
|
|
||||||
error: "failed to deliver one or more recorded steps",
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
|
|
||||||
|
|
||||||
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
|
|
||||||
await vi.waitFor(() =>
|
|
||||||
expect(harness.outbound).toContainEqual({
|
|
||||||
type: "stopped",
|
|
||||||
requestId: "rec-1",
|
|
||||||
commandId: "stop-2",
|
|
||||||
ok: true,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sendMessage).toHaveBeenCalledTimes(3);
|
|
||||||
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import type { RecordStepMessage } from "@/lib/record-bridge";
|
|
||||||
import { RecordStepDelivery } from "../record-step-delivery";
|
|
||||||
|
|
||||||
describe("RecordStepDelivery", () => {
|
|
||||||
it("retries from the first unacknowledged sequence before sending later steps", async () => {
|
|
||||||
const sent: RecordStepMessage[] = [];
|
|
||||||
const send = vi.fn(async (message: RecordStepMessage) => {
|
|
||||||
sent.push(message);
|
|
||||||
if (sent.length === 1) throw new Error("service worker unavailable");
|
|
||||||
return { ok: true, sequence: message.sequence };
|
|
||||||
});
|
|
||||||
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
|
|
||||||
|
|
||||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
|
|
||||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
|
|
||||||
|
|
||||||
await expect(delivery.flush()).resolves.toBe(true);
|
|
||||||
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
|
|
||||||
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps unacknowledged steps pending for a later flush", async () => {
|
|
||||||
const send = vi
|
|
||||||
.fn<(message: RecordStepMessage) => Promise<unknown>>()
|
|
||||||
.mockRejectedValueOnce(new Error("offline"))
|
|
||||||
.mockRejectedValueOnce(new Error("still offline"))
|
|
||||||
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
|
|
||||||
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
|
|
||||||
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
|
|
||||||
|
|
||||||
await expect(delivery.flush()).resolves.toBe(false);
|
|
||||||
await expect(delivery.flush()).resolves.toBe(true);
|
|
||||||
expect(send).toHaveBeenCalledTimes(3);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
|
||||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
|
||||||
|
|
||||||
export interface HelpRequestDataDeps {
|
|
||||||
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
|
||||||
query(): Promise<HelpQueryResponse | undefined>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Build the content-side representation shared by live delivery and recovery. */
|
|
||||||
export function createHelpRequestData(
|
|
||||||
message: Omit<HelpRequestMessage, "type">,
|
|
||||||
deps: HelpRequestDataDeps,
|
|
||||||
): HelpRequestData {
|
|
||||||
return {
|
|
||||||
id: message.requestId,
|
|
||||||
prompt: message.prompt,
|
|
||||||
...(message.title ? { title: message.title } : {}),
|
|
||||||
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
|
||||||
selectors: message.selectors,
|
|
||||||
rects: message.rects,
|
|
||||||
refreshRects: async () => {
|
|
||||||
const response = await deps.query();
|
|
||||||
return response?.active && response.request?.requestId === message.requestId
|
|
||||||
? response.request.rects
|
|
||||||
: undefined;
|
|
||||||
},
|
|
||||||
onContinue: (note: string) =>
|
|
||||||
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
|
||||||
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -12,8 +12,22 @@ import {
|
|||||||
hasDirectHoverInteractiveSignal,
|
hasDirectHoverInteractiveSignal,
|
||||||
hasStrongHoverExpansionSignal,
|
hasStrongHoverExpansionSignal,
|
||||||
} from "@/lib/hover-trigger-policy";
|
} from "@/lib/hover-trigger-policy";
|
||||||
import type { RecordStepPayload } from "@/lib/record-bridge";
|
import {
|
||||||
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
isRecordCancelMessage,
|
||||||
|
isRecordStartMessage,
|
||||||
|
isRecordStopMessage,
|
||||||
|
RECORD_CANCEL,
|
||||||
|
RECORD_START,
|
||||||
|
RECORD_STEP,
|
||||||
|
RECORD_STOP,
|
||||||
|
type RecordCancelMessage,
|
||||||
|
type RecordStartAck,
|
||||||
|
type RecordStartMessage,
|
||||||
|
type RecordStepPayload,
|
||||||
|
type RecordStopAck,
|
||||||
|
type RecordStopMessage,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
collectHoverSurfaceStates,
|
collectHoverSurfaceStates,
|
||||||
@@ -23,6 +37,37 @@ import {
|
|||||||
isLikelyHoverSurfaceOwner,
|
isLikelyHoverSurfaceOwner,
|
||||||
} from "./record-hover-surface";
|
} from "./record-hover-surface";
|
||||||
|
|
||||||
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
|
const failedStepDeliveries = new Set<string>();
|
||||||
|
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
|
||||||
|
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
|
||||||
|
const knownRecordRequests = new Set<string>();
|
||||||
|
|
||||||
|
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
|
||||||
|
const payload = { type: RECORD_STEP, requestId, step };
|
||||||
|
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||||
|
() => true,
|
||||||
|
() => {
|
||||||
|
failedStepDeliveries.add(requestId);
|
||||||
|
const failed = failedStepPayloads.get(requestId) ?? [];
|
||||||
|
failed.push(step);
|
||||||
|
failedStepPayloads.set(requestId, failed);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||||
|
const pending = deliverRecordStep(requestId, step);
|
||||||
|
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||||
|
sends.add(pending);
|
||||||
|
pendingStepSends.set(requestId, sends);
|
||||||
|
void pending.then(() => {
|
||||||
|
sends.delete(pending);
|
||||||
|
if (sends.size === 0) pendingStepSends.delete(requestId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export interface RecordCaptureController {
|
export interface RecordCaptureController {
|
||||||
dispose(): void;
|
dispose(): void;
|
||||||
}
|
}
|
||||||
@@ -124,8 +169,12 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
|||||||
|
|
||||||
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||||
const rect = el.getBoundingClientRect();
|
const rect = el.getBoundingClientRect();
|
||||||
|
const style = window.getComputedStyle(el);
|
||||||
return {
|
return {
|
||||||
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||||
|
scrollX: window.scrollX,
|
||||||
|
scrollY: window.scrollY,
|
||||||
|
position: style.position,
|
||||||
tag: el.tagName.toLowerCase(),
|
tag: el.tagName.toLowerCase(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -359,7 +408,6 @@ function scheduleInputCompletionCommit(
|
|||||||
export function startRecordCapture(
|
export function startRecordCapture(
|
||||||
_requestId: string,
|
_requestId: string,
|
||||||
sendStep: (step: RecordStepPayload) => void,
|
sendStep: (step: RecordStepPayload) => void,
|
||||||
options: { captureNavigation?: boolean } = {},
|
|
||||||
): RecordCaptureController {
|
): RecordCaptureController {
|
||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({ page_url: location.href, ...step });
|
||||||
@@ -896,33 +944,26 @@ export function startRecordCapture(
|
|||||||
document.addEventListener("change", onChange, true);
|
document.addEventListener("change", onChange, true);
|
||||||
document.addEventListener("keydown", onKeyDown, true);
|
document.addEventListener("keydown", onKeyDown, true);
|
||||||
|
|
||||||
const captureNavigation = options.captureNavigation ?? true;
|
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||||
const urlObserver = captureNavigation
|
urlObserver.observe(document, { subtree: true, childList: true });
|
||||||
? new MutationObserver(() => emitNavigateIfChanged())
|
|
||||||
: null;
|
|
||||||
urlObserver?.observe(document, { subtree: true, childList: true });
|
|
||||||
const onUrlEvent = () => emitNavigateIfChanged();
|
const onUrlEvent = () => emitNavigateIfChanged();
|
||||||
// Wrap: passing commitFillSession directly would forward the DOM event as
|
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||||
// the `commit` argument and stamp it onto the recorded fill step.
|
// the `commit` argument and stamp it onto the recorded fill step.
|
||||||
const onPageHide = () => commitFillSession();
|
const onPageHide = () => commitFillSession();
|
||||||
if (captureNavigation) {
|
window.addEventListener("hashchange", onUrlEvent);
|
||||||
window.addEventListener("hashchange", onUrlEvent);
|
window.addEventListener("popstate", onUrlEvent);
|
||||||
window.addEventListener("popstate", onUrlEvent);
|
|
||||||
}
|
|
||||||
window.addEventListener("pagehide", onPageHide);
|
window.addEventListener("pagehide", onPageHide);
|
||||||
|
|
||||||
const originalPushState = history.pushState;
|
const originalPushState = history.pushState;
|
||||||
const originalReplaceState = history.replaceState;
|
const originalReplaceState = history.replaceState;
|
||||||
if (captureNavigation) {
|
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
originalPushState.apply(this, args);
|
||||||
originalPushState.apply(this, args);
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
};
|
||||||
};
|
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
originalReplaceState.apply(this, args);
|
||||||
originalReplaceState.apply(this, args);
|
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
};
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
dispose() {
|
dispose() {
|
||||||
@@ -936,16 +977,118 @@ export function startRecordCapture(
|
|||||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||||
document.removeEventListener("change", onChange, true);
|
document.removeEventListener("change", onChange, true);
|
||||||
document.removeEventListener("keydown", onKeyDown, true);
|
document.removeEventListener("keydown", onKeyDown, true);
|
||||||
urlObserver?.disconnect();
|
urlObserver.disconnect();
|
||||||
if (captureNavigation) {
|
window.removeEventListener("hashchange", onUrlEvent);
|
||||||
window.removeEventListener("hashchange", onUrlEvent);
|
window.removeEventListener("popstate", onUrlEvent);
|
||||||
window.removeEventListener("popstate", onUrlEvent);
|
|
||||||
}
|
|
||||||
window.removeEventListener("pagehide", onPageHide);
|
window.removeEventListener("pagehide", onPageHide);
|
||||||
if (captureNavigation) {
|
history.pushState = originalPushState;
|
||||||
history.pushState = originalPushState;
|
history.replaceState = originalReplaceState;
|
||||||
history.replaceState = originalReplaceState;
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
||||||
|
|
||||||
|
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
||||||
|
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function handleRecordContentMessage(
|
||||||
|
message: RecordContentMessage,
|
||||||
|
state: {
|
||||||
|
activeRequestId: string | null;
|
||||||
|
capture: RecordCaptureController | null;
|
||||||
|
setActiveRequestId(id: string | null): void;
|
||||||
|
setCapture(capture: RecordCaptureController | null): void;
|
||||||
|
onStart(requestId: string, startedAtMs?: number): void;
|
||||||
|
onStop(): void;
|
||||||
|
},
|
||||||
|
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||||
|
): boolean {
|
||||||
|
if (isRecordStartMessage(message)) {
|
||||||
|
if (!knownRecordRequests.has(message.requestId)) {
|
||||||
|
knownRecordRequests.add(message.requestId);
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
}
|
||||||
|
state.capture?.dispose();
|
||||||
|
state.setCapture(
|
||||||
|
startRecordCapture(message.requestId, (step) => {
|
||||||
|
sendRecordStep(message.requestId, step);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
state.setActiveRequestId(message.requestId);
|
||||||
|
state.onStart(message.requestId, message.startedAtMs);
|
||||||
|
sendResponse?.({ ok: true });
|
||||||
|
return sendResponse !== undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||||
|
// Require an active recording that matches this requestId — otherwise a
|
||||||
|
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
||||||
|
// clear overlay state even though nothing was capturing.
|
||||||
|
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
state.capture?.dispose();
|
||||||
|
state.setCapture(null);
|
||||||
|
const finishStop = () => {
|
||||||
|
state.onStop();
|
||||||
|
state.setActiveRequestId(null);
|
||||||
|
};
|
||||||
|
if (isRecordStopMessage(message) && sendResponse) {
|
||||||
|
const existingFlush = pendingStopFlushes.get(message.requestId);
|
||||||
|
if (existingFlush) {
|
||||||
|
void existingFlush.then(sendResponse);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
|
||||||
|
if (failedBeforeStop.length > 0) {
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
}
|
||||||
|
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||||
|
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
|
||||||
|
const retried = await Promise.all(
|
||||||
|
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
|
||||||
|
);
|
||||||
|
const succeeded =
|
||||||
|
delivered.every(Boolean) &&
|
||||||
|
retried.every(Boolean) &&
|
||||||
|
!failedStepDeliveries.has(message.requestId);
|
||||||
|
if (succeeded) {
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
knownRecordRequests.delete(message.requestId);
|
||||||
|
finishStop();
|
||||||
|
}
|
||||||
|
return succeeded
|
||||||
|
? { ok: true }
|
||||||
|
: {
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
};
|
||||||
|
});
|
||||||
|
pendingStopFlushes.set(message.requestId, flush);
|
||||||
|
void flush.then(sendResponse).finally(() => {
|
||||||
|
if (pendingStopFlushes.get(message.requestId) === flush) {
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (isRecordCancelMessage(message)) {
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
|
knownRecordRequests.delete(message.requestId);
|
||||||
|
}
|
||||||
|
finishStop();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
import {
|
|
||||||
isAcceptedRecordStepAck,
|
|
||||||
RECORD_STEP,
|
|
||||||
type RecordStepMessage,
|
|
||||||
type RecordStepPayload,
|
|
||||||
} from "@/lib/record-bridge";
|
|
||||||
|
|
||||||
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
|
|
||||||
|
|
||||||
export class RecordStepDelivery {
|
|
||||||
readonly #requestId: string;
|
|
||||||
readonly #producerId: string;
|
|
||||||
readonly #send: SendRecordStepMessage;
|
|
||||||
readonly #pending = new Map<number, RecordStepPayload>();
|
|
||||||
#nextSequence = 1;
|
|
||||||
#tail = Promise.resolve();
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
requestId: string,
|
|
||||||
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
|
|
||||||
producerId: string = crypto.randomUUID(),
|
|
||||||
) {
|
|
||||||
this.#requestId = requestId;
|
|
||||||
this.#producerId = producerId;
|
|
||||||
this.#send = send;
|
|
||||||
}
|
|
||||||
|
|
||||||
enqueue(step: RecordStepPayload): void {
|
|
||||||
const sequence = this.#nextSequence;
|
|
||||||
this.#nextSequence += 1;
|
|
||||||
this.#pending.set(sequence, step);
|
|
||||||
void this.#schedule();
|
|
||||||
}
|
|
||||||
|
|
||||||
async flush(): Promise<boolean> {
|
|
||||||
await this.#tail;
|
|
||||||
await this.#schedule();
|
|
||||||
return this.#pending.size === 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
#schedule(): Promise<void> {
|
|
||||||
const drain = () => this.#drain();
|
|
||||||
this.#tail = this.#tail.then(drain, drain);
|
|
||||||
return this.#tail;
|
|
||||||
}
|
|
||||||
|
|
||||||
async #drain(): Promise<void> {
|
|
||||||
for (const [sequence, step] of this.#pending) {
|
|
||||||
const message: RecordStepMessage = {
|
|
||||||
type: RECORD_STEP,
|
|
||||||
requestId: this.#requestId,
|
|
||||||
producerId: this.#producerId,
|
|
||||||
sequence,
|
|
||||||
step,
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
const response = await this.#send(message);
|
|
||||||
if (!isAcceptedRecordStepAck(response, sequence)) return;
|
|
||||||
this.#pending.delete(sequence);
|
|
||||||
} catch {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,209 +0,0 @@
|
|||||||
import {
|
|
||||||
markRecordingDocument,
|
|
||||||
type RecordingDocumentMarker,
|
|
||||||
waitForDocumentElement,
|
|
||||||
} from "@/lib/recording/document-marker";
|
|
||||||
import {
|
|
||||||
isRecordFrameStartMessage,
|
|
||||||
RECORD_FRAME_PORT,
|
|
||||||
RECORD_FRAME_QUERY,
|
|
||||||
RECORD_FRAME_START,
|
|
||||||
type RecordFramePortMessage,
|
|
||||||
type RecordFrameQueryResponse,
|
|
||||||
type RecordFrameStartMessage,
|
|
||||||
} from "@/lib/recording/frame-bridge";
|
|
||||||
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
|
|
||||||
import { RecordStepDelivery } from "../record-step-delivery";
|
|
||||||
|
|
||||||
interface ActiveFrameRecording {
|
|
||||||
requestId: string;
|
|
||||||
producerId: string;
|
|
||||||
port: chrome.runtime.Port;
|
|
||||||
delivery: RecordStepDelivery;
|
|
||||||
capture: RecordCaptureController;
|
|
||||||
marker: RecordingDocumentMarker;
|
|
||||||
}
|
|
||||||
|
|
||||||
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
|
|
||||||
|
|
||||||
export class RecordFrameAgent {
|
|
||||||
#active: ActiveFrameRecording | null = null;
|
|
||||||
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
|
|
||||||
#disposed = false;
|
|
||||||
|
|
||||||
async start(
|
|
||||||
message: RecordFrameStartMessage,
|
|
||||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
||||||
if (this.#active?.requestId === message.requestId) return { ok: true };
|
|
||||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
|
||||||
if (this.#startPromise) {
|
|
||||||
await this.#startPromise;
|
|
||||||
return this.start(message);
|
|
||||||
}
|
|
||||||
if (this.#active) this.cancel(this.#active.requestId);
|
|
||||||
const starting = this.#start(message);
|
|
||||||
this.#startPromise = starting;
|
|
||||||
try {
|
|
||||||
return await starting;
|
|
||||||
} finally {
|
|
||||||
if (this.#startPromise === starting) this.#startPromise = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel(requestId: string): void {
|
|
||||||
const active = this.#active;
|
|
||||||
if (!active || active.requestId !== requestId) return;
|
|
||||||
active.capture.dispose();
|
|
||||||
active.marker.restore();
|
|
||||||
active.port.disconnect();
|
|
||||||
this.#active = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
dispose(): void {
|
|
||||||
this.#disposed = true;
|
|
||||||
if (this.#active) this.cancel(this.#active.requestId);
|
|
||||||
}
|
|
||||||
|
|
||||||
async #start(
|
|
||||||
message: RecordFrameStartMessage,
|
|
||||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
||||||
const producerId = crypto.randomUUID();
|
|
||||||
const root = await waitForDocumentElement();
|
|
||||||
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
|
|
||||||
const marker = markRecordingDocument(producerId, root);
|
|
||||||
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
|
|
||||||
try {
|
|
||||||
await registerPort(port, message.requestId, producerId);
|
|
||||||
} catch {
|
|
||||||
marker.restore();
|
|
||||||
port.disconnect();
|
|
||||||
return { ok: false, error: "failed to register recording document" };
|
|
||||||
}
|
|
||||||
if (this.#disposed) {
|
|
||||||
marker.restore();
|
|
||||||
port.disconnect();
|
|
||||||
return { ok: false, error: "recording document is disposed" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
|
|
||||||
const capture = startRecordCapture(
|
|
||||||
message.requestId,
|
|
||||||
(step) => {
|
|
||||||
marker.ensure();
|
|
||||||
delivery.enqueue(step);
|
|
||||||
},
|
|
||||||
{ captureNavigation: window.top === window },
|
|
||||||
);
|
|
||||||
const active: ActiveFrameRecording = {
|
|
||||||
requestId: message.requestId,
|
|
||||||
producerId,
|
|
||||||
port,
|
|
||||||
delivery,
|
|
||||||
capture,
|
|
||||||
marker,
|
|
||||||
};
|
|
||||||
this.#active = active;
|
|
||||||
port.onMessage.addListener((raw: unknown) => {
|
|
||||||
const command = raw as Partial<RecordFramePortMessage>;
|
|
||||||
if (command.type === "stop" && command.requestId === active.requestId) {
|
|
||||||
void this.#stop(active, command.commandId);
|
|
||||||
} else if (command.type === "cancel" && command.requestId === active.requestId) {
|
|
||||||
this.cancel(active.requestId);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
port.onDisconnect.addListener(() => {
|
|
||||||
if (this.#active !== active) return;
|
|
||||||
active.capture.dispose();
|
|
||||||
active.marker.restore();
|
|
||||||
this.#active = null;
|
|
||||||
});
|
|
||||||
return { ok: true };
|
|
||||||
}
|
|
||||||
|
|
||||||
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
|
|
||||||
if (!commandId || this.#active !== active) return;
|
|
||||||
active.capture.dispose();
|
|
||||||
const succeeded = await active.delivery.flush();
|
|
||||||
try {
|
|
||||||
active.port.postMessage({
|
|
||||||
type: "stopped",
|
|
||||||
requestId: active.requestId,
|
|
||||||
commandId,
|
|
||||||
ok: succeeded,
|
|
||||||
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
|
|
||||||
} satisfies RecordFramePortMessage);
|
|
||||||
} catch {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!succeeded || this.#active !== active) return;
|
|
||||||
active.marker.restore();
|
|
||||||
this.#active = null;
|
|
||||||
active.port.disconnect();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function registerPort(
|
|
||||||
port: chrome.runtime.Port,
|
|
||||||
requestId: string,
|
|
||||||
producerId: string,
|
|
||||||
): Promise<void> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const timer = setTimeout(() => {
|
|
||||||
cleanup();
|
|
||||||
reject(new Error("recording document registration timed out"));
|
|
||||||
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
|
|
||||||
const cleanup = () => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
port.onMessage.removeListener(onMessage);
|
|
||||||
port.onDisconnect.removeListener(onDisconnect);
|
|
||||||
};
|
|
||||||
const onMessage = (raw: unknown) => {
|
|
||||||
const message = raw as Partial<RecordFramePortMessage>;
|
|
||||||
if (
|
|
||||||
message.type !== "ready_ack" ||
|
|
||||||
message.requestId !== requestId ||
|
|
||||||
message.producerId !== producerId
|
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
cleanup();
|
|
||||||
resolve();
|
|
||||||
};
|
|
||||||
const onDisconnect = () => {
|
|
||||||
cleanup();
|
|
||||||
reject(new Error("recording document port disconnected"));
|
|
||||||
};
|
|
||||||
port.onMessage.addListener(onMessage);
|
|
||||||
port.onDisconnect.addListener(onDisconnect);
|
|
||||||
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function attachRecordFrameAgent(): () => void {
|
|
||||||
const agent = new RecordFrameAgent();
|
|
||||||
const onMessage = (
|
|
||||||
message: unknown,
|
|
||||||
_sender: chrome.runtime.MessageSender,
|
|
||||||
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
|
|
||||||
) => {
|
|
||||||
if (!isRecordFrameStartMessage(message)) return false;
|
|
||||||
void agent.start(message).then(sendResponse);
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
chrome.runtime.onMessage.addListener(onMessage);
|
|
||||||
void chrome.runtime
|
|
||||||
.sendMessage({ type: RECORD_FRAME_QUERY })
|
|
||||||
.then((response: RecordFrameQueryResponse | undefined) => {
|
|
||||||
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
|
|
||||||
return agent.start({
|
|
||||||
type: RECORD_FRAME_START,
|
|
||||||
requestId: response.requestId,
|
|
||||||
startedAtMs: response.startedAtMs,
|
|
||||||
});
|
|
||||||
})
|
|
||||||
.catch(() => {});
|
|
||||||
return () => {
|
|
||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
|
||||||
agent.dispose();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -21,7 +21,6 @@ import {
|
|||||||
type OverlayMode,
|
type OverlayMode,
|
||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||||
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
|
||||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||||
@@ -39,7 +38,6 @@ import {
|
|||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
attachRecordQueryListener,
|
attachRecordQueryListener,
|
||||||
attachRecordStepListener,
|
attachRecordStepListener,
|
||||||
type RecordRuntimeDeps,
|
|
||||||
} from "@/tools/record";
|
} from "@/tools/record";
|
||||||
import { detectBrowserMeta } from "@/transport/handshake";
|
import { detectBrowserMeta } from "@/transport/handshake";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
@@ -147,33 +145,10 @@ export default defineBackground(() => {
|
|||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const recordDeps = {
|
|
||||||
tabsApi: chrome.tabs,
|
|
||||||
cdp,
|
|
||||||
frameCoordinator: recordFrameCoordinator,
|
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
|
||||||
try {
|
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
} satisfies RecordRuntimeDeps;
|
|
||||||
recordFrameCoordinator.attach();
|
|
||||||
attachRecordStepListener(recordDeps);
|
|
||||||
attachRecordFinishListener(recordDeps);
|
|
||||||
attachRecordQueryListener(recordDeps);
|
|
||||||
|
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
cdp,
|
cdp,
|
||||||
recording: recordDeps,
|
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
onBrowserControlResumed,
|
onBrowserControlResumed,
|
||||||
approveBorrow: (ctx) =>
|
approveBorrow: (ctx) =>
|
||||||
@@ -195,6 +170,28 @@ export default defineBackground(() => {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
|
const recordDeps = {
|
||||||
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// Message listeners stay up (cheap; fire only for record message types).
|
||||||
|
// Tab / webNavigation observation attaches lazily while a recording is
|
||||||
|
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||||
|
attachRecordStepListener(recordDeps);
|
||||||
|
attachRecordFinishListener(recordDeps);
|
||||||
|
attachRecordQueryListener(recordDeps);
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
attachBorrowNotificationClickHandler({
|
attachBorrowNotificationClickHandler({
|
||||||
onClicked: chrome.notifications.onClicked,
|
onClicked: chrome.notifications.onClicked,
|
||||||
|
|||||||
@@ -7,10 +7,14 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
|||||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||||
import { createHelpRequestData } from "@/content/help-request";
|
|
||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||||
|
import {
|
||||||
|
handleRecordContentMessage,
|
||||||
|
isRecordContentMessage,
|
||||||
|
type RecordCaptureController,
|
||||||
|
} from "@/content/record-capture";
|
||||||
import {
|
import {
|
||||||
type CaptureSuppressAck,
|
type CaptureSuppressAck,
|
||||||
type CaptureSuppressMessage,
|
type CaptureSuppressMessage,
|
||||||
@@ -40,17 +44,11 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
isRecordCancelMessage,
|
|
||||||
isRecordStartMessage,
|
|
||||||
isRecordStopMessage,
|
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
type RecordCancelMessage,
|
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
type RecordStopMessage,
|
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
@@ -70,6 +68,7 @@ export default defineContentScript({
|
|||||||
if (window.top !== window) return;
|
if (window.top !== window) return;
|
||||||
|
|
||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
@@ -228,6 +227,8 @@ export default defineContentScript({
|
|||||||
if (previousHelp) {
|
if (previousHelp) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
|
recordCapture?.dispose();
|
||||||
|
recordCapture = null;
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
renderAll();
|
renderAll();
|
||||||
}
|
}
|
||||||
@@ -257,9 +258,6 @@ export default defineContentScript({
|
|||||||
| HelpRequestMessage
|
| HelpRequestMessage
|
||||||
| HelpCancelMessage
|
| HelpCancelMessage
|
||||||
| CaptureSuppressMessage
|
| CaptureSuppressMessage
|
||||||
| RecordStartMessage
|
|
||||||
| RecordStopMessage
|
|
||||||
| RecordCancelMessage
|
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
@@ -270,32 +268,41 @@ export default defineContentScript({
|
|||||||
return captureSuppress.handleMessage(message, sendResponse);
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isRecordStartMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
activeRecordRequestId = message.requestId;
|
const needsAsync = handleRecordContentMessage(
|
||||||
overlays.setAgentRecordRequest({
|
message,
|
||||||
id: message.requestId,
|
{
|
||||||
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
|
activeRequestId: activeRecordRequestId,
|
||||||
onFinish: () => {
|
capture: recordCapture,
|
||||||
void chrome.runtime.sendMessage({
|
setActiveRequestId: (id) => {
|
||||||
type: RECORD_FINISH,
|
activeRecordRequestId = id;
|
||||||
requestId: message.requestId,
|
},
|
||||||
});
|
setCapture: (capture) => {
|
||||||
|
recordCapture = capture;
|
||||||
|
},
|
||||||
|
onStart: (requestId, startedAtMs) => {
|
||||||
|
overlays.setAgentRecordRequest({
|
||||||
|
id: requestId,
|
||||||
|
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||||
|
onFinish: () => {
|
||||||
|
void chrome.runtime.sendMessage({
|
||||||
|
type: RECORD_FINISH,
|
||||||
|
requestId,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
renderAll();
|
||||||
|
},
|
||||||
|
onStop: () => {
|
||||||
|
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||||
|
renderAll();
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
sendResponse as unknown as
|
||||||
renderAll();
|
| ((response: RecordStartAck | RecordStopAck) => void)
|
||||||
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
|
| undefined,
|
||||||
return false;
|
);
|
||||||
}
|
return needsAsync;
|
||||||
|
|
||||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
|
||||||
if (activeRecordRequestId !== message.requestId) return false;
|
|
||||||
overlays.clearAgentRecordRequest(message.requestId);
|
|
||||||
activeRecordRequestId = null;
|
|
||||||
renderAll();
|
|
||||||
if (isRecordStopMessage(message)) {
|
|
||||||
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -337,7 +344,16 @@ export default defineContentScript({
|
|||||||
|
|
||||||
if (isHelpRequestMessage(message)) {
|
if (isHelpRequestMessage(message)) {
|
||||||
const helpMsg = message as HelpRequestMessage;
|
const helpMsg = message as HelpRequestMessage;
|
||||||
const previousHelp = mountHelpRequest(helpMsg);
|
const previousHelp = overlays.setAgentHelpRequest({
|
||||||
|
id: helpMsg.requestId,
|
||||||
|
prompt: helpMsg.prompt,
|
||||||
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
|
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||||
|
selectors: helpMsg.selectors,
|
||||||
|
onContinue: (note: string) =>
|
||||||
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
|
});
|
||||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||||
}
|
}
|
||||||
@@ -390,18 +406,17 @@ export default defineContentScript({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||||
return overlays.setAgentHelpRequest(
|
overlays.setAgentHelpRequest({
|
||||||
createHelpRequestData(helpMsg, {
|
id: helpMsg.requestId,
|
||||||
finish: (requestId, outcome, note) => {
|
prompt: helpMsg.prompt,
|
||||||
void sendHelpFinish(requestId, outcome, note);
|
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||||
},
|
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||||
query: () =>
|
selectors: helpMsg.selectors,
|
||||||
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
onContinue: (note: string) =>
|
||||||
HelpQueryResponse | undefined
|
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||||
>,
|
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||||
}),
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||||
@@ -518,6 +533,9 @@ export default defineContentScript({
|
|||||||
chrome.runtime.onMessage.removeListener(onMessage);
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||||
window.removeEventListener("pageshow", onPageShow);
|
window.removeEventListener("pageshow", onPageShow);
|
||||||
|
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||||
|
recordCapture?.dispose();
|
||||||
|
recordCapture = null;
|
||||||
activeRecordRequestId = null;
|
activeRecordRequestId = null;
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -100,6 +100,24 @@ describe("App", () => {
|
|||||||
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
|
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders localized copy for a version failure instead of protocol details", () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
lastError:
|
||||||
|
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.0",
|
||||||
|
},
|
||||||
|
statusState: "disconnected",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
|
||||||
|
expect(error.textContent).not.toContain("min_compatible_protocol");
|
||||||
|
});
|
||||||
|
|
||||||
it("shows single-line compact metadata and copies the instance id", async () => {
|
it("shows single-line compact metadata and copies the instance id", async () => {
|
||||||
mockUseConnectionState.mockReturnValue({
|
mockUseConnectionState.mockReturnValue({
|
||||||
snapshot: {
|
snapshot: {
|
||||||
@@ -202,7 +220,11 @@ describe("App", () => {
|
|||||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||||
expect(copied).toContain("./trace");
|
expect(copied).toContain("./trace");
|
||||||
expect(copied).toContain("trace.json");
|
expect(copied).toContain("trace.json");
|
||||||
expect(copied).toContain("states/");
|
expect(copied).toContain("pages/");
|
||||||
|
expect(copied).toContain("总结我做了什么");
|
||||||
|
expect(copied).not.toContain("不要重跑");
|
||||||
|
expect(copied).not.toContain("pages + steps");
|
||||||
|
expect(copied).not.toContain("@eN");
|
||||||
// Button label stays static; a transient toast confirms the copy.
|
// Button label stays static; a transient toast confirms the copy.
|
||||||
expect(copyButton.textContent).toContain("复制录制指令");
|
expect(copyButton.textContent).toContain("复制录制指令");
|
||||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||||
@@ -247,41 +269,6 @@ describe("App", () => {
|
|||||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
|
||||||
mockUseConnectionState.mockReturnValue({
|
|
||||||
snapshot: {
|
|
||||||
...baseSnapshot,
|
|
||||||
state: "version_skew",
|
|
||||||
instanceId: "03c3e47f",
|
|
||||||
handshake: {
|
|
||||||
server: "bh",
|
|
||||||
version: mockDaemonVersion,
|
|
||||||
protocol_version: "1.0",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
statusState: "version_skew",
|
|
||||||
setLabel,
|
|
||||||
setConnectionEnabled,
|
|
||||||
});
|
|
||||||
|
|
||||||
render(<App />);
|
|
||||||
|
|
||||||
expect(screen.getByText("已连接")).toBeTruthy();
|
|
||||||
expect(screen.getByText("可升级")).toBeTruthy();
|
|
||||||
expect(screen.queryByText("协议不一致")).toBeNull();
|
|
||||||
expect(screen.queryByText("Action needed")).toBeNull();
|
|
||||||
expect(screen.queryByText("兼容")).toBeNull();
|
|
||||||
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
|
||||||
expect(warning.textContent).toContain("CLI 协议");
|
|
||||||
expect(warning.textContent).toContain("扩展协议");
|
|
||||||
|
|
||||||
openRecordView();
|
|
||||||
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
|
||||||
expect(copyButton.getAttribute("disabled")).toBeNull();
|
|
||||||
fireEvent.click(copyButton);
|
|
||||||
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("control hints toggle", () => {
|
describe("control hints toggle", () => {
|
||||||
|
|||||||
@@ -79,11 +79,13 @@ export function App() {
|
|||||||
}, [copiedTick]);
|
}, [copiedTick]);
|
||||||
|
|
||||||
const isSkewed = statusState === "version_skew";
|
const isSkewed = statusState === "version_skew";
|
||||||
const connectionLive = statusState === "connected" || isSkewed;
|
|
||||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
const instanceId = snapshot.instanceId || "—";
|
const instanceId = snapshot.instanceId || "—";
|
||||||
|
const errorMessage = snapshot.lastError?.includes("version_too_old")
|
||||||
|
? t("popup.versionIncompatibleError")
|
||||||
|
: snapshot.lastError;
|
||||||
|
|
||||||
const copyInstanceId = async () => {
|
const copyInstanceId = async () => {
|
||||||
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
||||||
@@ -91,7 +93,7 @@ export function App() {
|
|||||||
setCopiedInstanceId(true);
|
setCopiedInstanceId(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
||||||
const recordPurpose = purposeDraft.trim();
|
const recordPurpose = purposeDraft.trim();
|
||||||
const recordStartUrl = startUrlDraft.trim();
|
const recordStartUrl = startUrlDraft.trim();
|
||||||
const recordCommand = snapshot.instanceId
|
const recordCommand = snapshot.instanceId
|
||||||
@@ -207,12 +209,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
<p
|
<p
|
||||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
||||||
data-slot="popup-version-skew-warning"
|
data-slot="popup-version-skew-warning"
|
||||||
>
|
>
|
||||||
{t("popup.versionSkewWarning", {
|
{t("popup.versionSkewWarning", {
|
||||||
extensionProtocol: PROTOCOL_VERSION,
|
extensionProtocol: PROTOCOL_VERSION,
|
||||||
cliProtocol: daemonProtocol,
|
daemonProtocol,
|
||||||
})}
|
})}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
@@ -253,12 +255,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
{snapshot.lastError && (
|
{errorMessage && (
|
||||||
<div
|
<div
|
||||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||||
data-slot="popup-error"
|
data-slot="popup-error"
|
||||||
>
|
>
|
||||||
{snapshot.lastError}
|
{errorMessage}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
|
|
||||||
|
|
||||||
export default defineContentScript({
|
|
||||||
matches: ["<all_urls>"],
|
|
||||||
runAt: "document_start",
|
|
||||||
allFrames: true,
|
|
||||||
matchAboutBlank: true,
|
|
||||||
matchOriginAsFallback: true,
|
|
||||||
|
|
||||||
main(ctx) {
|
|
||||||
const dispose = attachRecordFrameAgent();
|
|
||||||
ctx.onInvalidated(dispose);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
@@ -26,20 +26,20 @@ function handshake(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("computeConnectedState (protocol-based compat)", () => {
|
describe("computeConnectedState (protocol-based compat)", () => {
|
||||||
it("returns connected when daemon protocol equals extension protocol", () => {
|
it("returns connected when protocol strings match", () => {
|
||||||
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||||
kind: "connected",
|
kind: "connected",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||||
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||||
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -48,7 +48,9 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState(handshake("2.0", "1.0"));
|
const result = computeConnectedState(handshake("2.0", "1.0"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("protocol-major mismatch");
|
expect(result.reason).toContain("bsk CLI");
|
||||||
|
expect(result.reason).toContain("BrowserSkill extension");
|
||||||
|
expect(result.reason).not.toContain("protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -56,8 +58,9 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState(handshake("1.1", "1.5"));
|
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("min_compatible_protocol");
|
expect(result.reason).toContain("BrowserSkill extension is out of date");
|
||||||
expect(result.reason).toContain("1.5");
|
expect(result.reason).toContain("Update and reload the extension");
|
||||||
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -75,16 +78,24 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("below extension min_compatible_protocol");
|
expect(result.reason).toContain("bsk CLI is out of date");
|
||||||
|
expect(result.reason).toContain("bsk daemon restart");
|
||||||
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns version_skew when daemon protocol is 1.0 and floor is satisfied", () => {
|
||||||
|
expect(computeConnectedState(handshake("1.0", "1.0"))).toEqual({
|
||||||
|
kind: "version_skew",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||||
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
expect(result.reason).toContain("could not verify version compatibility");
|
||||||
expect(result.reason).toContain("not-a-protocol");
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -242,11 +253,11 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
expect(second.id).not.toBe(first.id);
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
|
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
expect(controller.snapshot().state).not.toBe("connected");
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
|
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
|
||||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,101 +0,0 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
||||||
import type { CdpRunner } from "@/tools/shared";
|
|
||||||
import { waitForDocumentSettled } from "../recording/document-settle";
|
|
||||||
|
|
||||||
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
|
||||||
const send = vi.fn(async () => ({
|
|
||||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
|
||||||
}));
|
|
||||||
return {
|
|
||||||
cdp: { send: send as unknown as CdpRunner["send"] },
|
|
||||||
send,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("waitForDocumentSettled", () => {
|
|
||||||
afterEach(() => {
|
|
||||||
vi.useRealTimers();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const { cdp, send } = quietCdp();
|
|
||||||
|
|
||||||
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
|
|
||||||
await expect(settled).resolves.toBe("quiet");
|
|
||||||
expect(send).toHaveBeenCalledTimes(3);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("cancels before probing a superseded observation", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const { cdp, send } = quietCdp();
|
|
||||||
|
|
||||||
const controller = new AbortController();
|
|
||||||
const settled = waitForDocumentSettled(
|
|
||||||
cdp,
|
|
||||||
{ target: { tabId: 7 } },
|
|
||||||
{
|
|
||||||
signal: controller.signal,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
controller.abort();
|
|
||||||
await vi.advanceTimersByTimeAsync(60);
|
|
||||||
|
|
||||||
await expect(settled).resolves.toBe("cancelled");
|
|
||||||
expect(send).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("probes a same-process iframe through its isolated execution context", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
|
||||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
|
||||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
|
||||||
});
|
|
||||||
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
|
||||||
|
|
||||||
const settled = waitForDocumentSettled(cdp, {
|
|
||||||
target: { tabId: 7 },
|
|
||||||
frameId: "child-frame",
|
|
||||||
});
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
|
|
||||||
await expect(settled).resolves.toBe("quiet");
|
|
||||||
expect(send).toHaveBeenCalledWith(
|
|
||||||
7,
|
|
||||||
"Page.createIsolatedWorld",
|
|
||||||
expect.objectContaining({ frameId: "child-frame" }),
|
|
||||||
);
|
|
||||||
expect(send).toHaveBeenCalledWith(
|
|
||||||
7,
|
|
||||||
"Runtime.evaluate",
|
|
||||||
expect.objectContaining({ contextId: 91 }),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("routes an OOPIF probe through its CDP target session", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const send = vi.fn();
|
|
||||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
|
||||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
|
||||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
|
||||||
});
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: send as unknown as CdpRunner["send"],
|
|
||||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
|
||||||
};
|
|
||||||
|
|
||||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
|
||||||
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
|
|
||||||
await expect(settled).resolves.toBe("quiet");
|
|
||||||
expect(send).not.toHaveBeenCalled();
|
|
||||||
expect(sendToTarget).toHaveBeenCalledWith(
|
|
||||||
target,
|
|
||||||
"Runtime.evaluate",
|
|
||||||
expect.objectContaining({ contextId: 27 }),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -32,7 +32,6 @@ describe("help-bridge", () => {
|
|||||||
prompt: "log in",
|
prompt: "log in",
|
||||||
displayMode: "compact",
|
displayMode: "compact",
|
||||||
selectors: ["#login"],
|
selectors: ["#login"],
|
||||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
|
||||||
timeoutMs: 1000,
|
timeoutMs: 1000,
|
||||||
}),
|
}),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import { matchTarget } from "../match-target";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "button",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchTarget", () => {
|
||||||
|
const refs: RenderedRef[] = [
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||||
|
];
|
||||||
|
|
||||||
|
it("matches a unique node by viewport coordinates", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42)],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 40,
|
||||||
|
scrollY: 300,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 250,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched when multiple candidates match", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42), node(43)],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses viewport coordinates for fixed elements", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||||
|
scrollX: 100,
|
||||||
|
scrollY: 200,
|
||||||
|
position: "fixed",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import {
|
|
||||||
isAcceptedRecordStepAck,
|
|
||||||
isRecordStepMessage,
|
|
||||||
RECORD_STEP,
|
|
||||||
type RecordStepMessage,
|
|
||||||
} from "../record-bridge";
|
|
||||||
|
|
||||||
const message: RecordStepMessage = {
|
|
||||||
type: RECORD_STEP,
|
|
||||||
requestId: "rec-1",
|
|
||||||
producerId: "document-1",
|
|
||||||
sequence: 1,
|
|
||||||
step: { op: "click", target: { tag: "button", name: "Submit" } },
|
|
||||||
};
|
|
||||||
|
|
||||||
describe("record step delivery protocol", () => {
|
|
||||||
it("requires a producer and a positive integer sequence", () => {
|
|
||||||
expect(isRecordStepMessage(message)).toBe(true);
|
|
||||||
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
|
|
||||||
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
|
|
||||||
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("accepts only the acknowledgement for the sent sequence", () => {
|
|
||||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
|
|
||||||
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
|
|
||||||
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,213 +0,0 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
||||||
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
|
|
||||||
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
|
||||||
|
|
||||||
class ListenerSet<T extends (...args: never[]) => unknown> {
|
|
||||||
readonly listeners = new Set<T>();
|
|
||||||
addListener = (listener: T) => this.listeners.add(listener);
|
|
||||||
removeListener = (listener: T) => this.listeners.delete(listener);
|
|
||||||
}
|
|
||||||
|
|
||||||
function fakePort(sender: chrome.runtime.MessageSender) {
|
|
||||||
const messages: unknown[] = [];
|
|
||||||
const onMessage = new ListenerSet<(message: unknown) => void>();
|
|
||||||
const onDisconnect = new ListenerSet<() => void>();
|
|
||||||
return {
|
|
||||||
port: {
|
|
||||||
name: RECORD_FRAME_PORT,
|
|
||||||
sender,
|
|
||||||
onMessage,
|
|
||||||
onDisconnect,
|
|
||||||
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
|
||||||
disconnect: vi.fn(),
|
|
||||||
} as unknown as chrome.runtime.Port,
|
|
||||||
messages,
|
|
||||||
receive(message: unknown) {
|
|
||||||
for (const listener of onMessage.listeners) listener(message);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("RecordFrameCoordinator", () => {
|
|
||||||
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
|
||||||
const onMessage = new ListenerSet<
|
|
||||||
(
|
|
||||||
message: unknown,
|
|
||||||
sender: chrome.runtime.MessageSender,
|
|
||||||
sendResponse: (response: unknown) => void,
|
|
||||||
) => boolean
|
|
||||||
>();
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
onConnect.listeners.clear();
|
|
||||||
onMessage.listeners.clear();
|
|
||||||
vi.stubGlobal("chrome", {
|
|
||||||
runtime: { onConnect, onMessage },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("answers child-document queries for the initial tab before it is armed", () => {
|
|
||||||
const coordinator = new RecordFrameCoordinator({
|
|
||||||
getAllFrames: async () => [],
|
|
||||||
sendToDocument: async () => ({ ok: true }),
|
|
||||||
});
|
|
||||||
coordinator.attach();
|
|
||||||
coordinator.begin("rec-1", 10, 3);
|
|
||||||
|
|
||||||
const query = (tabId: number) => {
|
|
||||||
let response: unknown;
|
|
||||||
for (const listener of onMessage.listeners) {
|
|
||||||
listener(
|
|
||||||
{ type: RECORD_FRAME_QUERY },
|
|
||||||
{
|
|
||||||
tab: { id: tabId },
|
|
||||||
frameId: 7,
|
|
||||||
documentId: "child-document",
|
|
||||||
} as chrome.runtime.MessageSender,
|
|
||||||
(value) => {
|
|
||||||
response = value;
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return response;
|
|
||||||
};
|
|
||||||
|
|
||||||
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
|
|
||||||
expect(query(4)).toEqual({ active: false });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("starts a child Document that appears after the initial frame snapshot", async () => {
|
|
||||||
let notifyFrameNavigation:
|
|
||||||
| ((frame: {
|
|
||||||
tabId: number;
|
|
||||||
frameId: number;
|
|
||||||
documentId?: string;
|
|
||||||
lifecycle: "committed" | "completed";
|
|
||||||
}) => void)
|
|
||||||
| undefined;
|
|
||||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
|
||||||
const coordinator = new RecordFrameCoordinator({
|
|
||||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
|
||||||
sendToDocument,
|
|
||||||
subscribeFrameNavigation(listener) {
|
|
||||||
notifyFrameNavigation = listener;
|
|
||||||
return () => {};
|
|
||||||
},
|
|
||||||
});
|
|
||||||
coordinator.attach();
|
|
||||||
const onDocumentReady = vi.fn();
|
|
||||||
coordinator.begin("rec-1", 10, 3, onDocumentReady);
|
|
||||||
await coordinator.armTab("rec-1", 3);
|
|
||||||
sendToDocument.mockClear();
|
|
||||||
|
|
||||||
notifyFrameNavigation?.({
|
|
||||||
tabId: 3,
|
|
||||||
frameId: 7,
|
|
||||||
documentId: "late-child",
|
|
||||||
lifecycle: "completed",
|
|
||||||
});
|
|
||||||
await vi.waitFor(() => {
|
|
||||||
expect(sendToDocument).toHaveBeenCalledWith(
|
|
||||||
3,
|
|
||||||
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
|
|
||||||
{ documentId: "late-child" },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
expect(onDocumentReady).not.toHaveBeenCalled();
|
|
||||||
|
|
||||||
const child = fakePort({
|
|
||||||
tab: { id: 3 },
|
|
||||||
frameId: 7,
|
|
||||||
documentId: "late-child",
|
|
||||||
} as chrome.runtime.MessageSender);
|
|
||||||
for (const listener of onConnect.listeners) listener(child.port);
|
|
||||||
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
|
||||||
notifyFrameNavigation?.({
|
|
||||||
tabId: 3,
|
|
||||||
frameId: 7,
|
|
||||||
documentId: "late-child",
|
|
||||||
lifecycle: "completed",
|
|
||||||
});
|
|
||||||
await vi.waitFor(() => {
|
|
||||||
expect(onDocumentReady).toHaveBeenCalledWith({
|
|
||||||
tabId: 3,
|
|
||||||
documentId: "late-child",
|
|
||||||
browserFrameId: 7,
|
|
||||||
producerId: "producer-1",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
|
||||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
|
||||||
const coordinator = new RecordFrameCoordinator({
|
|
||||||
getAllFrames: async () => [
|
|
||||||
{ frameId: 0, documentId: "top-document" },
|
|
||||||
{ frameId: 7, documentId: "child-document" },
|
|
||||||
],
|
|
||||||
sendToDocument,
|
|
||||||
});
|
|
||||||
coordinator.attach();
|
|
||||||
coordinator.begin("rec-1", 10, 3);
|
|
||||||
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
|
||||||
|
|
||||||
const sender = {
|
|
||||||
tab: { id: 3 },
|
|
||||||
frameId: 7,
|
|
||||||
documentId: "child-document",
|
|
||||||
} as chrome.runtime.MessageSender;
|
|
||||||
const frame = fakePort(sender);
|
|
||||||
for (const listener of onConnect.listeners) listener(frame.port);
|
|
||||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
|
||||||
|
|
||||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
|
||||||
tabId: 3,
|
|
||||||
documentId: "child-document",
|
|
||||||
browserFrameId: 7,
|
|
||||||
producerId: "producer-1",
|
|
||||||
});
|
|
||||||
|
|
||||||
const stopping = coordinator.stop("rec-1");
|
|
||||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
|
||||||
const stop = frame.messages.find(
|
|
||||||
(message): message is { type: "stop"; commandId: string } =>
|
|
||||||
typeof message === "object" &&
|
|
||||||
message !== null &&
|
|
||||||
(message as { type?: string }).type === "stop",
|
|
||||||
);
|
|
||||||
expect(stop).toBeDefined();
|
|
||||||
frame.receive({
|
|
||||||
type: "stopped",
|
|
||||||
requestId: "rec-1",
|
|
||||||
commandId: stop!.commandId,
|
|
||||||
ok: true,
|
|
||||||
});
|
|
||||||
await expect(stopping).resolves.toBe(true);
|
|
||||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects a producer used from another Document", async () => {
|
|
||||||
const coordinator = new RecordFrameCoordinator({
|
|
||||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
|
||||||
sendToDocument: async () => ({ ok: true }),
|
|
||||||
});
|
|
||||||
coordinator.attach();
|
|
||||||
coordinator.begin("rec-1", 10, 3);
|
|
||||||
await coordinator.armTab("rec-1", 3);
|
|
||||||
const sender = {
|
|
||||||
tab: { id: 3 },
|
|
||||||
frameId: 0,
|
|
||||||
documentId: "top-document",
|
|
||||||
} as chrome.runtime.MessageSender;
|
|
||||||
const frame = fakePort(sender);
|
|
||||||
for (const listener of onConnect.listeners) listener(frame.port);
|
|
||||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
|
||||||
|
|
||||||
expect(
|
|
||||||
coordinator.sourceFor("rec-1", "producer-1", {
|
|
||||||
...sender,
|
|
||||||
documentId: "other-document",
|
|
||||||
}),
|
|
||||||
).toBeNull();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
import { beforeEach, describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep } from "@/transport/types";
|
||||||
|
import { resetStateIdCounterForTests } from "../record-constants";
|
||||||
|
import {
|
||||||
|
applyTargetMatching,
|
||||||
|
buildTraceV3,
|
||||||
|
createObservationState,
|
||||||
|
rememberStepOnPage,
|
||||||
|
} from "../record-observation";
|
||||||
|
import { registerObservation } from "../trace-reducer";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function capturedInput(): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId: 42,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
const obs = createObservationState({ redactValues });
|
||||||
|
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||||
|
obs.lastSettled = {
|
||||||
|
stateId,
|
||||||
|
captured: [capturedInput()],
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
url: URL,
|
||||||
|
vomText: rawVomText,
|
||||||
|
};
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "fill",
|
||||||
|
target: { unmatched: true },
|
||||||
|
value,
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "input",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
draft.postStateId = stateId;
|
||||||
|
rememberStepOnPage(obs, draft, 1);
|
||||||
|
|
||||||
|
return (
|
||||||
|
buildTraceV3({
|
||||||
|
obs,
|
||||||
|
steps: [draft],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
}).states[0]?.body ?? ""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const body = finalizedFillBody(secret, true);
|
||||||
|
|
||||||
|
expect(body).toContain("step 1: fill");
|
||||||
|
expect(body).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps fill details in ordinary recording annotations", () => {
|
||||||
|
const value = "ordinary text";
|
||||||
|
const body = finalizedFillBody(value, false);
|
||||||
|
|
||||||
|
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("applyTargetMatching without a settled observation", () => {
|
||||||
|
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||||
|
const obs = createObservationState();
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect(draft.target).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "新建",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import {
|
|
||||||
RECORD_DOCUMENT_ATTRIBUTE,
|
|
||||||
recordingDocumentMarkerValue,
|
|
||||||
} from "@/shared/recording-document-identity";
|
|
||||||
import { markRecordingDocument } from "../recording/document-marker";
|
|
||||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
|
||||||
|
|
||||||
describe("recording document marker", () => {
|
|
||||||
it("rejects a non-random Document identity", () => {
|
|
||||||
expect(() => markRecordingDocument("user@example.com")).toThrow(
|
|
||||||
"recording Document identity must be a random UUID",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("restores the Document attribute exactly", () => {
|
|
||||||
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
|
||||||
const producerId = "123e4567-e89b-42d3-a456-426614174000";
|
|
||||||
const marker = markRecordingDocument(producerId);
|
|
||||||
|
|
||||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
|
|
||||||
recordingDocumentMarkerValue(producerId),
|
|
||||||
);
|
|
||||||
marker.restore();
|
|
||||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
|
||||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("indexes the CDP scope from the safe frame identity", () => {
|
|
||||||
const index = new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
frames: [
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
target: { tabId: 3, sessionId: "oopif-session" },
|
|
||||||
recordingDocumentId: "producer-1",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
matchNodes: [],
|
|
||||||
refs: [],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(index.documentScope("producer-1")).toEqual({
|
|
||||||
frameId: "child",
|
|
||||||
target: { tabId: 3, sessionId: "oopif-session" },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("fails closed when a recording identity maps to multiple Documents", () => {
|
|
||||||
const index = new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
frames: [
|
|
||||||
{
|
|
||||||
frameId: "left",
|
|
||||||
target: { tabId: 3, sessionId: "left-session" },
|
|
||||||
recordingDocumentId: "duplicate",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
frameId: "right",
|
|
||||||
target: { tabId: 3, sessionId: "right-session" },
|
|
||||||
recordingDocumentId: "duplicate",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
matchNodes: [],
|
|
||||||
refs: [],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(index.documentScope("duplicate")).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
|
||||||
import { RecordingObservationSession } from "../recording/observation-session";
|
|
||||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
|
||||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
|
||||||
import type { RecordingDraftStep } from "../recording/types";
|
|
||||||
|
|
||||||
const URL = "https://example.com/login";
|
|
||||||
|
|
||||||
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
|
||||||
const session = new RecordingObservationSession({ redactValues });
|
|
||||||
const state = session.registry.register({
|
|
||||||
url: URL,
|
|
||||||
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
|
||||||
});
|
|
||||||
session.cursor.lastSettled = {
|
|
||||||
stateId: state.id,
|
|
||||||
rootFrameId: "root",
|
|
||||||
index: new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
matchNodes: [
|
|
||||||
{
|
|
||||||
frameId: "root",
|
|
||||||
backendNodeId: 42,
|
|
||||||
tag: "input",
|
|
||||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
|
||||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
|
||||||
}),
|
|
||||||
url: URL,
|
|
||||||
};
|
|
||||||
return session;
|
|
||||||
}
|
|
||||||
|
|
||||||
function finalizedFillTrace(value: string, redactValues: boolean) {
|
|
||||||
const session = sessionWithInput(redactValues);
|
|
||||||
const draft: RecordingDraftStep = {
|
|
||||||
op: "fill",
|
|
||||||
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
|
||||||
value,
|
|
||||||
targetHint: {
|
|
||||||
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
|
||||||
},
|
|
||||||
};
|
|
||||||
session.bindDraft(draft, 1);
|
|
||||||
draft.postStateId = draft.preStateId;
|
|
||||||
return buildTraceV3({
|
|
||||||
registry: session.registry,
|
|
||||||
drafts: [draft],
|
|
||||||
annotations: session.annotations,
|
|
||||||
startedAt: "2026-08-12T00:00:00.000Z",
|
|
||||||
stoppedBy: "user_finish",
|
|
||||||
bskVersion: "test",
|
|
||||||
redactValues,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
|
||||||
return finalizedFillTrace(value, redactValues).states[0]!.body;
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("record observation annotations", () => {
|
|
||||||
it("omits a fill literal when values are redacted", () => {
|
|
||||||
const secret = "hunter2-private";
|
|
||||||
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
|
|
||||||
expect(dumped).toContain("step 1: fill");
|
|
||||||
expect(dumped).not.toContain(secret);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps ordinary fill details", () => {
|
|
||||||
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
|
||||||
});
|
|
||||||
|
|
||||||
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
|
||||||
const registry = new RecordingStateRegistry();
|
|
||||||
const state = registry.register({
|
|
||||||
url: "https://example.com/a\nb",
|
|
||||||
title: "hello\nworld",
|
|
||||||
vomText: "@vom 1",
|
|
||||||
});
|
|
||||||
const trace = buildTraceV3({
|
|
||||||
registry,
|
|
||||||
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
|
||||||
startedAt: "2026-08-12T00:00:00.000Z",
|
|
||||||
stoppedBy: "user_finish",
|
|
||||||
bskVersion: "test",
|
|
||||||
});
|
|
||||||
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
|
||||||
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("recording state ownership", () => {
|
|
||||||
it("deduplicates within one recording and isolates ids between recordings", () => {
|
|
||||||
const first = new RecordingStateRegistry();
|
|
||||||
const second = new RecordingStateRegistry();
|
|
||||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
|
||||||
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
|
|
||||||
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
|
||||||
const registry = new RecordingStateRegistry();
|
|
||||||
registry.register({ url: URL, vomText: "same" });
|
|
||||||
const state = registry.register({
|
|
||||||
url: URL,
|
|
||||||
title: "Login",
|
|
||||||
vomText: "same",
|
|
||||||
truncated: true,
|
|
||||||
});
|
|
||||||
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("draft binding", () => {
|
|
||||||
it("keeps capture semantics when no observation exists", () => {
|
|
||||||
const session = new RecordingObservationSession();
|
|
||||||
const draft: RecordingDraftStep = {
|
|
||||||
op: "hover",
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
|
||||||
};
|
|
||||||
session.bindDraft(draft, 1);
|
|
||||||
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
|
||||||
expect(draft.preStateId).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not bind an unmatched new action to a stale observation", () => {
|
|
||||||
const session = sessionWithInput();
|
|
||||||
const draft: RecordingDraftStep = {
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
|
||||||
targetHint: {
|
|
||||||
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
};
|
|
||||||
session.bindDraft(draft, 2, true);
|
|
||||||
expect(draft.preStateId).toBeUndefined();
|
|
||||||
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
|
||||||
role: "button",
|
|
||||||
name: "Confirm",
|
|
||||||
unmatched: true,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,289 +0,0 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
|
|
||||||
const captureRecordingObservation = vi.hoisted(() => vi.fn());
|
|
||||||
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
|
|
||||||
|
|
||||||
vi.mock("../recording/observation-capture", async (importOriginal) => {
|
|
||||||
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
|
|
||||||
return { ...actual, captureRecordingObservation };
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.mock("../recording/document-settle", async (importOriginal) => {
|
|
||||||
const actual = await importOriginal<typeof import("../recording/document-settle")>();
|
|
||||||
return { ...actual, waitForDocumentSettled };
|
|
||||||
});
|
|
||||||
|
|
||||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
|
||||||
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
|
||||||
import type { RecordingDraftStep } from "../recording/types";
|
|
||||||
|
|
||||||
function observation(url = "https://example.com/") {
|
|
||||||
return {
|
|
||||||
rootFrameId: "root",
|
|
||||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
|
||||||
url,
|
|
||||||
title: "Example",
|
|
||||||
vomText: '@vom 1\nRootWebArea "Example"',
|
|
||||||
truncated: false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function runtime(): RecordingObservationRuntime {
|
|
||||||
return new RecordingObservationRuntime({
|
|
||||||
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
|
|
||||||
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("RecordingObservationRuntime", () => {
|
|
||||||
afterEach(() => {
|
|
||||||
captureRecordingObservation.mockReset();
|
|
||||||
waitForDocumentSettled.mockClear();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("shares one initial capture and includes it in flush", async () => {
|
|
||||||
let release!: (value: ReturnType<typeof observation>) => void;
|
|
||||||
captureRecordingObservation.mockReturnValueOnce(
|
|
||||||
new Promise((resolve) => {
|
|
||||||
release = resolve;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
const first = recording.captureInitial(7);
|
|
||||||
const second = recording.captureInitial(7);
|
|
||||||
const flushed = recording.flush();
|
|
||||||
let flushCompleted = false;
|
|
||||||
void flushed.then(() => {
|
|
||||||
flushCompleted = true;
|
|
||||||
});
|
|
||||||
|
|
||||||
await Promise.resolve();
|
|
||||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
|
|
||||||
expect(flushCompleted).toBe(false);
|
|
||||||
|
|
||||||
release(observation());
|
|
||||||
await Promise.all([first, second, flushed]);
|
|
||||||
expect(flushCompleted).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("cancels an in-flight initial capture", async () => {
|
|
||||||
captureRecordingObservation.mockImplementationOnce(
|
|
||||||
(input: { signal?: AbortSignal }) =>
|
|
||||||
new Promise((_, reject) => {
|
|
||||||
input.signal?.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
const pending = recording.captureInitial(7);
|
|
||||||
recording.cancel();
|
|
||||||
|
|
||||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("retries a failed initial capture when the recording settles at stop", async () => {
|
|
||||||
captureRecordingObservation
|
|
||||||
.mockRejectedValueOnce(new Error("document swapped"))
|
|
||||||
.mockResolvedValueOnce(observation());
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
|
|
||||||
await recording.settleTrailing(7, []);
|
|
||||||
const trace = recording.buildTrace({
|
|
||||||
drafts: [],
|
|
||||||
startedAt: "2026-08-19T00:00:00.000Z",
|
|
||||||
stoppedBy: "user_finish",
|
|
||||||
bskVersion: "0.1.6",
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
|
|
||||||
expect(trace.states).toHaveLength(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps runtime matching data out of the serialized trace", async () => {
|
|
||||||
const captured = observation();
|
|
||||||
captured.index = new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
matchNodes: [
|
|
||||||
{
|
|
||||||
frameId: "root",
|
|
||||||
backendNodeId: 42,
|
|
||||||
tag: "button",
|
|
||||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
|
||||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
|
|
||||||
});
|
|
||||||
captureRecordingObservation.mockResolvedValueOnce(captured);
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
await recording.captureInitial(7);
|
|
||||||
const dumped = JSON.stringify(
|
|
||||||
recording.buildTrace({
|
|
||||||
drafts: [],
|
|
||||||
startedAt: "2026-08-19T00:00:00.000Z",
|
|
||||||
stoppedBy: "user_finish",
|
|
||||||
bskVersion: "0.1.6",
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(dumped).toContain("RootWebArea");
|
|
||||||
expect(dumped).not.toContain("matchNodes");
|
|
||||||
expect(dumped).not.toContain("localRect");
|
|
||||||
expect(dumped).not.toContain("backendNodeId");
|
|
||||||
expect(dumped).not.toContain("ObservationNodeIndex");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("captures a tab transition without sharing observation cursors", async () => {
|
|
||||||
captureRecordingObservation
|
|
||||||
.mockResolvedValueOnce(observation("https://example.com/first"))
|
|
||||||
.mockResolvedValueOnce(observation("https://example.com/second"));
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
await recording.captureInitial(4);
|
|
||||||
const transition = await recording.captureTabTransition(4, 5);
|
|
||||||
await recording.captureInitial(4);
|
|
||||||
await recording.captureInitial(5);
|
|
||||||
|
|
||||||
expect(transition).toEqual({
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s2",
|
|
||||||
targetUrl: "https://example.com/second",
|
|
||||||
});
|
|
||||||
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
|
||||||
captureRecordingObservation.mockResolvedValueOnce({
|
|
||||||
...observation(),
|
|
||||||
index: new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
frames: [
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
target: { tabId: 7, sessionId: "oopif-session" },
|
|
||||||
recordingDocumentId: "producer-1",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
matchNodes: [
|
|
||||||
{
|
|
||||||
backendNodeId: 42,
|
|
||||||
frameId: "child",
|
|
||||||
tag: "button",
|
|
||||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
|
||||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
const recording = runtime();
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{
|
|
||||||
op: "click" as const,
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
|
||||||
targetHint: {
|
|
||||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
await recording.captureInitial(7);
|
|
||||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
|
||||||
recording.cancel();
|
|
||||||
|
|
||||||
const draft = drafts[0];
|
|
||||||
expect(draft?.op).toBe("click");
|
|
||||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
|
||||||
expect(draft.targetHint).toMatchObject({
|
|
||||||
frameId: "child",
|
|
||||||
geometrySpace: "local",
|
|
||||||
});
|
|
||||||
expect(draft.matchedTarget?.ref).toBe("e1");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("refreshes a late iframe before matching its first action", async () => {
|
|
||||||
const iframeObservation = {
|
|
||||||
...observation(),
|
|
||||||
index: new ObservationNodeIndex({
|
|
||||||
rootFrameId: "root",
|
|
||||||
frames: [
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
target: { tabId: 7, sessionId: "oopif-session" },
|
|
||||||
recordingDocumentId: "producer-1",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
matchNodes: [
|
|
||||||
{
|
|
||||||
backendNodeId: 42,
|
|
||||||
frameId: "child",
|
|
||||||
tag: "button",
|
|
||||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
|
||||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
refs: [
|
|
||||||
{
|
|
||||||
ref: "e1",
|
|
||||||
backendNodeId: 42,
|
|
||||||
frameId: "child",
|
|
||||||
role: "button",
|
|
||||||
name: "表格视图",
|
|
||||||
line: 1,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
captureRecordingObservation
|
|
||||||
.mockResolvedValueOnce(observation())
|
|
||||||
.mockResolvedValueOnce(iframeObservation)
|
|
||||||
.mockResolvedValueOnce(iframeObservation);
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
await recording.captureInitial(7);
|
|
||||||
await recording.refreshDocument(7, "producer-1");
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "表格视图" },
|
|
||||||
targetHint: {
|
|
||||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
|
||||||
recording.cancel();
|
|
||||||
|
|
||||||
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
|
|
||||||
expect(waitForDocumentSettled).toHaveBeenCalledWith(
|
|
||||||
expect.anything(),
|
|
||||||
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
|
|
||||||
{ signal: expect.any(AbortSignal) },
|
|
||||||
);
|
|
||||||
const draft = drafts[0];
|
|
||||||
expect(draft?.op).toBe("click");
|
|
||||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
|
||||||
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not poison final flush when a frame readiness refresh fails", async () => {
|
|
||||||
captureRecordingObservation
|
|
||||||
.mockResolvedValueOnce(observation())
|
|
||||||
.mockRejectedValueOnce(new Error("child Document replaced"));
|
|
||||||
const recording = runtime();
|
|
||||||
|
|
||||||
await recording.captureInitial(7);
|
|
||||||
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
|
|
||||||
"child Document replaced",
|
|
||||||
);
|
|
||||||
await expect(recording.flush()).resolves.toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
+39
-62
@@ -1,9 +1,9 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
||||||
|
|
||||||
describe("recording-step-buffer", () => {
|
describe("recording-step-buffer", () => {
|
||||||
it("stores semantic click without summary", () => {
|
it("stores semantic click without summary", () => {
|
||||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
const buffer = { steps: [], pendingNavigation: false };
|
||||||
appendRecordedPayload(buffer, {
|
appendRecordedPayload(buffer, {
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
target: { tag: "button", role: "button", name: "发布" },
|
||||||
@@ -12,40 +12,37 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
|
target: { unmatched: true },
|
||||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(buffer.navigation.pendingNavigation).toBe(true);
|
expect(buffer.pendingNavigation).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps the hovered element description as a capture fallback", () => {
|
it("keeps the hovered element description as a capture fallback", () => {
|
||||||
const buffer = { steps: [], navigation: { pendingNavigation: false } };
|
const buffer = { steps: [], pendingNavigation: false };
|
||||||
appendRecordedPayload(
|
appendRecordedPayload(buffer, {
|
||||||
buffer,
|
op: "hover",
|
||||||
{
|
target: { tag: "button", role: "button", name: "新建" },
|
||||||
op: "hover",
|
geometry: {
|
||||||
target: { tag: "button", role: "button", name: "新建" },
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
geometry: {
|
scrollX: 0,
|
||||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
scrollY: 0,
|
||||||
tag: "button",
|
position: "static",
|
||||||
},
|
tag: "button",
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
geometry: {
|
|
||||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
|
||||||
tag: "button",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
);
|
|
||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
targetHint: {
|
geometry: {
|
||||||
geometry: {
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
scrollX: 0,
|
||||||
tag: "button",
|
scrollY: 0,
|
||||||
},
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
@@ -56,21 +53,19 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "click" as const,
|
op: "click" as const,
|
||||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
target: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
navigation: {
|
currentUrl: "https://example.com/a",
|
||||||
currentUrl: "https://example.com/a",
|
pendingNavigation: true,
|
||||||
pendingNavigation: true,
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
target: { tag: "button", role: "button", name: "发布" },
|
||||||
navigatedTo: "https://example.com/b",
|
navigated_to: "https://example.com/b",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||||
@@ -81,23 +76,21 @@ describe("recording-step-buffer", () => {
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
op: "select" as const,
|
op: "select" as const,
|
||||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
target: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
navigation: {
|
currentUrl: "https://example.com/list",
|
||||||
currentUrl: "https://example.com/list",
|
pendingNavigation: true,
|
||||||
pendingNavigation: true,
|
pendingNavigationDeadline: Date.now() + 5_000,
|
||||||
pendingNavigationDeadline: Date.now() + 5_000,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "select",
|
||||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
target: { tag: "select", role: "combobox", name: "分类" },
|
||||||
values: ["tech"],
|
values: ["tech"],
|
||||||
navigatedTo: "https://example.com/list?cat=tech",
|
navigated_to: "https://example.com/list?cat=tech",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -105,7 +98,8 @@ describe("recording-step-buffer", () => {
|
|||||||
it("emits navigate for uncaused URL changes", () => {
|
it("emits navigate for uncaused URL changes", () => {
|
||||||
const buffer = {
|
const buffer = {
|
||||||
steps: [],
|
steps: [],
|
||||||
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
|
currentUrl: "https://example.com/a",
|
||||||
|
pendingNavigation: false,
|
||||||
};
|
};
|
||||||
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
@@ -118,10 +112,8 @@ describe("recording-step-buffer", () => {
|
|||||||
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||||
const buffer = {
|
const buffer = {
|
||||||
steps: [],
|
steps: [],
|
||||||
navigation: {
|
currentUrl: "https://passport.example/login",
|
||||||
currentUrl: "https://passport.example/login",
|
pendingNavigation: false,
|
||||||
pendingNavigation: false,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
const hop1 = observeRecordedNavigation(
|
const hop1 = observeRecordedNavigation(
|
||||||
buffer,
|
buffer,
|
||||||
@@ -144,21 +136,6 @@ describe("recording-step-buffer", () => {
|
|||||||
url: "https://app.example/dashboard",
|
url: "https://app.example/dashboard",
|
||||||
});
|
});
|
||||||
expect(buffer.steps).toEqual([]);
|
expect(buffer.steps).toEqual([]);
|
||||||
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
|
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
|
||||||
});
|
|
||||||
|
|
||||||
it("does not reuse redirect metadata for a later content-observed URL change", () => {
|
|
||||||
const buffer = {
|
|
||||||
steps: [],
|
|
||||||
navigation: {
|
|
||||||
currentUrl: "https://example.com/redirect",
|
|
||||||
pendingNavigation: false,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
|
|
||||||
|
|
||||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
|
||||||
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
|
||||||
import { RecordingObservationSession } from "../recording/observation-session";
|
|
||||||
import { SettleController } from "../recording/settle-controller";
|
|
||||||
import type { RecordingDraftStep } from "../recording/types";
|
|
||||||
|
|
||||||
const OBSERVATION: RegisteredObservation = {
|
|
||||||
stateId: "s-next",
|
|
||||||
rootFrameId: "root",
|
|
||||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
|
||||||
url: "https://example.com/next",
|
|
||||||
};
|
|
||||||
|
|
||||||
describe("SettleController", () => {
|
|
||||||
afterEach(() => vi.useRealTimers());
|
|
||||||
|
|
||||||
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async () => ({
|
|
||||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
|
||||||
})) as unknown as CdpRunner["send"],
|
|
||||||
};
|
|
||||||
const tabsApi: ChromeTabsApi = {
|
|
||||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
|
||||||
query: vi.fn(async () => []),
|
|
||||||
};
|
|
||||||
const session = new RecordingObservationSession();
|
|
||||||
let firstSignal: AbortSignal | undefined;
|
|
||||||
vi.spyOn(session, "capture")
|
|
||||||
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
|
||||||
firstSignal = signal;
|
|
||||||
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
|
||||||
signal?.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
})
|
|
||||||
.mockResolvedValueOnce(OBSERVATION);
|
|
||||||
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
|
||||||
];
|
|
||||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
|
||||||
controller.schedule(drafts, 0);
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
|
|
||||||
drafts.push({
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "button", name: "Second" },
|
|
||||||
preStateId: "s-next",
|
|
||||||
});
|
|
||||||
controller.schedule(drafts, 1);
|
|
||||||
expect(firstSignal?.aborted).toBe(true);
|
|
||||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(500);
|
|
||||||
await controller.flush();
|
|
||||||
expect(drafts[1]?.postStateId).toBe("s-next");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async () => ({
|
|
||||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
|
||||||
})) as unknown as CdpRunner["send"],
|
|
||||||
};
|
|
||||||
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
|
||||||
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
|
||||||
resolveFirstTab = resolve;
|
|
||||||
});
|
|
||||||
const tabsApi: ChromeTabsApi = {
|
|
||||||
get: vi
|
|
||||||
.fn()
|
|
||||||
.mockImplementationOnce(() => firstTab)
|
|
||||||
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
|
||||||
query: vi.fn(async () => []),
|
|
||||||
};
|
|
||||||
const session = new RecordingObservationSession();
|
|
||||||
session.cursor.lastSettled = OBSERVATION;
|
|
||||||
const drafts: RecordingDraftStep[] = [];
|
|
||||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
|
||||||
|
|
||||||
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
|
||||||
|
|
||||||
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
|
||||||
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
|
||||||
await vi.advanceTimersByTimeAsync(500);
|
|
||||||
await controller.flushRedirects();
|
|
||||||
|
|
||||||
expect(drafts).toEqual([]);
|
|
||||||
expect(controller.hasPending).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("settles an action in its own OOPIF document scope", async () => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
const send = vi.fn();
|
|
||||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
|
||||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
|
||||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
|
||||||
});
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: send as unknown as CdpRunner["send"],
|
|
||||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
|
||||||
};
|
|
||||||
const tabsApi: ChromeTabsApi = {
|
|
||||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
|
||||||
query: vi.fn(async () => []),
|
|
||||||
};
|
|
||||||
const session = new RecordingObservationSession();
|
|
||||||
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
|
||||||
];
|
|
||||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
|
||||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
|
||||||
|
|
||||||
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
|
||||||
await vi.advanceTimersByTimeAsync(180);
|
|
||||||
await controller.flush();
|
|
||||||
|
|
||||||
expect(send).not.toHaveBeenCalled();
|
|
||||||
expect(sendToTarget).toHaveBeenCalledWith(
|
|
||||||
target,
|
|
||||||
"Runtime.evaluate",
|
|
||||||
expect.objectContaining({ contextId: 39 }),
|
|
||||||
);
|
|
||||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
|
|
||||||
|
|
||||||
describe("RecordingTabCoordinator", () => {
|
|
||||||
it("keeps navigation state isolated per tab", () => {
|
|
||||||
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
|
|
||||||
const first = tabs.navigation(4);
|
|
||||||
first.pendingNavigation = true;
|
|
||||||
const second = tabs.navigation(5, "https://example.com/second");
|
|
||||||
|
|
||||||
expect(second).toEqual({
|
|
||||||
currentUrl: "https://example.com/second",
|
|
||||||
pendingNavigation: false,
|
|
||||||
});
|
|
||||||
expect(tabs.navigation(4)).toBe(first);
|
|
||||||
expect(tabs.navigation(4).pendingNavigation).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("invalidates an earlier activation when a newer tab becomes active", () => {
|
|
||||||
const tabs = new RecordingTabCoordinator(4);
|
|
||||||
const second = tabs.noteActivation(5);
|
|
||||||
const third = tabs.noteActivation(6);
|
|
||||||
|
|
||||||
expect(tabs.isLatest(second)).toBe(false);
|
|
||||||
expect(tabs.isLatest(third)).toBe(true);
|
|
||||||
tabs.commit(6);
|
|
||||||
expect(tabs.currentTabId).toBe(6);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("commits the freshly observed URL when returning to an existing tab", () => {
|
|
||||||
const tabs = new RecordingTabCoordinator(4);
|
|
||||||
tabs.navigation(5, "https://example.com/old");
|
|
||||||
|
|
||||||
tabs.commit(5, "https://example.com/new");
|
|
||||||
|
|
||||||
expect(tabs.currentTabId).toBe(5);
|
|
||||||
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,159 +0,0 @@
|
|||||||
import type { RenderedRef } from "@browser-skill/vom";
|
|
||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
|
||||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
|
||||||
import { matchObservationTarget } from "../recording/target-matcher";
|
|
||||||
|
|
||||||
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
|
||||||
return {
|
|
||||||
backendNodeId,
|
|
||||||
frameId,
|
|
||||||
tag: "button",
|
|
||||||
rect: { x, y: 20, w: 100, h: 30 },
|
|
||||||
localRect: { x, y: 20, w: 100, h: 30 },
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function observation(
|
|
||||||
matchNodes: CaptureVomMatchNode[],
|
|
||||||
refs: RenderedRef[],
|
|
||||||
): RegisteredObservation {
|
|
||||||
return {
|
|
||||||
stateId: "s1",
|
|
||||||
rootFrameId: "root",
|
|
||||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
|
||||||
url: "https://example.com",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("matchObservationTarget", () => {
|
|
||||||
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
|
||||||
const geometry = node(42, "child");
|
|
||||||
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
|
||||||
const ref: RenderedRef = {
|
|
||||||
ref: "e1",
|
|
||||||
backendNodeId: 42,
|
|
||||||
frameId: "child",
|
|
||||||
line: 1,
|
|
||||||
};
|
|
||||||
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
|
||||||
|
|
||||||
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("matches a unique node using canonical top-level viewport geometry", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "root")],
|
|
||||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
|
||||||
),
|
|
||||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
|
||||||
});
|
|
||||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "left"), node(42, "right")],
|
|
||||||
[
|
|
||||||
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
|
||||||
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
|
||||||
],
|
|
||||||
),
|
|
||||||
hint: {
|
|
||||||
frameId: "right",
|
|
||||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
expect(target.ref).toBe("e2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
|
||||||
const child = node(42, "child", 410);
|
|
||||||
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[child],
|
|
||||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
|
||||||
),
|
|
||||||
hint: {
|
|
||||||
frameId: "child",
|
|
||||||
geometrySpace: "local",
|
|
||||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
expect(target.ref).toBe("e1");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("restricts a missing frame hint to the root frame", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "child")],
|
|
||||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
|
||||||
),
|
|
||||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
|
||||||
fallback: { tag: "button", name: "发布" },
|
|
||||||
});
|
|
||||||
expect(target).toEqual({ name: "发布", unmatched: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("fails closed when the source Document has no VOM frame mapping", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "root")],
|
|
||||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
|
||||||
),
|
|
||||||
hint: {
|
|
||||||
frameId: null,
|
|
||||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
|
||||||
},
|
|
||||||
fallback: { tag: "button", role: "button", name: "发布" },
|
|
||||||
});
|
|
||||||
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns unmatched for ambiguous geometry", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "root"), node(43, "root")],
|
|
||||||
[
|
|
||||||
{ ref: "e1", backendNodeId: 42, line: 1 },
|
|
||||||
{ ref: "e2", backendNodeId: 43, line: 2 },
|
|
||||||
],
|
|
||||||
),
|
|
||||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
|
||||||
});
|
|
||||||
expect(target.unmatched).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(41, "root"), node(42, "child")],
|
|
||||||
[
|
|
||||||
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
|
||||||
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
|
||||||
],
|
|
||||||
),
|
|
||||||
hint: { frameId: "child" },
|
|
||||||
fallback: { tag: "button", role: "button", name: "保存" },
|
|
||||||
});
|
|
||||||
expect(target.ref).toBe("e2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
|
||||||
const target = matchObservationTarget({
|
|
||||||
observation: observation(
|
|
||||||
[node(42, "root"), node(43, "root")],
|
|
||||||
[
|
|
||||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
|
||||||
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
|
||||||
],
|
|
||||||
),
|
|
||||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
|
||||||
fallback: { tag: "button", role: "button", name: "取消" },
|
|
||||||
});
|
|
||||||
expect(target.ref).toBe("e2");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,202 +1,70 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
|
||||||
import { shouldRecordPress } from "../recording/draft-policy";
|
import type { DraftTraceStep } from "@/transport/types";
|
||||||
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
|
||||||
|
|
||||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
describe("trace-reducer-v2", () => {
|
||||||
return buildTraceV2({
|
it("drops scroll and bare character press steps", () => {
|
||||||
steps,
|
const steps: DraftTraceStep[] = [
|
||||||
startedAt: "2026-01-01T00:00:00.000Z",
|
|
||||||
...(startUrl ? { startUrl } : {}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
|
||||||
it("keeps Enter and Escape", () => {
|
|
||||||
expect(shouldRecordPress("Enter")).toBe(true);
|
|
||||||
expect(shouldRecordPress("Escape")).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
|
|
||||||
expect(shouldRecordPress("Meta")).toBe(false);
|
|
||||||
expect(shouldRecordPress("c", ["meta"])).toBe(false);
|
|
||||||
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
|
|
||||||
expect(shouldRecordPress("x")).toBe(false);
|
|
||||||
expect(shouldRecordPress("中")).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
|
||||||
it("does not expose internal tab transitions in trace v2", () => {
|
|
||||||
const trace = reduceTraceSteps([
|
|
||||||
{
|
{
|
||||||
op: "switch_tab",
|
op: "scroll",
|
||||||
preStateId: "s1",
|
page_url: "https://example.com/",
|
||||||
postStateId: "s2",
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
expect(trace.steps).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{
|
|
||||||
op: "navigate",
|
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "fill",
|
|
||||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
|
||||||
value: "browser skill",
|
|
||||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
key: "Enter",
|
|
||||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
|
||||||
navigatedTo: "https://example.com/results/42",
|
|
||||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
|
||||||
navigatedTo: "https://example.com/p/99",
|
|
||||||
pageUrl: "https://example.com/results/42",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "a",
|
key: "a",
|
||||||
pageUrl: "https://example.com/p/99",
|
page_url: "https://example.com/",
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
const trace = buildTraceV2({
|
||||||
const { pages, steps } = reduceTraceSteps(
|
steps,
|
||||||
drafts,
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
startUrl: "https://example.com/",
|
||||||
);
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
|
||||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
|
||||||
"https://example.com/results/42",
|
|
||||||
"https://example.com/p/99",
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
id: 1,
|
|
||||||
op: "navigate",
|
|
||||||
page: "p1",
|
|
||||||
to: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(steps[1]).toMatchObject({
|
|
||||||
op: "fill",
|
|
||||||
page: "p1",
|
|
||||||
value: "browser skill",
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(steps[2]).toMatchObject({
|
|
||||||
op: "press",
|
|
||||||
key: "Enter",
|
|
||||||
page: "p1",
|
|
||||||
effect: { navigated_to: "p2" },
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(steps[3]).toMatchObject({
|
|
||||||
op: "click",
|
|
||||||
page: "p2",
|
|
||||||
effect: { navigated_to: "p3" },
|
|
||||||
});
|
});
|
||||||
|
expect(trace.steps).toHaveLength(0);
|
||||||
|
expect(trace.pages).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("collapses consecutive navigations", () => {
|
it("drops hover steps unsupported by historical v2 clients", () => {
|
||||||
const { steps } = reduceTraceSteps([
|
const trace = buildTraceV2({
|
||||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
steps: [
|
||||||
{ op: "navigate", url: "https://a.example/final" },
|
|
||||||
]);
|
|
||||||
expect(steps).toHaveLength(1);
|
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
op: "navigate",
|
|
||||||
to: "https://a.example/final",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps committed empty fill steps", () => {
|
|
||||||
const { steps } = reduceTraceSteps(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
op: "fill",
|
|
||||||
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
|
||||||
value: "",
|
|
||||||
pageUrl: "https://example.com/search",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"https://example.com/search",
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(steps).toEqual([
|
|
||||||
expect.objectContaining({
|
|
||||||
op: "fill",
|
|
||||||
value: "",
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
|
||||||
const { pages, steps } = reduceTraceSteps(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
op: "select",
|
|
||||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
|
||||||
values: ["tech"],
|
|
||||||
labels: ["技术"],
|
|
||||||
navigatedTo: "https://example.com/list?cat=tech",
|
|
||||||
pageUrl: "https://example.com/list",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"https://example.com/list",
|
|
||||||
);
|
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/list",
|
|
||||||
"https://example.com/list?cat=tech",
|
|
||||||
]);
|
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
op: "select",
|
|
||||||
page: "p1",
|
|
||||||
selection: [{ value: "tech", label: "技术" }],
|
|
||||||
effect: { navigated_to: "p2" },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("drops hover steps, which only exist in trace v3", () => {
|
|
||||||
const { steps } = reduceTraceSteps(
|
|
||||||
[
|
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
target: { unmatched: true },
|
||||||
pageUrl: "https://example.com/app",
|
captureTarget: { tag: "button", role: "button", name: "结束" },
|
||||||
},
|
page_url: "https://example.com/",
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
|
||||||
pageUrl: "https://example.com/app",
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
);
|
startUrl: "https://example.com/",
|
||||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
});
|
||||||
|
|
||||||
|
expect(trace.steps).toHaveLength(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("maps captureTarget to v2 target with required tag", () => {
|
||||||
expect(
|
const steps: DraftTraceStep[] = [
|
||||||
buildTraceV2({
|
{
|
||||||
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
op: "click",
|
||||||
startedAt: "2026-01-01T00:00:00.000Z",
|
target: { unmatched: true },
|
||||||
startUrl: "https://example.com/start",
|
captureTarget: { tag: "button", role: "button", name: "Submit" },
|
||||||
}).entry.start_url,
|
page_url: "https://example.com/",
|
||||||
).toBe("https://example.com/start");
|
},
|
||||||
|
];
|
||||||
|
const trace = buildTraceV2({
|
||||||
|
steps,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
startUrl: "https://example.com/",
|
||||||
|
});
|
||||||
|
expect(trace.steps).toHaveLength(1);
|
||||||
|
expect(trace.steps[0]?.op).toBe("click");
|
||||||
|
if (trace.steps[0]?.op === "click") {
|
||||||
|
expect(trace.steps[0].target.tag).toBe("button");
|
||||||
|
expect(trace.steps[0].target.name).toBe("Submit");
|
||||||
|
}
|
||||||
|
expect("version" in trace).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records Enter presses", () => {
|
||||||
|
expect(shouldRecordPress("Enter")).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,132 +0,0 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
|
||||||
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
|
||||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
|
||||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
|
||||||
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
|
||||||
import type { RecordingDraftStep } from "../recording/types";
|
|
||||||
|
|
||||||
describe("trace reducer v3", () => {
|
|
||||||
it("removes form literals when value redaction is requested", () => {
|
|
||||||
const reduced = reduceTraceStepsV3(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
op: "fill",
|
|
||||||
value: "private@example.com",
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s1",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "select",
|
|
||||||
values: ["private-account-id"],
|
|
||||||
labels: ["Private account"],
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s1",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
{ redactValues: true },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
|
|
||||||
expect(reduced.steps[1]).toMatchObject({ op: "select" });
|
|
||||||
expect(reduced.steps[1]).not.toHaveProperty("selection");
|
|
||||||
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
|
|
||||||
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("emits tab transitions only for callers that advertised support", () => {
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{
|
|
||||||
op: "switch_tab",
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s2",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
|
|
||||||
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
|
|
||||||
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps hover steps before menu clicks", () => {
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{
|
|
||||||
op: "hover",
|
|
||||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s1",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
|
||||||
preStateId: "s1",
|
|
||||||
postStateId: "s1",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not export a tab transition without both observation endpoints", () => {
|
|
||||||
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
|
|
||||||
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
|
|
||||||
|
|
||||||
expect(
|
|
||||||
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
|
|
||||||
).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
|
||||||
const drafts: RecordingDraftStep[] = [
|
|
||||||
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
|
||||||
{
|
|
||||||
op: "navigate",
|
|
||||||
url: "https://example.com/final",
|
|
||||||
transitionQualifiers: ["server_redirect"],
|
|
||||||
preStateId: "s2",
|
|
||||||
postStateId: "s3",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
const before = structuredClone(drafts);
|
|
||||||
const reduced = reduceTraceStepsV3(drafts);
|
|
||||||
|
|
||||||
expect(drafts).toEqual(before);
|
|
||||||
expect(reduced.steps).toEqual([
|
|
||||||
expect.objectContaining({
|
|
||||||
id: 1,
|
|
||||||
op: "navigate",
|
|
||||||
state: "s1",
|
|
||||||
to: "https://example.com/final",
|
|
||||||
result: { state: "s3" },
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
|
||||||
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("builds the wire model from protocol constants", () => {
|
|
||||||
const registry = new RecordingStateRegistry();
|
|
||||||
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
|
|
||||||
const trace = buildTraceV3({
|
|
||||||
registry,
|
|
||||||
drafts: [
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
|
||||||
preStateId: state.id,
|
|
||||||
postStateId: state.id,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
startedAt: "2026-08-12T00:00:00.000Z",
|
|
||||||
stoppedBy: "user_finish",
|
|
||||||
bskVersion: "test",
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(trace.version).toBe(TRACE_VERSION_V3);
|
|
||||||
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
|
||||||
expect(trace.steps[0]).toMatchObject({
|
|
||||||
op: "click",
|
|
||||||
target: { role: "button", name: "Save", unmatched: true },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { resetStateIdCounterForTests } from "@/lib/record-constants";
|
||||||
|
import {
|
||||||
|
reduceTraceSteps,
|
||||||
|
registerObservation,
|
||||||
|
resolveTraceStartUrl,
|
||||||
|
shouldRecordPress,
|
||||||
|
} from "../trace-reducer";
|
||||||
|
|
||||||
|
describe("shouldRecordPress", () => {
|
||||||
|
it("keeps Enter and Escape", () => {
|
||||||
|
expect(shouldRecordPress("Enter")).toBe(true);
|
||||||
|
expect(shouldRecordPress("Escape")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
|
||||||
|
expect(shouldRecordPress("Meta")).toBe(false);
|
||||||
|
expect(shouldRecordPress("c", ["meta"])).toBe(false);
|
||||||
|
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
|
||||||
|
expect(shouldRecordPress("x")).toBe(false);
|
||||||
|
expect(shouldRecordPress("中")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("reduceTraceSteps v3", () => {
|
||||||
|
it("builds steps with state/result references and deduped states", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const body = "@vom 1\nL1 page";
|
||||||
|
const s1 = registerObservation(registry, {
|
||||||
|
url: "https://example.com/search",
|
||||||
|
rawVomText: body,
|
||||||
|
});
|
||||||
|
const s2 = registerObservation(registry, {
|
||||||
|
url: "https://example.com/results/42",
|
||||||
|
rawVomText: "@vom 1\nL1 results",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { states, steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/search",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
value: "browser skill",
|
||||||
|
commit: "enter",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "press",
|
||||||
|
key: "Enter",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s2,
|
||||||
|
navigated_to: "https://example.com/results/42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { ref: "e2", role: "button", name: "发布" },
|
||||||
|
preStateId: s2,
|
||||||
|
postStateId: s2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(states).toHaveLength(2);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "fill",
|
||||||
|
state: s1,
|
||||||
|
result: { state: s1 },
|
||||||
|
commit: "enter",
|
||||||
|
});
|
||||||
|
expect(steps[2]).toMatchObject({
|
||||||
|
op: "press",
|
||||||
|
state: s1,
|
||||||
|
result: { state: s2 },
|
||||||
|
});
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("page");
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("effect");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("collapses consecutive navigations", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, {
|
||||||
|
url: "https://a.example/redirect1",
|
||||||
|
rawVomText: "a",
|
||||||
|
});
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
expect(steps).toHaveLength(1);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "navigate",
|
||||||
|
to: "https://a.example/final",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const start = registerObservation(registry, {
|
||||||
|
url: "https://example.com/",
|
||||||
|
rawVomText: "start",
|
||||||
|
});
|
||||||
|
const hop = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/",
|
||||||
|
rawVomText: "hop",
|
||||||
|
});
|
||||||
|
const landing = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
rawVomText: "landing",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://iwiki.woa.com/",
|
||||||
|
preStateId: start,
|
||||||
|
postStateId: hop,
|
||||||
|
transitionType: "typed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
preStateId: hop,
|
||||||
|
postStateId: landing,
|
||||||
|
transitionType: "link",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps).toHaveLength(1);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "navigate",
|
||||||
|
id: 1,
|
||||||
|
state: start,
|
||||||
|
result: { state: landing },
|
||||||
|
to: "https://iwiki.woa.com/dashboard",
|
||||||
|
cause: "user_typed",
|
||||||
|
});
|
||||||
|
// Both drafts fold into step 1, so anything tagged with either draft id
|
||||||
|
// must resolve to it.
|
||||||
|
expect(stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps draft ids to published step ids across dropped drafts", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
// Dropped: bare typing press.
|
||||||
|
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps.map((step) => step.op)).toEqual(["click"]);
|
||||||
|
expect(stepIdByDraftId.get(1)).toBeUndefined();
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps committed empty fills", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||||
|
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "Search" },
|
||||||
|
value: "",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { ref: "e2", role: "button", name: "Apply" },
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const state = registerObservation(registry, {
|
||||||
|
url: "https://example.com/app",
|
||||||
|
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
|
||||||
|
});
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { ref: "e1", role: "button", name: "Account" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { ref: "e2", role: "link", name: "Profile" },
|
||||||
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
state,
|
||||||
|
result: { state },
|
||||||
|
target: { name: "Account" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
|
expect(
|
||||||
|
resolveTraceStartUrl(
|
||||||
|
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
|
||||||
|
"https://example.com/start",
|
||||||
|
),
|
||||||
|
).toBe("https://example.com/start");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
|
|||||||
| { kind: "version_skew" }
|
| { kind: "version_skew" }
|
||||||
| { kind: "rejected"; reason: string };
|
| { kind: "rejected"; reason: string };
|
||||||
|
|
||||||
|
const UPDATE_BOTH_VERSION_MESSAGE =
|
||||||
|
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
|
||||||
|
const UPDATE_BSK_VERSION_MESSAGE =
|
||||||
|
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
|
||||||
|
const UPDATE_EXTENSION_VERSION_MESSAGE =
|
||||||
|
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
|
||||||
|
const INVALID_VERSION_MESSAGE =
|
||||||
|
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Symmetric extension-side **protocol** compat check.
|
* Symmetric extension-side **protocol** compat check.
|
||||||
*
|
*
|
||||||
@@ -339,19 +348,19 @@ export function computeConnectedState(
|
|||||||
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`,
|
reason: UPDATE_BOTH_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon protocol v${handshake.protocol_version} is unparseable`,
|
reason: INVALID_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`,
|
reason: UPDATE_BSK_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const peerFloor = handshake.min_compatible_protocol;
|
const peerFloor = handshake.min_compatible_protocol;
|
||||||
@@ -359,13 +368,13 @@ export function computeConnectedState(
|
|||||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`,
|
reason: INVALID_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`,
|
reason: UPDATE_EXTENSION_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* Build a semantic capture descriptor for an interacted element.
|
* Build a semantic TargetDescriptor for an interacted element.
|
||||||
*
|
*
|
||||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||||
* on screen (usually a short visible name). Tag-only noise like
|
* on screen (usually a short visible name). Tag-only noise like
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import type { Step } from "@/transport/types";
|
||||||
|
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||||
|
|
||||||
|
export interface ObservationAnnotation {
|
||||||
|
stepId: number;
|
||||||
|
op: Step["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||||
|
const suffix = detail ? `: ${detail}` : "";
|
||||||
|
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||||
|
export function formatObservationFile(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepsHere: number[];
|
||||||
|
body: string;
|
||||||
|
annotations?: ObservationAnnotation[];
|
||||||
|
}): string {
|
||||||
|
const lines: string[] = [
|
||||||
|
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||||
|
`state: ${input.stateId}`,
|
||||||
|
`url: ${input.url}`,
|
||||||
|
];
|
||||||
|
if (input.title) lines.push(`title: ${input.title}`);
|
||||||
|
if (input.stepsHere.length > 0) {
|
||||||
|
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||||
|
}
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const bodyLines = input.body.split("\n");
|
||||||
|
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||||
|
for (const ann of input.annotations ?? []) {
|
||||||
|
const bucket = annotationMap.get(ann.line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
annotationMap.set(ann.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||||
|
let line = bodyLines[i] ?? "";
|
||||||
|
const anns = annotationMap.get(i);
|
||||||
|
if (anns) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
line += formatAnnotation(ann);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hash input: VOM body **before** annotations are inserted. */
|
||||||
|
export function observationBodyForHash(vomText: string): string {
|
||||||
|
return vomText;
|
||||||
|
}
|
||||||
@@ -16,13 +16,6 @@ export const HELP_ACK = "bsk-help-ack";
|
|||||||
export const HELP_QUERY = "bsk-help-query";
|
export const HELP_QUERY = "bsk-help-query";
|
||||||
export const HELP_FINISH = "bsk-help-finish";
|
export const HELP_FINISH = "bsk-help-finish";
|
||||||
|
|
||||||
export interface HelpHighlightRect {
|
|
||||||
top: number;
|
|
||||||
left: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface HelpRequestMessage {
|
export interface HelpRequestMessage {
|
||||||
type: typeof HELP_REQUEST;
|
type: typeof HELP_REQUEST;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
@@ -33,8 +26,6 @@ export interface HelpRequestMessage {
|
|||||||
displayMode?: "full" | "compact";
|
displayMode?: "full" | "compact";
|
||||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
|
||||||
rects?: HelpHighlightRect[];
|
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,16 +74,6 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
|||||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||||
Array.isArray(m.selectors) &&
|
Array.isArray(m.selectors) &&
|
||||||
m.selectors.every((selector) => typeof selector === "string") &&
|
m.selectors.every((selector) => typeof selector === "string") &&
|
||||||
(m.rects === undefined ||
|
|
||||||
(Array.isArray(m.rects) &&
|
|
||||||
m.rects.every(
|
|
||||||
(rect) =>
|
|
||||||
typeof rect === "object" &&
|
|
||||||
rect !== null &&
|
|
||||||
["top", "left", "width", "height"].every((key) =>
|
|
||||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
|
||||||
),
|
|
||||||
))) &&
|
|
||||||
typeof m.timeoutMs === "number"
|
typeof m.timeoutMs === "number"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||||
|
|
||||||
|
export interface MatchTargetInput {
|
||||||
|
geometry: CaptureGeometry;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
refs: RenderedRef[];
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}
|
||||||
|
|
||||||
|
function withinTolerance(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectsMatch(
|
||||||
|
a: { x: number; y: number; w: number; h: number },
|
||||||
|
b: { x: number; y: number; w: number; h: number },
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
withinTolerance(a.x, b.x) &&
|
||||||
|
withinTolerance(a.y, b.y) &&
|
||||||
|
withinTolerance(a.w, b.w) &&
|
||||||
|
withinTolerance(a.h, b.h)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeViewportRect(
|
||||||
|
node: CapturedNode,
|
||||||
|
): { x: number; y: number; w: number; h: number } | null {
|
||||||
|
return node.localRect ?? node.rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchingRects(
|
||||||
|
geometry: CaptureGeometry,
|
||||||
|
node: CapturedNode,
|
||||||
|
): {
|
||||||
|
target: { x: number; y: number; w: number; h: number };
|
||||||
|
node: { x: number; y: number; w: number; h: number } | null;
|
||||||
|
} {
|
||||||
|
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
||||||
|
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||||
|
if (!topFrame || viewportPosition || !node.documentRect) {
|
||||||
|
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
target: {
|
||||||
|
x: geometry.rect.x + geometry.scrollX,
|
||||||
|
y: geometry.rect.y + geometry.scrollY,
|
||||||
|
w: geometry.rect.w,
|
||||||
|
h: geometry.rect.h,
|
||||||
|
},
|
||||||
|
node: node.documentRect,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||||
|
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best description available when the element cannot be located in the VOM. */
|
||||||
|
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
|
||||||
|
if (!fallback) {
|
||||||
|
return { unmatched: true };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...(fallback.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Locate the interacted element in the last settled observation by geometry. */
|
||||||
|
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
|
||||||
|
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||||
|
|
||||||
|
const candidates = input.captured.filter((node) => {
|
||||||
|
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||||
|
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||||
|
const rects = matchingRects(input.geometry, node);
|
||||||
|
if (!rects.node) return false;
|
||||||
|
return rectsMatch(rects.target, rects.node);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (candidates.length !== 1) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backendNodeId = candidates[0]!.backendNodeId;
|
||||||
|
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||||
|
if (!refEntry) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ref: refEntry.ref,
|
||||||
|
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||||
|
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||||
|
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// Deciding *when* a page is worth observing.
|
||||||
|
//
|
||||||
|
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||||
|
// route change that renders for a second. Instead of guessing, ask the page:
|
||||||
|
// a document that has stopped mutating and is no longer loading is done
|
||||||
|
// reacting to whatever the user just did.
|
||||||
|
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Installed once per document. Records only the timestamp of the last DOM
|
||||||
|
* change so each poll stays O(1) no matter how large the page is.
|
||||||
|
*/
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export type SettleOutcome =
|
||||||
|
/** The page stopped changing on its own. */
|
||||||
|
| "quiet"
|
||||||
|
/** Still changing when the budget ran out; observe it as it is. */
|
||||||
|
| "timeout"
|
||||||
|
/** A newer action took over; this observation is no longer wanted. */
|
||||||
|
| "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||||
|
// not a reason to give up.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||||
|
export async function waitForPageSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS);
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, tabId);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe) continue;
|
||||||
|
if (probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,10 +16,6 @@ export interface RecordStartAck {
|
|||||||
ok: true;
|
ok: true;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RecordStepAck =
|
|
||||||
| { ok: true; sequence: number }
|
|
||||||
| { ok: false; expectedSequence: number; error: string };
|
|
||||||
|
|
||||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||||
|
|
||||||
export interface RecordQueryMessage {
|
export interface RecordQueryMessage {
|
||||||
@@ -45,7 +41,7 @@ export interface RecordStartMessage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
|
||||||
target?: CaptureTargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
@@ -57,10 +53,14 @@ export interface RecordStepPayload {
|
|||||||
commit?: "enter" | "suggestion" | "blur";
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
/** Page URL when the step was captured. */
|
/** Page URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
/** Event-target bounds in the source frame's viewport coordinate space. */
|
/** Document geometry at capture time for background geometric matching. */
|
||||||
geometry?: {
|
geometry?: {
|
||||||
rect: { x: number; y: number; w: number; h: number };
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
position: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
|
ownerFrameBackendNodeId?: number | null;
|
||||||
};
|
};
|
||||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||||
expects_navigation?: boolean;
|
expects_navigation?: boolean;
|
||||||
@@ -74,10 +74,6 @@ export interface RecordStepPayload {
|
|||||||
export interface RecordStepMessage {
|
export interface RecordStepMessage {
|
||||||
type: typeof RECORD_STEP;
|
type: typeof RECORD_STEP;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
/** Stable for the lifetime of one content-script document. */
|
|
||||||
producerId: string;
|
|
||||||
/** Monotonic within one content-script producer and recording request. */
|
|
||||||
sequence: number;
|
|
||||||
step: RecordStepPayload;
|
step: RecordStepPayload;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,27 +125,8 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
|||||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
if (
|
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
||||||
m.type !== RECORD_STEP ||
|
|
||||||
typeof m.requestId !== "string" ||
|
|
||||||
typeof m.producerId !== "string" ||
|
|
||||||
m.producerId.length === 0 ||
|
|
||||||
typeof m.sequence !== "number" ||
|
|
||||||
!Number.isSafeInteger(m.sequence) ||
|
|
||||||
m.sequence < 1
|
|
||||||
) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
const step = m.step;
|
const step = m.step;
|
||||||
if (typeof step !== "object" || step === null) return false;
|
if (typeof step !== "object" || step === null) return false;
|
||||||
return typeof (step as RecordStepPayload).op === "string";
|
return typeof (step as RecordStepPayload).op === "string";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isAcceptedRecordStepAck(
|
|
||||||
value: unknown,
|
|
||||||
sequence: number,
|
|
||||||
): value is Extract<RecordStepAck, { ok: true }> {
|
|
||||||
if (typeof value !== "object" || value === null) return false;
|
|
||||||
const ack = value as Partial<RecordStepAck>;
|
|
||||||
return ack.ok === true && ack.sequence === sequence;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Floor before a post-action observation may be taken. A page usually has not
|
||||||
|
* started reacting in the first frames after an action, and capturing then
|
||||||
|
* would record the page as it was, not as the action left it.
|
||||||
|
*/
|
||||||
|
export const SETTLE_MIN_MS = 150;
|
||||||
|
|
||||||
|
/** How long the DOM must stop changing before a page counts as settled. */
|
||||||
|
export const SETTLE_QUIET_MS = 250;
|
||||||
|
|
||||||
|
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||||
|
export const SETTLE_MAX_MS = 2_000;
|
||||||
|
|
||||||
|
/** How often to ask the page whether it has gone quiet. */
|
||||||
|
export const SETTLE_POLL_MS = 60;
|
||||||
|
|
||||||
|
/** Minimum interval between consecutive observations on the same tab. */
|
||||||
|
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
/** Delay before retrying a capture that lost its execution context. */
|
||||||
|
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||||
|
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||||
|
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||||
|
|
||||||
|
/** VOM observation file format version (front matter header). */
|
||||||
|
export const OBSERVATION_FILE_VERSION = 1;
|
||||||
|
|
||||||
|
/** FNV-1a 64-bit offset basis. */
|
||||||
|
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||||
|
/** FNV-1a 64-bit prime. */
|
||||||
|
const FNV_PRIME = 0x100000001b3n;
|
||||||
|
|
||||||
|
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||||
|
export function fnv1a64(text: string): string {
|
||||||
|
let hash = FNV_OFFSET;
|
||||||
|
for (let i = 0; i < text.length; i += 1) {
|
||||||
|
hash ^= BigInt(text.charCodeAt(i));
|
||||||
|
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||||
|
}
|
||||||
|
return hash.toString(16).padStart(16, "0");
|
||||||
|
}
|
||||||
|
|
||||||
|
let nextStateSerial = 0;
|
||||||
|
|
||||||
|
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||||
|
export function nextStateId(): string {
|
||||||
|
nextStateSerial += 1;
|
||||||
|
return `s${nextStateSerial}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seam: reset the state id counter. */
|
||||||
|
export function resetStateIdCounterForTests(): void {
|
||||||
|
nextStateSerial = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
|
||||||
|
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface NavigationTransitionMeta {
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
navigationActionPending?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||||
|
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||||
|
const qualifiers = meta.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
|
||||||
|
const type = meta.transitionType ?? "";
|
||||||
|
const mapped = TRANSITION_TO_CAUSE[type];
|
||||||
|
if (mapped) return mapped;
|
||||||
|
|
||||||
|
if (!type && meta.navigationActionPending === false) return "script";
|
||||||
|
return "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||||
@@ -0,0 +1,663 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||||
|
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||||
|
import { waitForPageSettled } from "@/lib/page-settled";
|
||||||
|
import {
|
||||||
|
CAPTURE_RETRY_DELAY_MS,
|
||||||
|
DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
OBSERVATION_MIN_INTERVAL_MS,
|
||||||
|
} from "@/lib/record-constants";
|
||||||
|
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||||
|
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
|
||||||
|
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||||
|
|
||||||
|
export interface LastSettledObservation {
|
||||||
|
stateId: string;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
refs: RenderedRef[];
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A post-action observation that has been queued but not yet written down. */
|
||||||
|
interface PendingSettle {
|
||||||
|
draftIndex: number;
|
||||||
|
/** Set when a newer action has already decided where this step landed. */
|
||||||
|
cancelled: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||||
|
export interface PendingRedirectLanding {
|
||||||
|
url: string;
|
||||||
|
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingObservationState {
|
||||||
|
stateRegistry: Map<string, StateRegistryEntry>;
|
||||||
|
lastSettled: LastSettledObservation | null;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
lastCaptureAtMs: number;
|
||||||
|
/**
|
||||||
|
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||||
|
* capture can never land after a faster one and report the pages out of the
|
||||||
|
* order the user visited them.
|
||||||
|
*/
|
||||||
|
settleQueue: Promise<void>;
|
||||||
|
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||||
|
settles: Map<number, PendingSettle>;
|
||||||
|
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||||
|
/**
|
||||||
|
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||||
|
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||||
|
*/
|
||||||
|
pendingRedirect: PendingRedirectLanding | null;
|
||||||
|
redirectFlushQueue: Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createObservationState(options?: {
|
||||||
|
maxPageTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
}): RecordingObservationState {
|
||||||
|
return {
|
||||||
|
stateRegistry: new Map(),
|
||||||
|
lastSettled: null,
|
||||||
|
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues: options?.redactValues ?? false,
|
||||||
|
lastCaptureAtMs: 0,
|
||||||
|
settleQueue: Promise.resolve(),
|
||||||
|
settles: new Map(),
|
||||||
|
stepAnnotations: new Map(),
|
||||||
|
pendingRedirect: null,
|
||||||
|
redirectFlushQueue: Promise.resolve(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureAndRegisterObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
const now = Date.now();
|
||||||
|
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||||
|
if (waitMs > 0) await sleep(waitMs);
|
||||||
|
|
||||||
|
const { url, title } = urlOverride
|
||||||
|
? { url: urlOverride, title: undefined }
|
||||||
|
: await readTabMeta(tabsApi, tabId);
|
||||||
|
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||||
|
const resolvedTitle = title ?? meta.title;
|
||||||
|
|
||||||
|
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||||
|
maxTokens: obs.maxPageTokens,
|
||||||
|
redactValues: obs.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, {
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
rawVomText: rendered.text,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
});
|
||||||
|
|
||||||
|
obs.lastCaptureAtMs = Date.now();
|
||||||
|
const settled: LastSettledObservation = {
|
||||||
|
stateId,
|
||||||
|
captured: rendered.captured,
|
||||||
|
refs: rendered.refs,
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
vomText: rendered.text,
|
||||||
|
};
|
||||||
|
obs.lastSettled = settled;
|
||||||
|
return settled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||||
|
* match, and the inline annotation. All three read the *current* observation,
|
||||||
|
* which is only the page the user acted on while the draft is still fresh —
|
||||||
|
* once the action settles, `lastSettled` is the destination page instead.
|
||||||
|
*/
|
||||||
|
export function applyTargetMatching(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
if (!obs.lastSettled) {
|
||||||
|
// Initial observation can fail or race the first action. Keep whatever
|
||||||
|
// the content script captured so the exported step is still teachable.
|
||||||
|
if ("target" in draft && "captureTarget" in draft) {
|
||||||
|
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
draft.preStateId = obs.lastSettled.stateId;
|
||||||
|
|
||||||
|
if ("target" in draft) {
|
||||||
|
draft.target =
|
||||||
|
"geometry" in draft && draft.geometry
|
||||||
|
? matchTarget({
|
||||||
|
geometry: draft.geometry,
|
||||||
|
captured: obs.lastSettled.captured,
|
||||||
|
refs: obs.lastSettled.refs,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: // No geometry to match on, but the capture still knew what the user
|
||||||
|
// touched — keep that instead of an anonymous unmatched target.
|
||||||
|
fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fillDetailForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): string | undefined {
|
||||||
|
if (obs.redactValues) return undefined;
|
||||||
|
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function refLineForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): number | undefined {
|
||||||
|
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||||
|
const targetRef = draft.target?.ref;
|
||||||
|
if (!targetRef) return undefined;
|
||||||
|
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rememberStepAnnotation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepId: number,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): void {
|
||||||
|
const line = refLineForDraft(obs, draft);
|
||||||
|
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||||
|
if (!draft.preStateId) return;
|
||||||
|
const ann: ObservationAnnotation = {
|
||||||
|
stepId,
|
||||||
|
op: draft.op,
|
||||||
|
line,
|
||||||
|
stateId: draft.preStateId,
|
||||||
|
detail: fillDetailForDraft(obs, draft),
|
||||||
|
};
|
||||||
|
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
obs.stepAnnotations.set(line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record the step under the page it was performed on, not the one it led to. */
|
||||||
|
export function rememberStepOnPage(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
const stateId = draft.preStateId;
|
||||||
|
if (!stateId) return;
|
||||||
|
const entry = obs.stateRegistry.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||||
|
entry.stepsHere.push(stepId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||||
|
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||||
|
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||||
|
*/
|
||||||
|
export function scheduleRedirectLandingFlush(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
url: string,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): void {
|
||||||
|
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||||
|
obs.pendingRedirect = { url, generation };
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain any coalesced redirect so the next action matches against the real
|
||||||
|
* landing page. Safe to call when nothing is pending.
|
||||||
|
*/
|
||||||
|
export async function flushPendingRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner | undefined,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
if (obs.pendingRedirect && cdp) {
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await obs.redirectFlushQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait until the redirect chain stops changing the document, then emit one
|
||||||
|
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||||
|
*/
|
||||||
|
async function coalesceRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean },
|
||||||
|
): Promise<void> {
|
||||||
|
while (obs.pendingRedirect) {
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const snap = obs.pendingRedirect;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||||
|
cancelled: () =>
|
||||||
|
!!options.cancelled?.() ||
|
||||||
|
obs.pendingRedirect === null ||
|
||||||
|
obs.pendingRedirect.generation !== snap.generation,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||||
|
if (outcome === "cancelled") continue;
|
||||||
|
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
|
||||||
|
if (!finalUrl || finalUrl === "about:blank") return;
|
||||||
|
if (obs.lastSettled?.url === finalUrl) return;
|
||||||
|
|
||||||
|
const last = steps[steps.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) {
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||||
|
await obs.settleQueue;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||||
|
`recording navigate to ${finalUrl}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
page_url: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
preStateId: obs.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
steps.push(draft);
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
const stepId = draftIndex + 1;
|
||||||
|
rememberStepOnPage(obs, draft, stepId);
|
||||||
|
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||||
|
// Callers that flush before the next action need `lastSettled` to already
|
||||||
|
// be the landing page so target matching does not use the pre-redirect view.
|
||||||
|
await obs.settleQueue;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||||
|
* That is transient, so give it one more chance before the step has to fall
|
||||||
|
* back to a stale state.
|
||||||
|
*/
|
||||||
|
async function captureWithRetry(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
try {
|
||||||
|
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk record] observation failed, retrying once", err);
|
||||||
|
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||||
|
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function settleDraftObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
pending?: PendingSettle,
|
||||||
|
): Promise<void> {
|
||||||
|
const cancelled = () => pending?.cancelled === true;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||||
|
"budget; observing it as it stands",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const started = steps[draftIndex];
|
||||||
|
if (!started) return;
|
||||||
|
|
||||||
|
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||||
|
// Once the page is settled, register the capture against live tab metadata.
|
||||||
|
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
// The observation still counts as the latest view of the page even when this
|
||||||
|
// step no longer wants it — the next action will start from it.
|
||||||
|
if (cancelled()) return;
|
||||||
|
// Re-read the slot: a navigation observed while the capture ran may have
|
||||||
|
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||||
|
// slot now — writing to the object we started with would strand it.
|
||||||
|
const draft = steps[draftIndex] ?? started;
|
||||||
|
draft.postStateId = settled.stateId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover landing states from the shape of the recording itself: wherever the
|
||||||
|
* next action was performed is, by definition, where the previous one landed.
|
||||||
|
* Only the immediately following draft counts — a later one would claim a page
|
||||||
|
* that several unobserved actions away, which is worse than admitting we saw
|
||||||
|
* no change.
|
||||||
|
*/
|
||||||
|
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||||
|
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
const next = steps[i + 1];
|
||||||
|
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||||
|
`using where step ${i + 2} started (${next.preStateId})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last chance for the closing actions of a recording. Stopping right after the
|
||||||
|
* final action is normal, and so is that action navigating away, so take one
|
||||||
|
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||||
|
* did not land on whatever the user happens to be looking at when they stop.
|
||||||
|
*/
|
||||||
|
export async function settleUnsettledDrafts(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): Promise<void> {
|
||||||
|
const trailing: DraftTraceStep[] = [];
|
||||||
|
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
|
||||||
|
let settled: LastSettledObservation;
|
||||||
|
try {
|
||||||
|
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||||
|
`(${settled.stateId} ${settled.url})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performing a new action ends the previous one: the page the user reached for
|
||||||
|
* is, by definition, where the earlier step left them. Taking the landing from
|
||||||
|
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||||
|
* would otherwise credit an earlier step with a page that only exists because
|
||||||
|
* of the newer action.
|
||||||
|
*/
|
||||||
|
function supersedeEarlierSettles(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
const landing = steps[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of obs.settles) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.cancelled = true;
|
||||||
|
obs.settles.delete(index);
|
||||||
|
|
||||||
|
const draft = steps[index];
|
||||||
|
if (!draft || draft.postStateId) continue;
|
||||||
|
if (!landing) continue;
|
||||||
|
draft.postStateId = landing;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||||
|
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scheduleDraftSettle(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
stepId: number,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||||
|
|
||||||
|
// Rescheduling the same step (a navigation showed up after the action)
|
||||||
|
// replaces the pending observation rather than racing it.
|
||||||
|
const superseded = obs.settles.get(draftIndex);
|
||||||
|
if (superseded) superseded.cancelled = true;
|
||||||
|
|
||||||
|
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||||
|
obs.settles.set(draftIndex, pending);
|
||||||
|
|
||||||
|
enqueueSettle(obs, async () => {
|
||||||
|
if (pending.cancelled) return;
|
||||||
|
try {
|
||||||
|
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||||
|
} catch (err) {
|
||||||
|
// Recoverable: stop-time repair still gives the step a landing page.
|
||||||
|
const op = steps[draftIndex]?.op ?? "?";
|
||||||
|
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||||
|
} finally {
|
||||||
|
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||||
|
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||||
|
obs.settles.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||||
|
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain settle work before exporting, including work queued while draining —
|
||||||
|
* a navigation observed during the last capture schedules another one, and
|
||||||
|
* the trace would drop it if stop did not wait.
|
||||||
|
*/
|
||||||
|
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||||
|
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||||
|
const drained = obs.settleQueue;
|
||||||
|
await drained;
|
||||||
|
if (obs.settleQueue === drained) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeStateBodies(
|
||||||
|
entries: StateRegistryEntry[],
|
||||||
|
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
idByOldId: Map<string, string>,
|
||||||
|
): TraceState[] {
|
||||||
|
return entries.map((entry) => {
|
||||||
|
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||||
|
const body = formatObservationFile({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||||
|
body: entry.rawVomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body,
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Draft ids only become step ids after collapsing and filtering. */
|
||||||
|
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
const mapped = new Set<number>();
|
||||||
|
for (const draftId of draftIds) {
|
||||||
|
const stepId = stepIdByDraftId.get(draftId);
|
||||||
|
if (stepId !== undefined) mapped.add(stepId);
|
||||||
|
}
|
||||||
|
return [...mapped].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectAnnotationsByState(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
): Map<string, ObservationAnnotation[]> {
|
||||||
|
const byState = new Map<string, ObservationAnnotation[]>();
|
||||||
|
for (const anns of obs.stepAnnotations.values()) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byState.get(ann.stateId) ?? [];
|
||||||
|
bucket.push({ ...ann, stepId });
|
||||||
|
byState.set(ann.stateId, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byState;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep only the pages the published steps point at. Redirect hops and
|
||||||
|
* mid-load captures land in the registry too, and shipping them would invite
|
||||||
|
* a reader to treat a page the flow merely passed through as a real stop.
|
||||||
|
* With no steps at all the first observation is the whole artifact, so it
|
||||||
|
* stays.
|
||||||
|
*/
|
||||||
|
function selectPublishedStates(
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
steps: Step[],
|
||||||
|
): StateRegistryEntry[] {
|
||||||
|
const entries = [...registry.values()];
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set<string>();
|
||||||
|
for (const step of steps) {
|
||||||
|
referenced.add(step.state);
|
||||||
|
referenced.add(step.result.state);
|
||||||
|
}
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
obs: RecordingObservationState;
|
||||||
|
steps: DraftTraceStep[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
}): Trace {
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||||
|
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||||
|
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||||
|
// dropped captures used to be.
|
||||||
|
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
for (const step of steps) {
|
||||||
|
step.state = idByOldId.get(step.state) ?? step.state;
|
||||||
|
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||||
|
}
|
||||||
|
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||||
|
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||||
|
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||||
|
return {
|
||||||
|
version: 3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: startUrl },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
|
||||||
|
import type { RecordStepPayload } from "./record-bridge";
|
||||||
|
import { mapNavigationCause } from "./record-constants";
|
||||||
|
|
||||||
|
export interface RecordingStepBuffer {
|
||||||
|
steps: DraftTraceStep[];
|
||||||
|
currentUrl?: string;
|
||||||
|
pendingNavigation: boolean;
|
||||||
|
pendingNavigationDeadline?: number;
|
||||||
|
lastTransitionType?: string;
|
||||||
|
lastTransitionQualifiers?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
|
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||||
|
const pageUrl = payload.page_url;
|
||||||
|
const geometry = payload.geometry;
|
||||||
|
switch (payload.op) {
|
||||||
|
case "click":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "click",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "hover":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "fill":
|
||||||
|
return payload.target
|
||||||
|
? {
|
||||||
|
op: "fill",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
|
value: payload.value ?? "",
|
||||||
|
...(payload.commit ? { commit: payload.commit } : {}),
|
||||||
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "press":
|
||||||
|
return payload.key
|
||||||
|
? {
|
||||||
|
op: "press",
|
||||||
|
key: payload.key,
|
||||||
|
...(payload.target
|
||||||
|
? { target: { unmatched: true }, captureTarget: payload.target }
|
||||||
|
: {}),
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "select":
|
||||||
|
return payload.target && payload.values
|
||||||
|
? {
|
||||||
|
op: "select",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
|
values: payload.values,
|
||||||
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
|
||||||
|
case "navigate":
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
|
||||||
|
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||||
|
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||||
|
const step = buffer.steps[i];
|
||||||
|
if (!step) continue;
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
// Mutate in place: an in-flight settle holds this object, and replacing
|
||||||
|
// it would strand that settle's observation on the discarded copy.
|
||||||
|
step.navigated_to = url;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
|
||||||
|
return mapNavigationCause({
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Outcome of folding a main-frame URL change into the draft buffer. */
|
||||||
|
export type NavigationObserveResult =
|
||||||
|
| { kind: "noop" }
|
||||||
|
| { kind: "annotated"; index: number }
|
||||||
|
| { kind: "appended"; index: number }
|
||||||
|
/**
|
||||||
|
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
|
||||||
|
* emits one `navigate` after `waitForPageSettled` on the final URL.
|
||||||
|
*/
|
||||||
|
| { kind: "coalesce_redirect"; url: string };
|
||||||
|
|
||||||
|
export function observeRecordedNavigation(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
url: string,
|
||||||
|
causedByAction?: boolean,
|
||||||
|
transitionType?: string,
|
||||||
|
transitionQualifiers?: string[],
|
||||||
|
): NavigationObserveResult {
|
||||||
|
if (!url || url === buffer.currentUrl) return { kind: "noop" };
|
||||||
|
buffer.currentUrl = url;
|
||||||
|
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
|
||||||
|
if (transitionQualifiers !== undefined) {
|
||||||
|
buffer.lastTransitionQualifiers = transitionQualifiers;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pendingIsCurrent =
|
||||||
|
buffer.pendingNavigation &&
|
||||||
|
(buffer.pendingNavigationDeadline === undefined ||
|
||||||
|
buffer.pendingNavigationDeadline >= Date.now());
|
||||||
|
|
||||||
|
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||||
|
buffer.pendingNavigation = false;
|
||||||
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
|
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||||
|
if (annotatedIndex >= 0) {
|
||||||
|
return { kind: "annotated", index: annotatedIndex };
|
||||||
|
}
|
||||||
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.pendingNavigation = false;
|
||||||
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Buffer one captured payload. Returns the index of the draft it appended, or
|
||||||
|
* `null` when the payload produced none (an unnamed click target, a navigation
|
||||||
|
* that only annotated the previous draft, …). Callers must not fall back to
|
||||||
|
* "the last draft" on `null`: that would re-match an already recorded step
|
||||||
|
* against a newer observation and overwrite its origin state.
|
||||||
|
*/
|
||||||
|
export function appendRecordedPayload(
|
||||||
|
buffer: RecordingStepBuffer,
|
||||||
|
payload: RecordStepPayload,
|
||||||
|
): number | null {
|
||||||
|
if (payload.op === "navigate") {
|
||||||
|
if (!payload.url) return null;
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
payload.url,
|
||||||
|
payload.navigation_caused_by_action,
|
||||||
|
payload.transitionType,
|
||||||
|
payload.transitionQualifiers,
|
||||||
|
);
|
||||||
|
return result.kind === "appended" ? result.index : null;
|
||||||
|
}
|
||||||
|
const step = toDraftStep({
|
||||||
|
...payload,
|
||||||
|
page_url: payload.page_url ?? buffer.currentUrl,
|
||||||
|
});
|
||||||
|
if (!step) return null;
|
||||||
|
buffer.steps.push(step);
|
||||||
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
|
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||||
|
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
||||||
|
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
return buffer.steps.length - 1;
|
||||||
|
}
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import {
|
|
||||||
RECORD_DOCUMENT_ATTRIBUTE,
|
|
||||||
recordingDocumentMarkerValue,
|
|
||||||
} from "@/shared/recording-document-identity";
|
|
||||||
|
|
||||||
export interface RecordingDocumentMarker {
|
|
||||||
restore(): void;
|
|
||||||
ensure(): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function waitForDocumentElement(): Promise<HTMLElement> {
|
|
||||||
if (document.documentElement) return Promise.resolve(document.documentElement);
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
const observer = new MutationObserver(() => {
|
|
||||||
if (!document.documentElement) return;
|
|
||||||
observer.disconnect();
|
|
||||||
resolve(document.documentElement);
|
|
||||||
});
|
|
||||||
observer.observe(document, { childList: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function markRecordingDocument(
|
|
||||||
producerId: string,
|
|
||||||
root: HTMLElement = document.documentElement,
|
|
||||||
): RecordingDocumentMarker {
|
|
||||||
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
|
||||||
const markerValue = recordingDocumentMarkerValue(producerId);
|
|
||||||
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
|
|
||||||
ensure();
|
|
||||||
return {
|
|
||||||
ensure,
|
|
||||||
restore() {
|
|
||||||
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
|
|
||||||
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
|
||||||
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
|
||||||
|
|
||||||
const SETTLE_MIN_MS = 150;
|
|
||||||
const SETTLE_QUIET_MS = 250;
|
|
||||||
const SETTLE_MAX_MS = 2_000;
|
|
||||||
const SETTLE_POLL_MS = 60;
|
|
||||||
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
|
||||||
|
|
||||||
const QUIET_PROBE = `(() => {
|
|
||||||
const scope = window;
|
|
||||||
let probe = scope.__bskRecordQuiet;
|
|
||||||
if (!probe) {
|
|
||||||
probe = { changedAt: Date.now() };
|
|
||||||
const observer = new MutationObserver(() => {
|
|
||||||
probe.changedAt = Date.now();
|
|
||||||
});
|
|
||||||
observer.observe(document.documentElement, {
|
|
||||||
subtree: true,
|
|
||||||
childList: true,
|
|
||||||
attributes: true,
|
|
||||||
characterData: true,
|
|
||||||
});
|
|
||||||
scope.__bskRecordQuiet = probe;
|
|
||||||
}
|
|
||||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
|
||||||
})()`;
|
|
||||||
|
|
||||||
export interface DocumentSettleScope {
|
|
||||||
target: CdpTarget;
|
|
||||||
/** Omit only for the target's root document. */
|
|
||||||
frameId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
|
||||||
|
|
||||||
interface QuietProbe {
|
|
||||||
idleMs: number;
|
|
||||||
readyState: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ProbeContext {
|
|
||||||
executionContextId?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
if (signal?.aborted) return resolve();
|
|
||||||
const timer = setTimeout(resolve, ms);
|
|
||||||
signal?.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function executionContextForScope(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
scope: DocumentSettleScope,
|
|
||||||
): Promise<number | undefined> {
|
|
||||||
if (!scope.frameId) return undefined;
|
|
||||||
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
|
||||||
cdp,
|
|
||||||
scope.target,
|
|
||||||
"Page.createIsolatedWorld",
|
|
||||||
{
|
|
||||||
frameId: scope.frameId,
|
|
||||||
worldName: SETTLE_WORLD_NAME,
|
|
||||||
grantUniveralAccess: false,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
return result.executionContextId;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readQuietProbe(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
scope: DocumentSettleScope,
|
|
||||||
context: ProbeContext,
|
|
||||||
): Promise<QuietProbe | null> {
|
|
||||||
try {
|
|
||||||
if (scope.frameId && context.executionContextId === undefined) {
|
|
||||||
context.executionContextId = await executionContextForScope(cdp, scope);
|
|
||||||
if (context.executionContextId === undefined) return null;
|
|
||||||
}
|
|
||||||
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
|
||||||
cdp,
|
|
||||||
scope.target,
|
|
||||||
"Runtime.evaluate",
|
|
||||||
{
|
|
||||||
expression: QUIET_PROBE,
|
|
||||||
returnByValue: true,
|
|
||||||
...(context.executionContextId !== undefined
|
|
||||||
? { contextId: context.executionContextId }
|
|
||||||
: {}),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const value = reply.result?.value;
|
|
||||||
if (!value || typeof value !== "object") return null;
|
|
||||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
|
||||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
|
||||||
return { idleMs, readyState };
|
|
||||||
} catch {
|
|
||||||
context.executionContextId = undefined;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function waitForDocumentSettled(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
scope: DocumentSettleScope,
|
|
||||||
options: { signal?: AbortSignal } = {},
|
|
||||||
): Promise<SettleOutcome> {
|
|
||||||
const startedAt = Date.now();
|
|
||||||
const floor = startedAt + SETTLE_MIN_MS;
|
|
||||||
const deadline = startedAt + SETTLE_MAX_MS;
|
|
||||||
const context: ProbeContext = {};
|
|
||||||
|
|
||||||
for (;;) {
|
|
||||||
if (options.signal?.aborted) return "cancelled";
|
|
||||||
await sleep(SETTLE_POLL_MS, options.signal);
|
|
||||||
if (options.signal?.aborted) return "cancelled";
|
|
||||||
|
|
||||||
const probe = await readQuietProbe(cdp, scope, context);
|
|
||||||
const now = Date.now();
|
|
||||||
if (now < floor) continue;
|
|
||||||
if (now >= deadline) return "timeout";
|
|
||||||
if (!probe || probe.readyState === "loading") continue;
|
|
||||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
import type { RecordingDraftStep } from "./types";
|
|
||||||
|
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
|
||||||
|
|
||||||
export function shouldRecordPress(
|
|
||||||
key: string,
|
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
|
||||||
): boolean {
|
|
||||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
|
||||||
const mods = modifiers ?? [];
|
|
||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
|
||||||
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveDraftStartUrl(
|
|
||||||
drafts: RecordingDraftStep[],
|
|
||||||
explicitStartUrl?: string,
|
|
||||||
fallbackUrl?: string,
|
|
||||||
): string {
|
|
||||||
if (explicitStartUrl) return explicitStartUrl;
|
|
||||||
const navigation = drafts.find(
|
|
||||||
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
|
||||||
);
|
|
||||||
if (navigation) return navigation.url;
|
|
||||||
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
export const RECORD_FRAME_PORT = "bsk-record-frame";
|
|
||||||
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
|
|
||||||
export const RECORD_FRAME_START = "bsk-record-frame-start";
|
|
||||||
|
|
||||||
export interface RecordFrameQueryMessage {
|
|
||||||
type: typeof RECORD_FRAME_QUERY;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordFrameQueryResponse {
|
|
||||||
active: boolean;
|
|
||||||
requestId?: string;
|
|
||||||
startedAtMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordFrameStartMessage {
|
|
||||||
type: typeof RECORD_FRAME_START;
|
|
||||||
requestId: string;
|
|
||||||
startedAtMs: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type RecordFramePortMessage =
|
|
||||||
| {
|
|
||||||
type: "ready";
|
|
||||||
requestId: string;
|
|
||||||
producerId: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "ready_ack";
|
|
||||||
requestId: string;
|
|
||||||
producerId: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "stop";
|
|
||||||
requestId: string;
|
|
||||||
commandId: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "cancel";
|
|
||||||
requestId: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "stopped";
|
|
||||||
requestId: string;
|
|
||||||
commandId: string;
|
|
||||||
ok: boolean;
|
|
||||||
error?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
|
|
||||||
return (
|
|
||||||
typeof value === "object" &&
|
|
||||||
value !== null &&
|
|
||||||
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
|
|
||||||
if (typeof value !== "object" || value === null) return false;
|
|
||||||
const message = value as Partial<RecordFrameStartMessage>;
|
|
||||||
return (
|
|
||||||
message.type === RECORD_FRAME_START &&
|
|
||||||
typeof message.requestId === "string" &&
|
|
||||||
typeof message.startedAtMs === "number"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,399 +0,0 @@
|
|||||||
import {
|
|
||||||
isRecordFrameQueryMessage,
|
|
||||||
RECORD_FRAME_PORT,
|
|
||||||
RECORD_FRAME_START,
|
|
||||||
type RecordFramePortMessage,
|
|
||||||
type RecordFrameQueryResponse,
|
|
||||||
type RecordFrameStartMessage,
|
|
||||||
} from "./frame-bridge";
|
|
||||||
|
|
||||||
export interface RecordingCaptureScope {
|
|
||||||
tabId: number;
|
|
||||||
documentId: string;
|
|
||||||
browserFrameId: number;
|
|
||||||
producerId: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ArmedRecording {
|
|
||||||
requestId: string;
|
|
||||||
startedAtMs: number;
|
|
||||||
tabIds: Set<number>;
|
|
||||||
agents: Map<string, FrameAgent>;
|
|
||||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
|
|
||||||
finishing: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FrameAgent extends RecordingCaptureScope {
|
|
||||||
requestId: string;
|
|
||||||
port: chrome.runtime.Port;
|
|
||||||
stopWaiters: Map<string, (ok: boolean) => void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BrowserFrame {
|
|
||||||
frameId: number;
|
|
||||||
documentId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BrowserFrameNavigation extends BrowserFrame {
|
|
||||||
tabId: number;
|
|
||||||
lifecycle: "committed" | "completed";
|
|
||||||
}
|
|
||||||
|
|
||||||
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
|
|
||||||
|
|
||||||
export interface RecordFrameCoordinatorDeps {
|
|
||||||
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
|
|
||||||
sendToDocument(
|
|
||||||
tabId: number,
|
|
||||||
message: RecordFrameStartMessage,
|
|
||||||
target: { documentId?: string; frameId?: number },
|
|
||||||
): Promise<unknown>;
|
|
||||||
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
function documentKey(tabId: number, documentId: string): string {
|
|
||||||
return `${tabId}:${documentId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function senderAddress(
|
|
||||||
sender: chrome.runtime.MessageSender,
|
|
||||||
): { tabId: number; documentId: string; browserFrameId: number } | null {
|
|
||||||
const tabId = sender.tab?.id;
|
|
||||||
const documentId = sender.documentId;
|
|
||||||
const browserFrameId = sender.frameId;
|
|
||||||
if (
|
|
||||||
typeof tabId !== "number" ||
|
|
||||||
typeof documentId !== "string" ||
|
|
||||||
typeof browserFrameId !== "number"
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return { tabId, documentId, browserFrameId };
|
|
||||||
}
|
|
||||||
|
|
||||||
function defaultDeps(): RecordFrameCoordinatorDeps {
|
|
||||||
return {
|
|
||||||
async getAllFrames(tabId) {
|
|
||||||
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
|
|
||||||
},
|
|
||||||
sendToDocument(tabId, message, target) {
|
|
||||||
return chrome.tabs.sendMessage(tabId, message, target);
|
|
||||||
},
|
|
||||||
subscribeFrameNavigation(listener) {
|
|
||||||
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
|
|
||||||
listener({ ...details, lifecycle: "committed" });
|
|
||||||
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
|
|
||||||
listener({ ...details, lifecycle: "completed" });
|
|
||||||
chrome.webNavigation.onCommitted.addListener(onCommitted);
|
|
||||||
chrome.webNavigation.onCompleted.addListener(onCompleted);
|
|
||||||
return () => {
|
|
||||||
chrome.webNavigation.onCommitted.removeListener(onCommitted);
|
|
||||||
chrome.webNavigation.onCompleted.removeListener(onCompleted);
|
|
||||||
};
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RecordFrameCoordinator {
|
|
||||||
readonly #deps: RecordFrameCoordinatorDeps;
|
|
||||||
readonly #recordings = new Map<string, ArmedRecording>();
|
|
||||||
#attached = false;
|
|
||||||
#detachFrameNavigation: (() => void) | null = null;
|
|
||||||
|
|
||||||
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
|
|
||||||
this.#deps = deps;
|
|
||||||
}
|
|
||||||
|
|
||||||
attach(): () => void {
|
|
||||||
if (this.#attached) return () => {};
|
|
||||||
this.#attached = true;
|
|
||||||
chrome.runtime.onConnect.addListener(this.#onConnect);
|
|
||||||
chrome.runtime.onMessage.addListener(this.#onMessage);
|
|
||||||
this.#detachFrameNavigation =
|
|
||||||
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
|
|
||||||
return () => {
|
|
||||||
if (!this.#attached) return;
|
|
||||||
this.#attached = false;
|
|
||||||
chrome.runtime.onConnect.removeListener(this.#onConnect);
|
|
||||||
chrome.runtime.onMessage.removeListener(this.#onMessage);
|
|
||||||
this.#detachFrameNavigation?.();
|
|
||||||
this.#detachFrameNavigation = null;
|
|
||||||
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
|
|
||||||
this.#recordings.clear();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
begin(
|
|
||||||
requestId: string,
|
|
||||||
startedAtMs: number,
|
|
||||||
initialTabId: number,
|
|
||||||
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
|
|
||||||
): void {
|
|
||||||
const previous = this.#recordings.get(requestId);
|
|
||||||
if (previous) this.#cancelAgents(previous);
|
|
||||||
this.#recordings.set(requestId, {
|
|
||||||
requestId,
|
|
||||||
startedAtMs,
|
|
||||||
// Register the initial tab before navigation begins. A newly-created
|
|
||||||
// child document can query at document_start before armTab's frame
|
|
||||||
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
|
|
||||||
tabIds: new Set([initialTabId]),
|
|
||||||
agents: new Map(),
|
|
||||||
...(onDocumentReady ? { onDocumentReady } : {}),
|
|
||||||
finishing: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async armTab(requestId: string, tabId: number): Promise<boolean> {
|
|
||||||
const recording = this.#recordings.get(requestId);
|
|
||||||
if (!recording || recording.finishing) return false;
|
|
||||||
recording.tabIds.add(tabId);
|
|
||||||
|
|
||||||
let frames: BrowserFrame[];
|
|
||||||
try {
|
|
||||||
frames = await this.#deps.getAllFrames(tabId);
|
|
||||||
} catch {
|
|
||||||
frames = [{ frameId: 0 }];
|
|
||||||
}
|
|
||||||
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
|
|
||||||
|
|
||||||
const results = await Promise.all(
|
|
||||||
frames.map(async (frame) => {
|
|
||||||
const started = await this.#startFrame(recording, tabId, frame);
|
|
||||||
return { frameId: frame.frameId, started };
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return results.some((result) => result.frameId === 0 && result.started);
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceFor(
|
|
||||||
requestId: string,
|
|
||||||
producerId: string,
|
|
||||||
sender: chrome.runtime.MessageSender,
|
|
||||||
): RecordingCaptureScope | null {
|
|
||||||
const address = senderAddress(sender);
|
|
||||||
const recording = this.#recordings.get(requestId);
|
|
||||||
if (!address || !recording) return null;
|
|
||||||
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
|
|
||||||
if (!agent || agent.producerId !== producerId) return null;
|
|
||||||
return {
|
|
||||||
tabId: agent.tabId,
|
|
||||||
documentId: agent.documentId,
|
|
||||||
browserFrameId: agent.browserFrameId,
|
|
||||||
producerId: agent.producerId,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async stop(requestId: string): Promise<boolean> {
|
|
||||||
const recording = this.#recordings.get(requestId);
|
|
||||||
if (!recording) return true;
|
|
||||||
recording.finishing = true;
|
|
||||||
const commandId = crypto.randomUUID();
|
|
||||||
const results = await Promise.all(
|
|
||||||
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
|
|
||||||
);
|
|
||||||
const succeeded = results.every(Boolean);
|
|
||||||
if (succeeded) this.#recordings.delete(requestId);
|
|
||||||
else recording.finishing = false;
|
|
||||||
return succeeded;
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel(requestId: string): void {
|
|
||||||
const recording = this.#recordings.get(requestId);
|
|
||||||
if (!recording) return;
|
|
||||||
this.#cancelAgents(recording);
|
|
||||||
this.#recordings.delete(requestId);
|
|
||||||
}
|
|
||||||
|
|
||||||
readonly #onMessage = (
|
|
||||||
message: unknown,
|
|
||||||
sender: chrome.runtime.MessageSender,
|
|
||||||
sendResponse: (response: RecordFrameQueryResponse) => void,
|
|
||||||
): boolean => {
|
|
||||||
if (!isRecordFrameQueryMessage(message)) return false;
|
|
||||||
const tabId = sender.tab?.id;
|
|
||||||
const recording =
|
|
||||||
typeof tabId === "number"
|
|
||||||
? [...this.#recordings.values()].find(
|
|
||||||
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
|
|
||||||
)
|
|
||||||
: undefined;
|
|
||||||
sendResponse(
|
|
||||||
recording
|
|
||||||
? {
|
|
||||||
active: true,
|
|
||||||
requestId: recording.requestId,
|
|
||||||
startedAtMs: recording.startedAtMs,
|
|
||||||
}
|
|
||||||
: { active: false },
|
|
||||||
);
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
|
|
||||||
if (frame.frameId === 0) return;
|
|
||||||
for (const recording of this.#recordings.values()) {
|
|
||||||
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
|
|
||||||
// onCommitted can race document_start, while onCompleted runs after the
|
|
||||||
// content script is available. Starting twice is safe and closes both
|
|
||||||
// the early-query and late-SPA-frame gaps.
|
|
||||||
void (async () => {
|
|
||||||
const started = await this.#startFrame(recording, frame.tabId, frame);
|
|
||||||
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
|
|
||||||
const agent = frame.documentId
|
|
||||||
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
|
|
||||||
: [...recording.agents.values()].find(
|
|
||||||
(candidate) =>
|
|
||||||
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
|
|
||||||
);
|
|
||||||
if (!agent) return;
|
|
||||||
await recording.onDocumentReady?.({
|
|
||||||
tabId: agent.tabId,
|
|
||||||
documentId: agent.documentId,
|
|
||||||
browserFrameId: agent.browserFrameId,
|
|
||||||
producerId: agent.producerId,
|
|
||||||
});
|
|
||||||
})();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
readonly #onConnect = (port: chrome.runtime.Port): void => {
|
|
||||||
if (port.name !== RECORD_FRAME_PORT) return;
|
|
||||||
const address = senderAddress(port.sender ?? {});
|
|
||||||
if (!address) {
|
|
||||||
port.disconnect();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let registered: FrameAgent | null = null;
|
|
||||||
port.onMessage.addListener((raw: unknown) => {
|
|
||||||
const message = raw as Partial<RecordFramePortMessage>;
|
|
||||||
if (message.type === "ready") {
|
|
||||||
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
|
|
||||||
port.disconnect();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const requestId = message.requestId;
|
|
||||||
const producerId = message.producerId;
|
|
||||||
if (registered) {
|
|
||||||
if (registered.requestId === requestId && registered.producerId === producerId) {
|
|
||||||
port.postMessage({
|
|
||||||
type: "ready_ack",
|
|
||||||
requestId,
|
|
||||||
producerId,
|
|
||||||
} satisfies RecordFramePortMessage);
|
|
||||||
} else {
|
|
||||||
port.disconnect();
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const recording = this.#recordings.get(requestId);
|
|
||||||
if (
|
|
||||||
!recording ||
|
|
||||||
recording.finishing ||
|
|
||||||
!recording.tabIds.has(address.tabId) ||
|
|
||||||
producerId.length === 0
|
|
||||||
) {
|
|
||||||
port.disconnect();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const agent: FrameAgent = {
|
|
||||||
...address,
|
|
||||||
requestId,
|
|
||||||
producerId,
|
|
||||||
port,
|
|
||||||
stopWaiters: new Map(),
|
|
||||||
};
|
|
||||||
registered = agent;
|
|
||||||
const key = documentKey(address.tabId, address.documentId);
|
|
||||||
const previous = recording.agents.get(key);
|
|
||||||
if (previous && previous !== agent) previous.port.disconnect();
|
|
||||||
recording.agents.set(key, agent);
|
|
||||||
port.postMessage({
|
|
||||||
type: "ready_ack",
|
|
||||||
requestId,
|
|
||||||
producerId,
|
|
||||||
} satisfies RecordFramePortMessage);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
message.type === "stopped" &&
|
|
||||||
registered &&
|
|
||||||
message.requestId === registered.requestId &&
|
|
||||||
typeof message.commandId === "string"
|
|
||||||
) {
|
|
||||||
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
|
|
||||||
registered.stopWaiters.delete(message.commandId);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
port.onDisconnect.addListener(() => {
|
|
||||||
if (!registered) return;
|
|
||||||
for (const resolve of registered.stopWaiters.values()) resolve(true);
|
|
||||||
const recording = this.#recordings.get(registered.requestId);
|
|
||||||
const key = documentKey(registered.tabId, registered.documentId);
|
|
||||||
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
const timer = setTimeout(() => {
|
|
||||||
agent.stopWaiters.delete(commandId);
|
|
||||||
resolve(false);
|
|
||||||
}, RECORD_FRAME_STOP_TIMEOUT_MS);
|
|
||||||
agent.stopWaiters.set(commandId, (ok) => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve(ok);
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
agent.port.postMessage({
|
|
||||||
type: "stop",
|
|
||||||
requestId,
|
|
||||||
commandId,
|
|
||||||
} satisfies RecordFramePortMessage);
|
|
||||||
} catch {
|
|
||||||
clearTimeout(timer);
|
|
||||||
agent.stopWaiters.delete(commandId);
|
|
||||||
resolve(true);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async #startFrame(
|
|
||||||
recording: ArmedRecording,
|
|
||||||
tabId: number,
|
|
||||||
frame: BrowserFrame,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const message: RecordFrameStartMessage = {
|
|
||||||
type: RECORD_FRAME_START,
|
|
||||||
requestId: recording.requestId,
|
|
||||||
startedAtMs: recording.startedAtMs,
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
const response = await this.#deps.sendToDocument(tabId, message, {
|
|
||||||
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
|
|
||||||
});
|
|
||||||
return isStarted(response);
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#cancelAgents(recording: ArmedRecording): void {
|
|
||||||
for (const agent of recording.agents.values()) {
|
|
||||||
try {
|
|
||||||
agent.port.postMessage({
|
|
||||||
type: "cancel",
|
|
||||||
requestId: recording.requestId,
|
|
||||||
} satisfies RecordFramePortMessage);
|
|
||||||
} catch {
|
|
||||||
// A destroyed Document has no remaining capture state to cancel.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isStarted(value: unknown): boolean {
|
|
||||||
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const recordFrameCoordinator = new RecordFrameCoordinator();
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
|
||||||
|
|
||||||
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
|
|
||||||
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
|
||||||
}
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
import type { RenderedRef } from "@browser-skill/vom";
|
|
||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import {
|
|
||||||
type CaptureVomMatchNode,
|
|
||||||
type CaptureVomObservationResult,
|
|
||||||
captureVomObservation,
|
|
||||||
} from "@/tools/capture-vom-observation";
|
|
||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
|
|
||||||
export interface IndexedObservationNode {
|
|
||||||
frameId: string;
|
|
||||||
geometry: CaptureVomMatchNode;
|
|
||||||
ref?: RenderedRef;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CapturedRecordingObservation {
|
|
||||||
rootFrameId: string;
|
|
||||||
index: ObservationNodeIndex;
|
|
||||||
url: string;
|
|
||||||
title?: string;
|
|
||||||
vomText: string;
|
|
||||||
truncated: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RegisteredObservation {
|
|
||||||
stateId: string;
|
|
||||||
rootFrameId: string;
|
|
||||||
index: ObservationNodeIndex;
|
|
||||||
url: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordingDocumentScope {
|
|
||||||
frameId: string;
|
|
||||||
target: CdpTarget;
|
|
||||||
}
|
|
||||||
|
|
||||||
function nodeKey(frameId: string, backendNodeId: number): string {
|
|
||||||
return `${frameId}:${backendNodeId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function frameTagKey(frameId: string, tag: string): string {
|
|
||||||
return `${frameId}:${tag.toLowerCase()}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ObservationNodeIndex {
|
|
||||||
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
|
||||||
readonly #refById = new Map<string, RenderedRef>();
|
|
||||||
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
|
||||||
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
|
|
||||||
Partial<Pick<CaptureVomObservationResult, "frames">>,
|
|
||||||
) {
|
|
||||||
const refByNode = new Map<string, RenderedRef>();
|
|
||||||
for (const ref of input.refs) {
|
|
||||||
const frameId = ref.frameId ?? input.rootFrameId;
|
|
||||||
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
|
||||||
this.#refById.set(ref.ref, ref);
|
|
||||||
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
|
||||||
frameRefs.push(ref);
|
|
||||||
this.#refsByFrame.set(frameId, frameRefs);
|
|
||||||
}
|
|
||||||
for (const frame of input.frames ?? []) {
|
|
||||||
const documentId = frame.recordingDocumentId;
|
|
||||||
if (!documentId) continue;
|
|
||||||
this.#scopeByProducer.set(
|
|
||||||
documentId,
|
|
||||||
this.#scopeByProducer.has(documentId)
|
|
||||||
? null
|
|
||||||
: { frameId: frame.frameId, target: frame.target },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
for (const geometry of input.matchNodes) {
|
|
||||||
const { frameId } = geometry;
|
|
||||||
const entry = {
|
|
||||||
frameId,
|
|
||||||
geometry,
|
|
||||||
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
|
||||||
};
|
|
||||||
const key = frameTagKey(frameId, geometry.tag);
|
|
||||||
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
|
||||||
bucket.push(entry);
|
|
||||||
this.#nodesByFrameTag.set(key, bucket);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
|
||||||
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
|
||||||
}
|
|
||||||
|
|
||||||
ref(refId: string): RenderedRef | undefined {
|
|
||||||
return this.#refById.get(refId);
|
|
||||||
}
|
|
||||||
|
|
||||||
refs(frameId: string): readonly RenderedRef[] {
|
|
||||||
return this.#refsByFrame.get(frameId) ?? [];
|
|
||||||
}
|
|
||||||
|
|
||||||
documentScope(producerId: string): RecordingDocumentScope | undefined {
|
|
||||||
return this.#scopeByProducer.get(producerId) ?? undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readTabMeta(
|
|
||||||
tabsApi: ChromeTabsApi,
|
|
||||||
tabId: number,
|
|
||||||
): Promise<{ url: string; title?: string }> {
|
|
||||||
try {
|
|
||||||
const tab = await tabsApi.get(tabId);
|
|
||||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
|
||||||
} catch {
|
|
||||||
return { url: "about:blank" };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function captureRecordingObservation(input: {
|
|
||||||
cdp: CdpRunner;
|
|
||||||
tabsApi: ChromeTabsApi;
|
|
||||||
tabId: number;
|
|
||||||
maxTokens: number;
|
|
||||||
redactValues: boolean;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}): Promise<CapturedRecordingObservation> {
|
|
||||||
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
|
||||||
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
|
||||||
maxTokens: input.maxTokens,
|
|
||||||
redactValues: input.redactValues,
|
|
||||||
conditionalSurfaceProbe: false,
|
|
||||||
signal: input.signal,
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
rootFrameId: captured.rootFrameId,
|
|
||||||
index: new ObservationNodeIndex(captured),
|
|
||||||
url,
|
|
||||||
title,
|
|
||||||
vomText: captured.text,
|
|
||||||
truncated: captured.truncated,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
|
||||||
import { RecordingStateRegistry } from "./state-registry";
|
|
||||||
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
|
||||||
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
|
||||||
|
|
||||||
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
|
||||||
const MIN_CAPTURE_INTERVAL_MS = 200;
|
|
||||||
|
|
||||||
export interface TabObservationCursor {
|
|
||||||
lastSettled: RegisteredObservation | null;
|
|
||||||
lastCaptureAt: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
if (signal?.aborted) return resolve();
|
|
||||||
const timer = setTimeout(resolve, ms);
|
|
||||||
signal?.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
|
||||||
return (
|
|
||||||
draft.op === "click" ||
|
|
||||||
draft.op === "hover" ||
|
|
||||||
draft.op === "fill" ||
|
|
||||||
draft.op === "press" ||
|
|
||||||
draft.op === "select"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RecordingObservationSession {
|
|
||||||
readonly registry: RecordingStateRegistry;
|
|
||||||
readonly cursor: TabObservationCursor;
|
|
||||||
readonly annotations: StepAnnotation[];
|
|
||||||
readonly #maxTokens: number;
|
|
||||||
readonly #redactValues: boolean;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
options: {
|
|
||||||
registry?: RecordingStateRegistry;
|
|
||||||
cursor?: TabObservationCursor;
|
|
||||||
annotations?: StepAnnotation[];
|
|
||||||
maxTokens?: number;
|
|
||||||
redactValues?: boolean;
|
|
||||||
} = {},
|
|
||||||
) {
|
|
||||||
this.registry = options.registry ?? new RecordingStateRegistry();
|
|
||||||
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
|
||||||
this.annotations = options.annotations ?? [];
|
|
||||||
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
|
||||||
this.#redactValues = options.redactValues ?? false;
|
|
||||||
}
|
|
||||||
|
|
||||||
async capture(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabsApi: ChromeTabsApi,
|
|
||||||
tabId: number,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<RegisteredObservation> {
|
|
||||||
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
|
||||||
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
|
||||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
|
||||||
const captured = await captureRecordingObservation({
|
|
||||||
cdp,
|
|
||||||
tabsApi,
|
|
||||||
tabId,
|
|
||||||
maxTokens: this.#maxTokens,
|
|
||||||
redactValues: this.#redactValues,
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
const state = this.registry.register({
|
|
||||||
url: captured.url,
|
|
||||||
title: captured.title,
|
|
||||||
vomText: captured.vomText,
|
|
||||||
truncated: captured.truncated,
|
|
||||||
});
|
|
||||||
const observation: RegisteredObservation = {
|
|
||||||
stateId: state.id,
|
|
||||||
rootFrameId: captured.rootFrameId,
|
|
||||||
index: captured.index,
|
|
||||||
url: captured.url,
|
|
||||||
};
|
|
||||||
this.cursor.lastSettled = observation;
|
|
||||||
this.cursor.lastCaptureAt = Date.now();
|
|
||||||
return observation;
|
|
||||||
}
|
|
||||||
|
|
||||||
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
|
||||||
const observation = this.cursor.lastSettled;
|
|
||||||
if (isTargeted(draft)) {
|
|
||||||
draft.matchedTarget = observation
|
|
||||||
? matchObservationTarget({
|
|
||||||
observation,
|
|
||||||
hint: draft.targetHint,
|
|
||||||
fallback: draft.captureTarget,
|
|
||||||
})
|
|
||||||
: unmatchedTarget(draft.captureTarget);
|
|
||||||
}
|
|
||||||
if (!observation) return;
|
|
||||||
|
|
||||||
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
|
||||||
if (previousActionPending && unmatched) return;
|
|
||||||
draft.preStateId = observation.stateId;
|
|
||||||
this.registry.markStep(observation.stateId, draftId);
|
|
||||||
|
|
||||||
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
|
||||||
const ref = observation.index.ref(draft.matchedTarget.ref);
|
|
||||||
if (!ref) return;
|
|
||||||
this.annotations.push({
|
|
||||||
draftId,
|
|
||||||
op: draft.op,
|
|
||||||
line: ref.line,
|
|
||||||
stateId: observation.stateId,
|
|
||||||
...(draft.op === "fill" && !this.#redactValues
|
|
||||||
? { detail: JSON.stringify(draft.value) }
|
|
||||||
: {}),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,277 +0,0 @@
|
|||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
import type { StopReason, TraceV3 } from "@/transport/types";
|
|
||||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
|
||||||
import type { RegisteredObservation } from "./observation-capture";
|
|
||||||
import { RecordingObservationSession } from "./observation-session";
|
|
||||||
import { inferMissingPostStates, SettleController } from "./settle-controller";
|
|
||||||
import { RecordingStateRegistry } from "./state-registry";
|
|
||||||
import { buildTraceV3 } from "./trace-builder-v3";
|
|
||||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
|
||||||
|
|
||||||
interface TabRecordingContext {
|
|
||||||
session: RecordingObservationSession;
|
|
||||||
settle: SettleController;
|
|
||||||
pendingCapture: PendingCapture | null;
|
|
||||||
frameRefresh: Promise<void>;
|
|
||||||
frameRefreshAbort: AbortController | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PendingCapture {
|
|
||||||
promise: Promise<RegisteredObservation>;
|
|
||||||
abort: AbortController;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RecordingObservationRuntime {
|
|
||||||
readonly #cdp: CdpRunner;
|
|
||||||
readonly #tabsApi: ChromeTabsApi;
|
|
||||||
readonly #registry = new RecordingStateRegistry();
|
|
||||||
readonly #annotations: StepAnnotation[] = [];
|
|
||||||
readonly #contexts = new Map<number, TabRecordingContext>();
|
|
||||||
readonly #maxTokens?: number;
|
|
||||||
readonly #redactValues: boolean;
|
|
||||||
|
|
||||||
constructor(input: {
|
|
||||||
cdp: CdpRunner;
|
|
||||||
tabsApi: ChromeTabsApi;
|
|
||||||
maxTokens?: number;
|
|
||||||
redactValues?: boolean;
|
|
||||||
}) {
|
|
||||||
this.#cdp = input.cdp;
|
|
||||||
this.#tabsApi = input.tabsApi;
|
|
||||||
this.#maxTokens = input.maxTokens;
|
|
||||||
this.#redactValues = input.redactValues ?? false;
|
|
||||||
}
|
|
||||||
|
|
||||||
#context(tabId: number): TabRecordingContext {
|
|
||||||
const existing = this.#contexts.get(tabId);
|
|
||||||
if (existing) return existing;
|
|
||||||
const session = new RecordingObservationSession({
|
|
||||||
registry: this.#registry,
|
|
||||||
annotations: this.#annotations,
|
|
||||||
maxTokens: this.#maxTokens,
|
|
||||||
redactValues: this.#redactValues,
|
|
||||||
});
|
|
||||||
const context: TabRecordingContext = {
|
|
||||||
session,
|
|
||||||
settle: new SettleController({
|
|
||||||
session,
|
|
||||||
cdp: this.#cdp,
|
|
||||||
tabsApi: this.#tabsApi,
|
|
||||||
tabId,
|
|
||||||
}),
|
|
||||||
pendingCapture: null,
|
|
||||||
frameRefresh: Promise.resolve(),
|
|
||||||
frameRefreshAbort: null,
|
|
||||||
};
|
|
||||||
this.#contexts.set(tabId, context);
|
|
||||||
return context;
|
|
||||||
}
|
|
||||||
|
|
||||||
async #capture(tabId: number): Promise<RegisteredObservation> {
|
|
||||||
const context = this.#context(tabId);
|
|
||||||
if (context.pendingCapture) return context.pendingCapture.promise;
|
|
||||||
|
|
||||||
const abort = new AbortController();
|
|
||||||
let pending!: PendingCapture;
|
|
||||||
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
|
|
||||||
pending = {
|
|
||||||
abort,
|
|
||||||
promise: promise.finally(() => {
|
|
||||||
if (context.pendingCapture === pending) context.pendingCapture = null;
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
context.pendingCapture = pending;
|
|
||||||
return pending.promise;
|
|
||||||
}
|
|
||||||
|
|
||||||
async captureInitial(tabId: number): Promise<void> {
|
|
||||||
const context = this.#context(tabId);
|
|
||||||
if (context.session.cursor.lastSettled) return;
|
|
||||||
await this.#capture(tabId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Refresh the safe observation after a late child Document becomes usable. */
|
|
||||||
async refreshDocument(tabId: number, producerId: string): Promise<void> {
|
|
||||||
const context = this.#context(tabId);
|
|
||||||
context.frameRefreshAbort?.abort();
|
|
||||||
const abort = new AbortController();
|
|
||||||
context.frameRefreshAbort = abort;
|
|
||||||
const previous = context.frameRefresh;
|
|
||||||
const refresh = (async () => {
|
|
||||||
await previous;
|
|
||||||
if (abort.signal.aborted) return;
|
|
||||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
|
||||||
if (abort.signal.aborted) return;
|
|
||||||
|
|
||||||
// The first capture discovers the safe producer -> CDP Document scope.
|
|
||||||
await this.#capture(tabId);
|
|
||||||
if (abort.signal.aborted) return;
|
|
||||||
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
|
|
||||||
if (scope) {
|
|
||||||
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
|
|
||||||
}
|
|
||||||
if (abort.signal.aborted) return;
|
|
||||||
|
|
||||||
// Capture again after the child SPA is quiet so the next action can be
|
|
||||||
// matched against refs that actually exist in its pre-action state.
|
|
||||||
await this.#capture(tabId);
|
|
||||||
})();
|
|
||||||
const tracked = refresh.catch(() => {});
|
|
||||||
context.frameRefresh = tracked;
|
|
||||||
try {
|
|
||||||
await refresh;
|
|
||||||
} finally {
|
|
||||||
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async captureTabTransition(
|
|
||||||
fromTabId: number,
|
|
||||||
toTabId: number,
|
|
||||||
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
|
|
||||||
const from = this.#context(fromTabId);
|
|
||||||
await this.#flushContext(from);
|
|
||||||
if (!from.session.cursor.lastSettled) {
|
|
||||||
try {
|
|
||||||
await this.captureInitial(fromTabId);
|
|
||||||
} catch {
|
|
||||||
// A target-tab observation can still preserve the forward transition.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const to = this.#context(toTabId);
|
|
||||||
let target = to.session.cursor.lastSettled;
|
|
||||||
try {
|
|
||||||
target = await this.#capture(toTabId);
|
|
||||||
} catch {
|
|
||||||
// Reuse the last settled target state when a refresh races navigation.
|
|
||||||
}
|
|
||||||
|
|
||||||
const source = from.session.cursor.lastSettled;
|
|
||||||
return {
|
|
||||||
...(source ? { preStateId: source.stateId } : {}),
|
|
||||||
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
bindTabTransition(
|
|
||||||
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
|
|
||||||
draftId: number,
|
|
||||||
): void {
|
|
||||||
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
|
|
||||||
}
|
|
||||||
|
|
||||||
async processDraft(
|
|
||||||
tabId: number,
|
|
||||||
drafts: RecordingDraftStep[],
|
|
||||||
draftIndex: number,
|
|
||||||
producerId?: string,
|
|
||||||
): Promise<void> {
|
|
||||||
const draft = drafts[draftIndex];
|
|
||||||
if (!draft) return;
|
|
||||||
const context = this.#context(tabId);
|
|
||||||
if (!context.session.cursor.lastSettled) {
|
|
||||||
try {
|
|
||||||
await this.captureInitial(tabId);
|
|
||||||
} catch {
|
|
||||||
// Post-action settle can still provide a usable state.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const scope = producerId
|
|
||||||
? context.session.cursor.lastSettled?.index.documentScope(producerId)
|
|
||||||
: undefined;
|
|
||||||
if (
|
|
||||||
producerId &&
|
|
||||||
(draft.op === "click" ||
|
|
||||||
draft.op === "hover" ||
|
|
||||||
draft.op === "fill" ||
|
|
||||||
draft.op === "press" ||
|
|
||||||
draft.op === "select")
|
|
||||||
) {
|
|
||||||
draft.targetHint = {
|
|
||||||
...(draft.targetHint ?? {}),
|
|
||||||
frameId: scope?.frameId ?? null,
|
|
||||||
geometrySpace: "local",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
|
|
||||||
context.settle.schedule(drafts, draftIndex, scope);
|
|
||||||
}
|
|
||||||
|
|
||||||
scheduleSettle(
|
|
||||||
tabId: number,
|
|
||||||
drafts: RecordingDraftStep[],
|
|
||||||
draftIndex: number,
|
|
||||||
scope?: DocumentSettleScope,
|
|
||||||
): void {
|
|
||||||
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
|
|
||||||
}
|
|
||||||
|
|
||||||
clearRedirect(tabId: number): void {
|
|
||||||
this.#contexts.get(tabId)?.settle.clearRedirect();
|
|
||||||
}
|
|
||||||
|
|
||||||
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
|
|
||||||
this.#context(tabId).settle.scheduleRedirect(drafts, url);
|
|
||||||
}
|
|
||||||
|
|
||||||
async #flushContext(context: TabRecordingContext): Promise<void> {
|
|
||||||
await context.frameRefresh;
|
|
||||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
|
||||||
await context.settle.flushRedirects();
|
|
||||||
await context.settle.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
async flushRedirects(tabId: number): Promise<void> {
|
|
||||||
await this.#contexts.get(tabId)?.settle.flushRedirects();
|
|
||||||
}
|
|
||||||
|
|
||||||
async flush(): Promise<void> {
|
|
||||||
for (const context of this.#contexts.values()) await this.#flushContext(context);
|
|
||||||
}
|
|
||||||
|
|
||||||
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
|
|
||||||
const context = this.#context(tabId);
|
|
||||||
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
|
|
||||||
if (!context.session.cursor.lastSettled) {
|
|
||||||
try {
|
|
||||||
await this.captureInitial(tabId);
|
|
||||||
} catch {
|
|
||||||
// The trace may remain observation-free when CDP is unavailable.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await context.settle.settleTrailing(drafts);
|
|
||||||
inferMissingPostStates(drafts);
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel(): void {
|
|
||||||
for (const context of this.#contexts.values()) {
|
|
||||||
context.frameRefreshAbort?.abort();
|
|
||||||
context.pendingCapture?.abort.abort();
|
|
||||||
context.settle.cancel();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
buildTrace(input: {
|
|
||||||
drafts: RecordingDraftStep[];
|
|
||||||
startedAt: string;
|
|
||||||
purpose?: string;
|
|
||||||
startUrl?: string;
|
|
||||||
stoppedBy: StopReason;
|
|
||||||
bskVersion: string;
|
|
||||||
includeTabSwitches?: boolean;
|
|
||||||
}): TraceV3 {
|
|
||||||
return buildTraceV3({
|
|
||||||
registry: this.#registry,
|
|
||||||
drafts: input.drafts,
|
|
||||||
annotations: this.#annotations,
|
|
||||||
startedAt: input.startedAt,
|
|
||||||
purpose: input.purpose,
|
|
||||||
startUrl: input.startUrl,
|
|
||||||
stoppedBy: input.stoppedBy,
|
|
||||||
bskVersion: input.bskVersion,
|
|
||||||
redactValues: this.#redactValues,
|
|
||||||
includeTabSwitches: input.includeTabSwitches,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
|
||||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
|
||||||
import { RecordingObservationSession } from "./observation-session";
|
|
||||||
import type { RecordingDraftStep } from "./types";
|
|
||||||
|
|
||||||
interface PendingSettle {
|
|
||||||
abort: AbortController;
|
|
||||||
scope: DocumentSettleScope;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PendingRedirect {
|
|
||||||
url: string;
|
|
||||||
abort: AbortController;
|
|
||||||
}
|
|
||||||
|
|
||||||
const CAPTURE_RETRY_DELAY_MS = 250;
|
|
||||||
|
|
||||||
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
if (signal?.aborted) return resolve();
|
|
||||||
const timer = setTimeout(resolve, ms);
|
|
||||||
signal?.addEventListener(
|
|
||||||
"abort",
|
|
||||||
() => {
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
},
|
|
||||||
{ once: true },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function isAbortError(error: unknown): boolean {
|
|
||||||
return (
|
|
||||||
typeof error === "object" &&
|
|
||||||
error !== null &&
|
|
||||||
(error as { name?: string }).name === "AbortError"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
|
||||||
for (let index = 0; index < drafts.length - 1; index += 1) {
|
|
||||||
const draft = drafts[index];
|
|
||||||
const next = drafts[index + 1];
|
|
||||||
if (draft && next && !draft.postStateId && next.preStateId) {
|
|
||||||
draft.postStateId = next.preStateId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class SettleController {
|
|
||||||
readonly #session: RecordingObservationSession;
|
|
||||||
readonly #cdp: CdpRunner;
|
|
||||||
readonly #tabsApi: ChromeTabsApi;
|
|
||||||
readonly #tabId: number;
|
|
||||||
readonly #rootScope: DocumentSettleScope;
|
|
||||||
readonly #pending = new Map<number, PendingSettle>();
|
|
||||||
#queue = Promise.resolve();
|
|
||||||
#pendingRedirect: PendingRedirect | null = null;
|
|
||||||
#redirectQueue = Promise.resolve();
|
|
||||||
|
|
||||||
constructor(input: {
|
|
||||||
session: RecordingObservationSession;
|
|
||||||
cdp: CdpRunner;
|
|
||||||
tabsApi: ChromeTabsApi;
|
|
||||||
tabId: number;
|
|
||||||
}) {
|
|
||||||
this.#session = input.session;
|
|
||||||
this.#cdp = input.cdp;
|
|
||||||
this.#tabsApi = input.tabsApi;
|
|
||||||
this.#tabId = input.tabId;
|
|
||||||
this.#rootScope = { target: { tabId: input.tabId } };
|
|
||||||
}
|
|
||||||
|
|
||||||
get hasPending(): boolean {
|
|
||||||
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async #captureWithRetry(signal?: AbortSignal) {
|
|
||||||
try {
|
|
||||||
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
|
||||||
} catch (error) {
|
|
||||||
if (signal?.aborted) throw error;
|
|
||||||
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
|
||||||
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
schedule(
|
|
||||||
drafts: RecordingDraftStep[],
|
|
||||||
draftIndex: number,
|
|
||||||
scope: DocumentSettleScope = this.#rootScope,
|
|
||||||
): void {
|
|
||||||
if (scope.target.tabId !== this.#tabId) {
|
|
||||||
throw new Error(
|
|
||||||
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const landing = drafts[draftIndex]?.preStateId;
|
|
||||||
for (const [index, pending] of this.#pending) {
|
|
||||||
if (index >= draftIndex) continue;
|
|
||||||
pending.abort.abort();
|
|
||||||
this.#pending.delete(index);
|
|
||||||
if (landing && drafts[index] && !drafts[index].postStateId) {
|
|
||||||
drafts[index].postStateId = landing;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.#pending.get(draftIndex)?.abort.abort();
|
|
||||||
const pending = { abort: new AbortController(), scope };
|
|
||||||
this.#pending.set(draftIndex, pending);
|
|
||||||
const task = async () => {
|
|
||||||
if (pending.abort.signal.aborted) return;
|
|
||||||
try {
|
|
||||||
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
|
||||||
signal: pending.abort.signal,
|
|
||||||
});
|
|
||||||
if (outcome === "cancelled") return;
|
|
||||||
const observation = await this.#captureWithRetry(pending.abort.signal);
|
|
||||||
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
|
||||||
drafts[draftIndex].postStateId = observation.stateId;
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (!isAbortError(error)) {
|
|
||||||
console.warn(
|
|
||||||
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
|
||||||
error,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
this.#queue = this.#queue.then(task, task).catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel(): void {
|
|
||||||
for (const pending of this.#pending.values()) pending.abort.abort();
|
|
||||||
this.#pending.clear();
|
|
||||||
this.clearRedirect();
|
|
||||||
}
|
|
||||||
|
|
||||||
async flush(): Promise<void> {
|
|
||||||
for (let round = 0; round < 10; round += 1) {
|
|
||||||
const current = this.#queue;
|
|
||||||
await current;
|
|
||||||
if (current === this.#queue) return;
|
|
||||||
}
|
|
||||||
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
|
||||||
}
|
|
||||||
|
|
||||||
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
|
||||||
const trailing: RecordingDraftStep[] = [];
|
|
||||||
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
|
||||||
const draft = drafts[index];
|
|
||||||
if (!draft || draft.postStateId) break;
|
|
||||||
trailing.push(draft);
|
|
||||||
}
|
|
||||||
if (trailing.length === 0) return;
|
|
||||||
try {
|
|
||||||
const observation = await this.#captureWithRetry();
|
|
||||||
for (const draft of trailing) draft.postStateId = observation.stateId;
|
|
||||||
} catch (error) {
|
|
||||||
console.warn("[bsk record] final observation at stop failed", error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
clearRedirect(): void {
|
|
||||||
this.#pendingRedirect?.abort.abort();
|
|
||||||
this.#pendingRedirect = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
|
||||||
this.#pendingRedirect?.abort.abort();
|
|
||||||
const pending: PendingRedirect = { url, abort: new AbortController() };
|
|
||||||
this.#pendingRedirect = pending;
|
|
||||||
const task = () => this.#settleRedirect(drafts, pending);
|
|
||||||
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
|
||||||
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
|
||||||
signal: pending.abort.signal,
|
|
||||||
});
|
|
||||||
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
|
||||||
|
|
||||||
let finalUrl = pending.url;
|
|
||||||
try {
|
|
||||||
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
|
||||||
} catch {
|
|
||||||
// The navigation event URL remains the best available destination.
|
|
||||||
}
|
|
||||||
if (this.#pendingRedirect !== pending) return;
|
|
||||||
this.#pendingRedirect = null;
|
|
||||||
if (
|
|
||||||
!finalUrl ||
|
|
||||||
finalUrl === "about:blank" ||
|
|
||||||
this.#session.cursor.lastSettled?.url === finalUrl
|
|
||||||
) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const last = drafts[drafts.length - 1];
|
|
||||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
|
||||||
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
|
||||||
await this.flush();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const draft: RecordingDraftStep = {
|
|
||||||
op: "navigate",
|
|
||||||
url: finalUrl,
|
|
||||||
pageUrl: finalUrl,
|
|
||||||
cause: "browser",
|
|
||||||
transitionQualifiers: ["server_redirect"],
|
|
||||||
preStateId: this.#session.cursor.lastSettled?.stateId,
|
|
||||||
};
|
|
||||||
drafts.push(draft);
|
|
||||||
const draftIndex = drafts.length - 1;
|
|
||||||
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
|
||||||
this.schedule(drafts, draftIndex);
|
|
||||||
await this.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
async flushRedirects(): Promise<void> {
|
|
||||||
for (let round = 0; round < 10; round += 1) {
|
|
||||||
const current = this.#redirectQueue;
|
|
||||||
await current;
|
|
||||||
if (current === this.#redirectQueue) return;
|
|
||||||
}
|
|
||||||
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
export interface RecordedStateEntry {
|
|
||||||
id: string;
|
|
||||||
url: string;
|
|
||||||
title?: string;
|
|
||||||
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
|
|
||||||
vomText: string;
|
|
||||||
truncated: boolean;
|
|
||||||
stepsHere: number[];
|
|
||||||
}
|
|
||||||
|
|
||||||
function stateIdentity(url: string, body: string): string {
|
|
||||||
return `${url}\0${body}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RecordingStateRegistry {
|
|
||||||
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
|
||||||
readonly #idByIdentity = new Map<string, string>();
|
|
||||||
#nextId = 1;
|
|
||||||
|
|
||||||
register(input: {
|
|
||||||
url: string;
|
|
||||||
title?: string;
|
|
||||||
vomText: string;
|
|
||||||
truncated?: boolean;
|
|
||||||
}): RecordedStateEntry {
|
|
||||||
const identity = stateIdentity(input.url, input.vomText);
|
|
||||||
const existingId = this.#idByIdentity.get(identity);
|
|
||||||
if (existingId) {
|
|
||||||
const existing = this.#entriesById.get(existingId)!;
|
|
||||||
if (!existing.title && input.title) existing.title = input.title;
|
|
||||||
if (input.truncated) existing.truncated = true;
|
|
||||||
return existing;
|
|
||||||
}
|
|
||||||
|
|
||||||
const entry: RecordedStateEntry = {
|
|
||||||
id: `s${this.#nextId}`,
|
|
||||||
url: input.url,
|
|
||||||
...(input.title ? { title: input.title } : {}),
|
|
||||||
vomText: input.vomText,
|
|
||||||
truncated: input.truncated ?? false,
|
|
||||||
stepsHere: [],
|
|
||||||
};
|
|
||||||
this.#nextId += 1;
|
|
||||||
this.#entriesById.set(entry.id, entry);
|
|
||||||
this.#idByIdentity.set(identity, entry.id);
|
|
||||||
return entry;
|
|
||||||
}
|
|
||||||
|
|
||||||
values(): RecordedStateEntry[] {
|
|
||||||
return [...this.#entriesById.values()];
|
|
||||||
}
|
|
||||||
|
|
||||||
markStep(stateId: string, draftId: number): void {
|
|
||||||
const entry = this.#entriesById.get(stateId);
|
|
||||||
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
import type { RecordStepPayload } from "../record-bridge";
|
|
||||||
import { hasRedirectQualifier } from "./navigation-policy";
|
|
||||||
import type { RecordingDraftStep, TargetMatchHint } from "./types";
|
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
|
||||||
steps: RecordingDraftStep[];
|
|
||||||
navigation: RecordingNavigationCursor;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RecordingNavigationCursor {
|
|
||||||
currentUrl?: string;
|
|
||||||
pendingNavigation: boolean;
|
|
||||||
pendingNavigationDeadline?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
|
||||||
|
|
||||||
function toDraftStep(
|
|
||||||
payload: RecordStepPayload,
|
|
||||||
targetHint?: TargetMatchHint,
|
|
||||||
): RecordingDraftStep | null {
|
|
||||||
const pageUrl = payload.page_url;
|
|
||||||
const common = {
|
|
||||||
...(pageUrl ? { pageUrl } : {}),
|
|
||||||
...(targetHint ? { targetHint } : {}),
|
|
||||||
};
|
|
||||||
switch (payload.op) {
|
|
||||||
case "click":
|
|
||||||
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
|
|
||||||
case "hover":
|
|
||||||
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
|
|
||||||
case "fill":
|
|
||||||
return payload.target
|
|
||||||
? {
|
|
||||||
op: "fill",
|
|
||||||
captureTarget: payload.target,
|
|
||||||
value: payload.value ?? "",
|
|
||||||
...(payload.commit ? { commit: payload.commit } : {}),
|
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
|
||||||
...common,
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "press":
|
|
||||||
return payload.key
|
|
||||||
? {
|
|
||||||
op: "press",
|
|
||||||
key: payload.key,
|
|
||||||
...(payload.target ? { captureTarget: payload.target } : {}),
|
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
|
||||||
...common,
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "select":
|
|
||||||
return payload.target && payload.values
|
|
||||||
? {
|
|
||||||
op: "select",
|
|
||||||
captureTarget: payload.target,
|
|
||||||
values: payload.values,
|
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
|
||||||
...common,
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
case "navigate":
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
|
||||||
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
|
|
||||||
const step = buffer.steps[index];
|
|
||||||
if (!step) continue;
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
|
||||||
step.navigatedTo = url;
|
|
||||||
return index;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type NavigationObserveResult =
|
|
||||||
| { kind: "noop" }
|
|
||||||
| { kind: "annotated"; index: number }
|
|
||||||
| { kind: "appended"; index: number }
|
|
||||||
| { kind: "coalesce_redirect"; url: string };
|
|
||||||
|
|
||||||
export function observeRecordedNavigation(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
url: string,
|
|
||||||
causedByAction?: boolean,
|
|
||||||
transitionType?: string,
|
|
||||||
transitionQualifiers?: string[],
|
|
||||||
): NavigationObserveResult {
|
|
||||||
const navigation = buffer.navigation;
|
|
||||||
if (!url || url === navigation.currentUrl) return { kind: "noop" };
|
|
||||||
navigation.currentUrl = url;
|
|
||||||
|
|
||||||
const pendingIsCurrent =
|
|
||||||
navigation.pendingNavigation &&
|
|
||||||
(navigation.pendingNavigationDeadline === undefined ||
|
|
||||||
navigation.pendingNavigationDeadline >= Date.now());
|
|
||||||
|
|
||||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
|
||||||
navigation.pendingNavigation = false;
|
|
||||||
navigation.pendingNavigationDeadline = undefined;
|
|
||||||
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
|
||||||
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
|
|
||||||
} else {
|
|
||||||
navigation.pendingNavigation = false;
|
|
||||||
navigation.pendingNavigationDeadline = undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (hasRedirectQualifier(transitionQualifiers)) {
|
|
||||||
return { kind: "coalesce_redirect", url };
|
|
||||||
}
|
|
||||||
buffer.steps.push({
|
|
||||||
op: "navigate",
|
|
||||||
url,
|
|
||||||
pageUrl: url,
|
|
||||||
transitionType,
|
|
||||||
transitionQualifiers,
|
|
||||||
});
|
|
||||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function appendRecordedPayload(
|
|
||||||
buffer: RecordingStepBuffer,
|
|
||||||
payload: RecordStepPayload,
|
|
||||||
targetHint?: TargetMatchHint,
|
|
||||||
): number | null {
|
|
||||||
if (payload.op === "navigate") {
|
|
||||||
if (!payload.url) return null;
|
|
||||||
const result = observeRecordedNavigation(
|
|
||||||
buffer,
|
|
||||||
payload.url,
|
|
||||||
payload.navigation_caused_by_action,
|
|
||||||
payload.transitionType,
|
|
||||||
payload.transitionQualifiers,
|
|
||||||
);
|
|
||||||
return result.kind === "appended" ? result.index : null;
|
|
||||||
}
|
|
||||||
const step = toDraftStep(
|
|
||||||
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
|
|
||||||
targetHint,
|
|
||||||
);
|
|
||||||
if (!step) return null;
|
|
||||||
buffer.steps.push(step);
|
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
|
||||||
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
|
|
||||||
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
|
|
||||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
|
||||||
: undefined;
|
|
||||||
}
|
|
||||||
return buffer.steps.length - 1;
|
|
||||||
}
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
import type { RecordingNavigationCursor } from "./step-buffer";
|
|
||||||
|
|
||||||
export interface TabActivation {
|
|
||||||
tabId: number;
|
|
||||||
revision: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RecordingTabCoordinator {
|
|
||||||
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
|
|
||||||
#activeTabId: number;
|
|
||||||
#currentTabId: number;
|
|
||||||
#activationRevision = 0;
|
|
||||||
|
|
||||||
constructor(initialTabId: number, initialUrl?: string) {
|
|
||||||
this.#activeTabId = initialTabId;
|
|
||||||
this.#currentTabId = initialTabId;
|
|
||||||
this.#navigationByTab.set(initialTabId, {
|
|
||||||
currentUrl: initialUrl,
|
|
||||||
pendingNavigation: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
get activeTabId(): number {
|
|
||||||
return this.#activeTabId;
|
|
||||||
}
|
|
||||||
|
|
||||||
get currentTabId(): number {
|
|
||||||
return this.#currentTabId;
|
|
||||||
}
|
|
||||||
|
|
||||||
noteActivation(tabId: number): TabActivation {
|
|
||||||
this.#activeTabId = tabId;
|
|
||||||
this.#activationRevision += 1;
|
|
||||||
return { tabId, revision: this.#activationRevision };
|
|
||||||
}
|
|
||||||
|
|
||||||
isLatest(activation: TabActivation): boolean {
|
|
||||||
return (
|
|
||||||
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
commit(tabId: number, currentUrl?: string): void {
|
|
||||||
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
|
|
||||||
this.#currentTabId = tabId;
|
|
||||||
}
|
|
||||||
|
|
||||||
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
|
|
||||||
const existing = this.#navigationByTab.get(tabId);
|
|
||||||
if (existing) {
|
|
||||||
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
|
|
||||||
return existing;
|
|
||||||
}
|
|
||||||
const created: RecordingNavigationCursor = {
|
|
||||||
currentUrl: fallbackUrl,
|
|
||||||
pendingNavigation: false,
|
|
||||||
};
|
|
||||||
this.#navigationByTab.set(tabId, created);
|
|
||||||
return created;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
import type { RenderedRef } from "@browser-skill/vom";
|
|
||||||
import type { TargetDescriptorV3 } from "@/transport/types";
|
|
||||||
import type { CaptureTargetDescriptor } from "../describe-target";
|
|
||||||
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
|
||||||
import type { TargetGeometry, TargetMatchHint } from "./types";
|
|
||||||
|
|
||||||
const MATCH_TOLERANCE_PX = 2;
|
|
||||||
|
|
||||||
function close(a: number, b: number): boolean {
|
|
||||||
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
|
||||||
}
|
|
||||||
|
|
||||||
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
|
||||||
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
|
||||||
}
|
|
||||||
|
|
||||||
function candidateMatches(
|
|
||||||
candidate: IndexedObservationNode,
|
|
||||||
geometry: TargetGeometry,
|
|
||||||
geometrySpace: "top" | "local",
|
|
||||||
): boolean {
|
|
||||||
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
|
|
||||||
return rect != null && rectMatches(geometry.rect, rect);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
|
||||||
return {
|
|
||||||
...(fallback?.role ? { role: fallback.role } : {}),
|
|
||||||
...(fallback?.name ? { name: fallback.name } : {}),
|
|
||||||
unmatched: true,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalized(value: string | undefined): string {
|
|
||||||
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
|
||||||
}
|
|
||||||
|
|
||||||
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
|
||||||
const role = normalized(fallback?.role);
|
|
||||||
const name = normalized(fallback?.name);
|
|
||||||
if (!role && !name) return false;
|
|
||||||
if (role && normalized(ref.role) !== role) return false;
|
|
||||||
if (name && normalized(ref.name) !== name) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
|
||||||
return {
|
|
||||||
ref: ref.ref,
|
|
||||||
...(ref.role ? { role: ref.role } : {}),
|
|
||||||
...(ref.name ? { name: ref.name } : {}),
|
|
||||||
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function matchObservationTarget(input: {
|
|
||||||
observation: RegisteredObservation;
|
|
||||||
hint?: TargetMatchHint;
|
|
||||||
fallback?: CaptureTargetDescriptor;
|
|
||||||
}): TargetDescriptorV3 {
|
|
||||||
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
|
|
||||||
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
|
||||||
const geometry = input.hint?.geometry;
|
|
||||||
if (geometry) {
|
|
||||||
const matches = input.observation.index
|
|
||||||
.candidates(frameId, geometry.tag)
|
|
||||||
.filter(
|
|
||||||
(candidate) =>
|
|
||||||
candidate.ref &&
|
|
||||||
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
|
|
||||||
);
|
|
||||||
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
|
||||||
const semanticMatches = matches.filter(
|
|
||||||
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
|
||||||
);
|
|
||||||
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
|
||||||
return unmatchedTarget(input.fallback);
|
|
||||||
}
|
|
||||||
|
|
||||||
const semanticMatches = input.observation.index
|
|
||||||
.refs(frameId)
|
|
||||||
.filter((ref) => matchesSemantics(ref, input.fallback));
|
|
||||||
return semanticMatches.length === 1
|
|
||||||
? descriptor(semanticMatches[0]!)
|
|
||||||
: unmatchedTarget(input.fallback);
|
|
||||||
}
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
import {
|
|
||||||
type StepV3,
|
|
||||||
type StopReason,
|
|
||||||
TRACE_VERSION_V3,
|
|
||||||
type TraceStateV3,
|
|
||||||
type TraceV3,
|
|
||||||
VOM_FORMAT_VERSION,
|
|
||||||
} from "@/transport/types";
|
|
||||||
import { resolveDraftStartUrl } from "./draft-policy";
|
|
||||||
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
|
||||||
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
|
||||||
import { formatTraceStateBody } from "./trace-state-body";
|
|
||||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
|
||||||
|
|
||||||
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
|
||||||
const entries = registry.values();
|
|
||||||
if (steps.length === 0) return entries.slice(0, 1);
|
|
||||||
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
|
||||||
return entries.filter((entry) => referenced.has(entry.id));
|
|
||||||
}
|
|
||||||
|
|
||||||
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
|
||||||
return [
|
|
||||||
...new Set(
|
|
||||||
draftIds.flatMap((id) => {
|
|
||||||
const stepId = stepIdByDraftId.get(id);
|
|
||||||
return stepId === undefined ? [] : [stepId];
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
].sort((a, b) => a - b);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildTraceV3(input: {
|
|
||||||
registry: RecordingStateRegistry;
|
|
||||||
drafts: RecordingDraftStep[];
|
|
||||||
annotations?: StepAnnotation[];
|
|
||||||
startedAt: string;
|
|
||||||
purpose?: string;
|
|
||||||
startUrl?: string;
|
|
||||||
stoppedBy: StopReason;
|
|
||||||
bskVersion: string;
|
|
||||||
redactValues?: boolean;
|
|
||||||
includeTabSwitches?: boolean;
|
|
||||||
}): TraceV3 {
|
|
||||||
const reduced = reduceTraceStepsV3(input.drafts, {
|
|
||||||
includeTabSwitches: input.includeTabSwitches,
|
|
||||||
redactValues: input.redactValues,
|
|
||||||
});
|
|
||||||
const entries = publishedEntries(input.registry, reduced.steps);
|
|
||||||
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
|
||||||
const annotationsByState = new Map<string, StepAnnotation[]>();
|
|
||||||
for (const annotation of input.annotations ?? []) {
|
|
||||||
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
|
||||||
bucket.push(annotation);
|
|
||||||
annotationsByState.set(annotation.stateId, bucket);
|
|
||||||
}
|
|
||||||
const steps = reduced.steps.map((step) => ({
|
|
||||||
...step,
|
|
||||||
state: publishedId.get(step.state) ?? step.state,
|
|
||||||
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
|
||||||
}));
|
|
||||||
const states: TraceStateV3[] = entries.map((entry) => {
|
|
||||||
const id = publishedId.get(entry.id) ?? entry.id;
|
|
||||||
return {
|
|
||||||
id,
|
|
||||||
url: entry.url,
|
|
||||||
...(entry.title ? { title: entry.title } : {}),
|
|
||||||
body: formatTraceStateBody({
|
|
||||||
stateId: id,
|
|
||||||
url: entry.url,
|
|
||||||
title: entry.title,
|
|
||||||
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
|
||||||
vomText: entry.vomText,
|
|
||||||
annotations: annotationsByState.get(entry.id) ?? [],
|
|
||||||
stepIdByDraftId: reduced.stepIdByDraftId,
|
|
||||||
}),
|
|
||||||
...(entry.truncated ? { truncated: true } : {}),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
version: TRACE_VERSION_V3,
|
|
||||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
|
||||||
recorded_at: new Date().toISOString(),
|
|
||||||
started_at: input.startedAt,
|
|
||||||
stopped_by: input.stoppedBy,
|
|
||||||
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
|
||||||
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
|
||||||
states,
|
|
||||||
steps,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,215 +0,0 @@
|
|||||||
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
|
||||||
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
|
||||||
import type { RecordingDraftStep } from "./types";
|
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
|
||||||
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
|
||||||
const out: RecordingDraftStep[] = [];
|
|
||||||
for (const step of steps) {
|
|
||||||
const prev = out[out.length - 1];
|
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
|
||||||
out[out.length - 1] = step;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
out.push(step);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
|
||||||
const urls: string[] = [];
|
|
||||||
if (startUrl) urls.push(startUrl);
|
|
||||||
for (const step of steps) {
|
|
||||||
if (step.op === "navigate") {
|
|
||||||
urls.push(step.url);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (step.pageUrl) urls.push(step.pageUrl);
|
|
||||||
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
|
||||||
}
|
|
||||||
const seen = new Set<string>();
|
|
||||||
const unique: string[] = [];
|
|
||||||
for (const url of urls) {
|
|
||||||
if (!url || seen.has(url)) continue;
|
|
||||||
seen.add(url);
|
|
||||||
unique.push(url);
|
|
||||||
}
|
|
||||||
return unique;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildPageRegistry(
|
|
||||||
steps: RecordingDraftStep[],
|
|
||||||
startUrl?: string,
|
|
||||||
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
|
||||||
const urls = collectUrls(steps, startUrl);
|
|
||||||
const urlToId = new Map<string, string>();
|
|
||||||
const pages = urls.map((url, index) => {
|
|
||||||
const id = `p${index + 1}`;
|
|
||||||
urlToId.set(url, id);
|
|
||||||
return { id, url };
|
|
||||||
});
|
|
||||||
return { pages, urlToId };
|
|
||||||
}
|
|
||||||
|
|
||||||
function pageIdFor(
|
|
||||||
url: string | undefined,
|
|
||||||
urlToId: Map<string, string>,
|
|
||||||
fallbackUrl?: string,
|
|
||||||
): string {
|
|
||||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
|
||||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
|
||||||
return urlToId.values().next().value ?? "p1";
|
|
||||||
}
|
|
||||||
|
|
||||||
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
|
||||||
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
|
||||||
if (step.pageUrl) return step.pageUrl;
|
|
||||||
return fallbackUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
function effectForNavigation(
|
|
||||||
navigatedTo: string | undefined,
|
|
||||||
urlToId: Map<string, string>,
|
|
||||||
): StepV2["effect"] {
|
|
||||||
if (!navigatedTo) return undefined;
|
|
||||||
const pageId = urlToId.get(navigatedTo);
|
|
||||||
if (!pageId) return undefined;
|
|
||||||
return { navigated_to: pageId };
|
|
||||||
}
|
|
||||||
|
|
||||||
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
|
||||||
if (!effect) return step;
|
|
||||||
return { ...step, effect };
|
|
||||||
}
|
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
|
||||||
return values.map((value, index) => ({
|
|
||||||
value,
|
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
function toV2Step(
|
|
||||||
step: RecordingDraftStep,
|
|
||||||
id: number,
|
|
||||||
urlToId: Map<string, string>,
|
|
||||||
fallbackUrl?: string,
|
|
||||||
): StepV2 | null {
|
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
|
||||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
|
||||||
|
|
||||||
switch (step.op) {
|
|
||||||
case "navigate":
|
|
||||||
return {
|
|
||||||
op: "navigate",
|
|
||||||
id,
|
|
||||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
|
||||||
to: step.url,
|
|
||||||
};
|
|
||||||
case "click":
|
|
||||||
if (!step.captureTarget) return null;
|
|
||||||
return withEffect(
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.captureTarget,
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigatedTo, urlToId),
|
|
||||||
);
|
|
||||||
case "fill":
|
|
||||||
if (!step.captureTarget) return null;
|
|
||||||
return {
|
|
||||||
op: "fill",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.captureTarget,
|
|
||||||
value: step.value,
|
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
|
||||||
};
|
|
||||||
case "press":
|
|
||||||
return withEffect(
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
key: step.key,
|
|
||||||
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigatedTo, urlToId),
|
|
||||||
);
|
|
||||||
case "select":
|
|
||||||
if (!step.captureTarget) return null;
|
|
||||||
return withEffect(
|
|
||||||
{
|
|
||||||
op: "select",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.captureTarget,
|
|
||||||
selection: toSelection(step.values, step.labels),
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigatedTo, urlToId),
|
|
||||||
);
|
|
||||||
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
|
|
||||||
// step variant and fail to decode the whole result if we emit it.
|
|
||||||
case "hover":
|
|
||||||
case "scroll":
|
|
||||||
case "switch_tab":
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ReducedTrace {
|
|
||||||
pages: PageRefV2[];
|
|
||||||
steps: StepV2[];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
|
||||||
*/
|
|
||||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
|
||||||
const collapsed = collapseNavigations(steps);
|
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
|
||||||
const out: StepV2[] = [];
|
|
||||||
let id = 1;
|
|
||||||
let lastUrl = startUrl;
|
|
||||||
for (const draft of collapsed) {
|
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
|
||||||
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
|
||||||
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
|
||||||
if (!step) continue;
|
|
||||||
out.push(step);
|
|
||||||
id += 1;
|
|
||||||
}
|
|
||||||
return { pages, steps: out };
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveTraceStartUrl(
|
|
||||||
drafts: RecordingDraftStep[],
|
|
||||||
startUrl?: string,
|
|
||||||
pages?: PageRefV2[],
|
|
||||||
): string {
|
|
||||||
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildTraceV2(input: {
|
|
||||||
steps: RecordingDraftStep[];
|
|
||||||
startedAt: string;
|
|
||||||
startUrl?: string;
|
|
||||||
purpose?: string;
|
|
||||||
}): TraceV2 {
|
|
||||||
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
|
||||||
return {
|
|
||||||
recorded_at: new Date().toISOString(),
|
|
||||||
started_at: input.startedAt,
|
|
||||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
|
||||||
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
|
||||||
pages,
|
|
||||||
steps,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
import type { NavigationCause, StepV3 } from "@/transport/types";
|
|
||||||
import { shouldIncludeDraft } from "./draft-policy";
|
|
||||||
import { hasRedirectQualifier } from "./navigation-policy";
|
|
||||||
import { unmatchedTarget } from "./target-matcher";
|
|
||||||
import type { RecordingDraftStep } from "./types";
|
|
||||||
|
|
||||||
interface CollapsedDraft {
|
|
||||||
draft: RecordingDraftStep;
|
|
||||||
draftIds: number[];
|
|
||||||
}
|
|
||||||
|
|
||||||
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
|
||||||
typed: "user_typed",
|
|
||||||
generated: "user_typed",
|
|
||||||
keyword: "user_typed",
|
|
||||||
keyword_generated: "user_typed",
|
|
||||||
link: "link",
|
|
||||||
form_submit: "form_submit",
|
|
||||||
reload: "reload",
|
|
||||||
auto_bookmark: "browser",
|
|
||||||
start_page: "browser",
|
|
||||||
};
|
|
||||||
|
|
||||||
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
|
||||||
return hasRedirectQualifier(step.transitionQualifiers);
|
|
||||||
}
|
|
||||||
|
|
||||||
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
|
||||||
const output: CollapsedDraft[] = [];
|
|
||||||
steps.forEach((step, index) => {
|
|
||||||
const previous = output[output.length - 1];
|
|
||||||
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
|
||||||
previous.draft = {
|
|
||||||
...previous.draft,
|
|
||||||
url: step.url,
|
|
||||||
postStateId: step.postStateId ?? previous.draft.postStateId,
|
|
||||||
};
|
|
||||||
previous.draftIds.push(index + 1);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
|
||||||
});
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
|
|
||||||
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
|
||||||
if (step.cause) return step.cause;
|
|
||||||
const qualifiers = step.transitionQualifiers ?? [];
|
|
||||||
if (qualifiers.includes("forward_back")) return "history";
|
|
||||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
|
||||||
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
|
||||||
}
|
|
||||||
|
|
||||||
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
|
||||||
return values.map((value, index) => ({
|
|
||||||
value,
|
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ReduceDraftOptions {
|
|
||||||
includeTabSwitches: boolean;
|
|
||||||
redactValues: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
function reduceDraft(
|
|
||||||
draft: RecordingDraftStep,
|
|
||||||
id: number,
|
|
||||||
options: ReduceDraftOptions,
|
|
||||||
): StepV3 | null {
|
|
||||||
if (!shouldIncludeDraft(draft)) return null;
|
|
||||||
if (draft.op === "switch_tab") {
|
|
||||||
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
|
|
||||||
return {
|
|
||||||
op: "switch_tab",
|
|
||||||
id,
|
|
||||||
state: draft.preStateId,
|
|
||||||
result: { state: draft.postStateId },
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const state = draft.preStateId ?? draft.postStateId;
|
|
||||||
const resultState = draft.postStateId ?? draft.preStateId;
|
|
||||||
if (!state || !resultState) return null;
|
|
||||||
const common = { id, state, result: { state: resultState } };
|
|
||||||
|
|
||||||
switch (draft.op) {
|
|
||||||
case "navigate":
|
|
||||||
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
|
||||||
case "click":
|
|
||||||
return {
|
|
||||||
op: "click",
|
|
||||||
...common,
|
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
|
||||||
};
|
|
||||||
case "hover":
|
|
||||||
return {
|
|
||||||
op: "hover",
|
|
||||||
...common,
|
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
|
||||||
};
|
|
||||||
case "fill":
|
|
||||||
const fillIsRedacted = options.redactValues || draft.redacted === true;
|
|
||||||
return {
|
|
||||||
op: "fill",
|
|
||||||
...common,
|
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
|
||||||
value: fillIsRedacted ? "***" : draft.value,
|
|
||||||
commit: draft.commit ?? "blur",
|
|
||||||
...(fillIsRedacted ? { redacted: true } : {}),
|
|
||||||
};
|
|
||||||
case "press":
|
|
||||||
return {
|
|
||||||
op: "press",
|
|
||||||
...common,
|
|
||||||
key: draft.key,
|
|
||||||
...(draft.captureTarget || draft.matchedTarget
|
|
||||||
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
|
||||||
: {}),
|
|
||||||
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
|
||||||
};
|
|
||||||
case "select":
|
|
||||||
return {
|
|
||||||
op: "select",
|
|
||||||
...common,
|
|
||||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
|
||||||
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
|
|
||||||
};
|
|
||||||
case "scroll":
|
|
||||||
return { op: "scroll", ...common };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ReducedTraceV3 {
|
|
||||||
steps: StepV3[];
|
|
||||||
stepIdByDraftId: Map<number, number>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function reduceTraceStepsV3(
|
|
||||||
steps: RecordingDraftStep[],
|
|
||||||
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
|
|
||||||
): ReducedTraceV3 {
|
|
||||||
const output: StepV3[] = [];
|
|
||||||
const stepIdByDraftId = new Map<number, number>();
|
|
||||||
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
|
||||||
const step = reduceDraft(draft, output.length + 1, {
|
|
||||||
includeTabSwitches: options.includeTabSwitches === true,
|
|
||||||
redactValues: options.redactValues === true,
|
|
||||||
});
|
|
||||||
if (!step) continue;
|
|
||||||
output.push(step);
|
|
||||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
|
||||||
}
|
|
||||||
return { steps: output, stepIdByDraftId };
|
|
||||||
}
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import type { StepAnnotation } from "./types";
|
|
||||||
|
|
||||||
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
|
||||||
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatTraceStateBody(input: {
|
|
||||||
stateId: string;
|
|
||||||
url: string;
|
|
||||||
title?: string;
|
|
||||||
stepIds: number[];
|
|
||||||
vomText: string;
|
|
||||||
annotations: StepAnnotation[];
|
|
||||||
stepIdByDraftId: Map<number, number>;
|
|
||||||
}): string {
|
|
||||||
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
|
||||||
lines.push(`url: ${JSON.stringify(input.url)}`);
|
|
||||||
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
|
||||||
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
|
||||||
lines.push("---");
|
|
||||||
|
|
||||||
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
|
||||||
for (const annotation of input.annotations) {
|
|
||||||
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
|
||||||
if (stepId === undefined) continue;
|
|
||||||
const bucket = byLine.get(annotation.line) ?? [];
|
|
||||||
bucket.push({ annotation, stepId });
|
|
||||||
byLine.set(annotation.line, bucket);
|
|
||||||
}
|
|
||||||
|
|
||||||
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
|
||||||
let line = bodyLine;
|
|
||||||
for (const item of byLine.get(lineIndex) ?? []) {
|
|
||||||
line += annotationText(item.annotation, item.stepId);
|
|
||||||
}
|
|
||||||
lines.push(line);
|
|
||||||
});
|
|
||||||
return `${lines.join("\n")}\n`;
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
|
||||||
import type {
|
|
||||||
FillCommit,
|
|
||||||
KeyModifier,
|
|
||||||
NavigationCause,
|
|
||||||
StepV3,
|
|
||||||
TargetDescriptorV3,
|
|
||||||
} from "@/transport/types";
|
|
||||||
|
|
||||||
export interface TargetGeometry {
|
|
||||||
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
|
||||||
rect: { x: number; y: number; w: number; h: number };
|
|
||||||
tag: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TargetMatchHint {
|
|
||||||
geometry?: TargetGeometry;
|
|
||||||
/** Missing means top frame; null means the source Document could not be resolved. */
|
|
||||||
frameId?: string | null;
|
|
||||||
geometrySpace?: "top" | "local";
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface StepAnnotation {
|
|
||||||
draftId: number;
|
|
||||||
op: StepV3["op"];
|
|
||||||
line: number;
|
|
||||||
stateId: string;
|
|
||||||
detail?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DraftStateLink {
|
|
||||||
pageUrl?: string;
|
|
||||||
preStateId?: string;
|
|
||||||
postStateId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DraftTarget {
|
|
||||||
captureTarget?: CaptureTargetDescriptor;
|
|
||||||
targetHint?: TargetMatchHint;
|
|
||||||
matchedTarget?: TargetDescriptorV3;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DraftNavigationEffect {
|
|
||||||
navigatedTo?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type RecordingDraftStep =
|
|
||||||
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
|
||||||
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
|
||||||
| ({
|
|
||||||
op: "fill";
|
|
||||||
value: string;
|
|
||||||
commit?: FillCommit;
|
|
||||||
redacted?: boolean;
|
|
||||||
} & DraftStateLink &
|
|
||||||
DraftTarget &
|
|
||||||
DraftNavigationEffect)
|
|
||||||
| ({
|
|
||||||
op: "press";
|
|
||||||
key: string;
|
|
||||||
modifiers?: KeyModifier[];
|
|
||||||
} & DraftStateLink &
|
|
||||||
DraftTarget &
|
|
||||||
DraftNavigationEffect)
|
|
||||||
| ({
|
|
||||||
op: "select";
|
|
||||||
values: string[];
|
|
||||||
labels?: string[];
|
|
||||||
} & DraftStateLink &
|
|
||||||
DraftTarget &
|
|
||||||
DraftNavigationEffect)
|
|
||||||
| ({ op: "scroll" } & DraftStateLink)
|
|
||||||
| ({ op: "switch_tab" } & DraftStateLink)
|
|
||||||
| ({
|
|
||||||
op: "navigate";
|
|
||||||
url: string;
|
|
||||||
cause?: NavigationCause;
|
|
||||||
transitionType?: string;
|
|
||||||
transitionQualifiers?: string[];
|
|
||||||
} & DraftStateLink);
|
|
||||||
|
|
||||||
export type TargetedRecordingDraft = Extract<
|
|
||||||
RecordingDraftStep,
|
|
||||||
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
|
||||||
>;
|
|
||||||
@@ -0,0 +1,297 @@
|
|||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
import type { DraftTraceStep, KeyModifier } from "@/transport/types";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export interface TargetDescriptorV2 {
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
tag: string;
|
||||||
|
name_attr?: string;
|
||||||
|
placeholder?: string;
|
||||||
|
nearby_label?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PageRef {
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SelectedOptionV2 {
|
||||||
|
value: string;
|
||||||
|
label?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepEffectV2 {
|
||||||
|
navigated_to: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StepV2 =
|
||||||
|
| { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 }
|
||||||
|
| { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 }
|
||||||
|
| {
|
||||||
|
op: "fill";
|
||||||
|
id: number;
|
||||||
|
page: string;
|
||||||
|
target: TargetDescriptorV2;
|
||||||
|
value: string;
|
||||||
|
redacted?: boolean;
|
||||||
|
effect?: StepEffectV2;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
op: "select";
|
||||||
|
id: number;
|
||||||
|
page: string;
|
||||||
|
target: TargetDescriptorV2;
|
||||||
|
selection: SelectedOptionV2[];
|
||||||
|
effect?: StepEffectV2;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
op: "press";
|
||||||
|
id: number;
|
||||||
|
page: string;
|
||||||
|
key: string;
|
||||||
|
modifiers?: KeyModifier[];
|
||||||
|
target?: TargetDescriptorV2;
|
||||||
|
effect?: StepEffectV2;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface TraceV2 {
|
||||||
|
recorded_at: string;
|
||||||
|
started_at?: string;
|
||||||
|
purpose?: string;
|
||||||
|
entry: { start_url: string };
|
||||||
|
pages: PageRef[];
|
||||||
|
steps: StepV2[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||||
|
if (step.op === "scroll" || step.op === "hover") return false;
|
||||||
|
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||||
|
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||||
|
const out: DraftTraceStep[] = [];
|
||||||
|
for (const step of steps) {
|
||||||
|
const prev = out[out.length - 1];
|
||||||
|
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||||
|
out[out.length - 1] = step;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
out.push(step);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||||
|
const urls: string[] = [];
|
||||||
|
if (startUrl) urls.push(startUrl);
|
||||||
|
for (const step of steps) {
|
||||||
|
if (step.op === "navigate") {
|
||||||
|
urls.push(step.url);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||||
|
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||||
|
}
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const unique: string[] = [];
|
||||||
|
for (const url of urls) {
|
||||||
|
if (!url || seen.has(url)) continue;
|
||||||
|
seen.add(url);
|
||||||
|
unique.push(url);
|
||||||
|
}
|
||||||
|
return unique;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildPageRegistry(
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
startUrl?: string,
|
||||||
|
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||||
|
const urls = collectUrls(steps, startUrl);
|
||||||
|
const urlToId = new Map<string, string>();
|
||||||
|
const pages = urls.map((url, index) => {
|
||||||
|
const id = `p${index + 1}`;
|
||||||
|
urlToId.set(url, id);
|
||||||
|
return { id, url };
|
||||||
|
});
|
||||||
|
return { pages, urlToId };
|
||||||
|
}
|
||||||
|
|
||||||
|
function pageIdFor(
|
||||||
|
url: string | undefined,
|
||||||
|
urlToId: Map<string, string>,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): string {
|
||||||
|
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||||
|
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||||
|
return urlToId.values().next().value ?? "p1";
|
||||||
|
}
|
||||||
|
|
||||||
|
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||||
|
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||||
|
if ("page_url" in step && step.page_url) return step.page_url;
|
||||||
|
return fallbackUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
function effectForNavigation(
|
||||||
|
navigatedTo: string | undefined,
|
||||||
|
urlToId: Map<string, string>,
|
||||||
|
): StepEffectV2 | undefined {
|
||||||
|
if (!navigatedTo) return undefined;
|
||||||
|
const pageId = urlToId.get(navigatedTo);
|
||||||
|
if (!pageId) return undefined;
|
||||||
|
return { navigated_to: pageId };
|
||||||
|
}
|
||||||
|
|
||||||
|
function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 {
|
||||||
|
return {
|
||||||
|
tag: capture?.tag ?? "unknown",
|
||||||
|
...(capture?.role ? { role: capture.role } : {}),
|
||||||
|
...(capture?.name ? { name: capture.name } : {}),
|
||||||
|
...(capture?.name_attr ? { name_attr: capture.name_attr } : {}),
|
||||||
|
...(capture?.placeholder ? { placeholder: capture.placeholder } : {}),
|
||||||
|
...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined {
|
||||||
|
if (!("target" in step) && !("captureTarget" in step)) return undefined;
|
||||||
|
const capture = "captureTarget" in step ? step.captureTarget : undefined;
|
||||||
|
if (capture) return captureTargetToV2Target(capture);
|
||||||
|
if ("target" in step && step.target && "tag" in (step.target as object)) {
|
||||||
|
const legacy = step.target as TargetDescriptorV2 & { tag?: string };
|
||||||
|
if (legacy.tag) return legacy;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function toV2Step(
|
||||||
|
step: DraftTraceStep,
|
||||||
|
id: number,
|
||||||
|
urlToId: Map<string, string>,
|
||||||
|
fallbackUrl?: string,
|
||||||
|
): StepV2 | null {
|
||||||
|
if (!shouldIncludeDraft(step)) return null;
|
||||||
|
|
||||||
|
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||||
|
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||||
|
const effect =
|
||||||
|
"navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined;
|
||||||
|
|
||||||
|
switch (step.op) {
|
||||||
|
case "navigate":
|
||||||
|
return {
|
||||||
|
op: "navigate",
|
||||||
|
id,
|
||||||
|
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||||
|
to: step.url,
|
||||||
|
};
|
||||||
|
case "click": {
|
||||||
|
const target = targetForDraft(step);
|
||||||
|
if (!target) return null;
|
||||||
|
return { op: "click", id, page, target, ...(effect ? { effect } : {}) };
|
||||||
|
}
|
||||||
|
case "hover":
|
||||||
|
return null;
|
||||||
|
case "fill": {
|
||||||
|
const target = targetForDraft(step);
|
||||||
|
if (!target) return null;
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
target,
|
||||||
|
value: step.value,
|
||||||
|
...(step.redacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
key: step.key,
|
||||||
|
...(step.target ? { target: targetForDraft(step) } : {}),
|
||||||
|
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||||
|
...(effect ? { effect } : {}),
|
||||||
|
};
|
||||||
|
case "select": {
|
||||||
|
const target = targetForDraft(step);
|
||||||
|
if (!target) return null;
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
id,
|
||||||
|
page,
|
||||||
|
target,
|
||||||
|
selection: toSelection(step.values, step.labels),
|
||||||
|
...(effect ? { effect } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
case "scroll":
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BuildTraceV2Input {
|
||||||
|
steps: DraftTraceStep[];
|
||||||
|
startedAt: string;
|
||||||
|
startUrl?: string;
|
||||||
|
purpose?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV2(input: BuildTraceV2Input): TraceV2 {
|
||||||
|
const collapsed = collapseNavigations(input.steps);
|
||||||
|
const startUrl =
|
||||||
|
input.startUrl ??
|
||||||
|
collapsed.find(
|
||||||
|
(step): step is Extract<DraftTraceStep, { op: "navigate" }> => step.op === "navigate",
|
||||||
|
)?.url ??
|
||||||
|
collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ??
|
||||||
|
"about:blank";
|
||||||
|
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||||
|
const out: StepV2[] = [];
|
||||||
|
let id = 1;
|
||||||
|
let lastUrl = startUrl;
|
||||||
|
for (const draft of collapsed) {
|
||||||
|
if (draft.op === "navigate") lastUrl = draft.url;
|
||||||
|
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||||
|
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||||
|
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||||
|
if (!step) continue;
|
||||||
|
out.push(step);
|
||||||
|
id += 1;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
entry: { start_url: startUrl },
|
||||||
|
pages,
|
||||||
|
steps: out,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
import type {
|
||||||
|
DraftTraceStep,
|
||||||
|
FillCommit,
|
||||||
|
NavigationCause,
|
||||||
|
Step,
|
||||||
|
TraceState,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
DEFAULT_FILL_COMMIT,
|
||||||
|
fnv1a64,
|
||||||
|
mapNavigationCause,
|
||||||
|
nextStateId,
|
||||||
|
resetStateIdCounterForTests,
|
||||||
|
} from "./record-constants";
|
||||||
|
|
||||||
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
|
|
||||||
|
export function shouldRecordPress(
|
||||||
|
key: string,
|
||||||
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||||
|
): boolean {
|
||||||
|
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||||
|
const mods = modifiers ?? [];
|
||||||
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||||
|
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A reduced draft plus the capture-order ids of every draft folded into it. */
|
||||||
|
interface CollapsedDraft {
|
||||||
|
draft: DraftTraceStep;
|
||||||
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collapse a redirect chain into a single hop.
|
||||||
|
*
|
||||||
|
* The surviving step keeps the *first* hop's origin state and cause — that is
|
||||||
|
* where the user was and why they left — and the *last* hop's destination, so
|
||||||
|
* the intermediate URL the browser only passed through never becomes the
|
||||||
|
* step's `state` or `to`.
|
||||||
|
*/
|
||||||
|
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
|
||||||
|
const out: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const draftId = index + 1;
|
||||||
|
const prev = out[out.length - 1];
|
||||||
|
if (step.op === "navigate" && prev?.draft.op === "navigate") {
|
||||||
|
prev.draft = {
|
||||||
|
...prev.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? prev.draft.postStateId,
|
||||||
|
};
|
||||||
|
prev.draftIds.push(draftId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
out.push({ draft: step, draftIds: [draftId] });
|
||||||
|
});
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StateRegistryEntry extends TraceState {
|
||||||
|
contentHash: string;
|
||||||
|
rawVomText: string;
|
||||||
|
stepsHere: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stateDedupKey(body: string, url: string): string {
|
||||||
|
return `${fnv1a64(body)}:${url}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Register or reuse a page observation in the states dictionary. */
|
||||||
|
export function registerObservation(
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
},
|
||||||
|
): string {
|
||||||
|
const key = stateDedupKey(input.rawVomText, input.url);
|
||||||
|
for (const entry of registry.values()) {
|
||||||
|
if (entry.contentHash === key) return entry.id;
|
||||||
|
}
|
||||||
|
const id = nextStateId();
|
||||||
|
registry.set(id, {
|
||||||
|
id,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
contentHash: key,
|
||||||
|
rawVomText: input.rawVomText,
|
||||||
|
stepsHere: [],
|
||||||
|
});
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toSelection(
|
||||||
|
values: string[],
|
||||||
|
labels?: string[],
|
||||||
|
): Array<{ value: string; label?: string }> {
|
||||||
|
return values.map((value, index) => ({
|
||||||
|
value,
|
||||||
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function navigateCauseForDraft(
|
||||||
|
draft: Extract<DraftTraceStep, { op: "navigate" }>,
|
||||||
|
): NavigationCause {
|
||||||
|
if (draft.cause) return draft.cause;
|
||||||
|
const mapped = mapNavigationCause({
|
||||||
|
transitionType: draft.transitionType,
|
||||||
|
transitionQualifiers: draft.transitionQualifiers,
|
||||||
|
});
|
||||||
|
return mapped ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
|
||||||
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
// An action the user really performed must not disappear because one of its
|
||||||
|
// observations is missing — a capture racing a document swap is routine, and
|
||||||
|
// the last step of a recording is the most exposed to it. Fall back to the
|
||||||
|
// endpoint we do know; only a step with no observation at all has nothing to
|
||||||
|
// point at.
|
||||||
|
const preState = draft.preStateId ?? draft.postStateId;
|
||||||
|
const postState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!preState || !postState) return null;
|
||||||
|
|
||||||
|
const common = { id, state: preState, result: { state: postState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
|
case "navigate":
|
||||||
|
return {
|
||||||
|
op: "navigate",
|
||||||
|
...common,
|
||||||
|
to: draft.url,
|
||||||
|
cause: navigateCauseForDraft(draft),
|
||||||
|
};
|
||||||
|
case "click":
|
||||||
|
return { op: "click", ...common, target: draft.target };
|
||||||
|
case "hover":
|
||||||
|
return { op: "hover", ...common, target: draft.target };
|
||||||
|
case "fill":
|
||||||
|
return {
|
||||||
|
op: "fill",
|
||||||
|
...common,
|
||||||
|
target: draft.target,
|
||||||
|
value: draft.value,
|
||||||
|
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
|
||||||
|
...(draft.redacted ? { redacted: true } : {}),
|
||||||
|
};
|
||||||
|
case "press":
|
||||||
|
return {
|
||||||
|
op: "press",
|
||||||
|
...common,
|
||||||
|
key: draft.key,
|
||||||
|
...(draft.target ? { target: draft.target } : {}),
|
||||||
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
|
};
|
||||||
|
case "select":
|
||||||
|
return {
|
||||||
|
op: "select",
|
||||||
|
...common,
|
||||||
|
target: draft.target,
|
||||||
|
selection: toSelection(draft.values, draft.labels),
|
||||||
|
};
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...common };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReducedTrace {
|
||||||
|
states: TraceState[];
|
||||||
|
steps: Step[];
|
||||||
|
/**
|
||||||
|
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
|
||||||
|
* and dropped drafts make the two diverge, so anything the recorder tagged
|
||||||
|
* with a draft id (`steps_here`, inline annotations) must be remapped.
|
||||||
|
*/
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compile capture drafts into record-only trace v3 steps.
|
||||||
|
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
|
||||||
|
*/
|
||||||
|
export function reduceTraceSteps(
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
): ReducedTrace {
|
||||||
|
const collapsed = collapseNavigations(steps);
|
||||||
|
const out: Step[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
|
let id = 1;
|
||||||
|
for (const { draft, draftIds } of collapsed) {
|
||||||
|
const step = toV3Step(draft, id);
|
||||||
|
if (!step) continue;
|
||||||
|
out.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
|
||||||
|
id += 1;
|
||||||
|
}
|
||||||
|
const states = [...registry.values()].map(
|
||||||
|
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
|
||||||
|
);
|
||||||
|
return { states, steps: out, stepIdByDraftId };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveTraceStartUrl(
|
||||||
|
drafts: DraftTraceStep[],
|
||||||
|
startUrl?: string,
|
||||||
|
states?: TraceState[],
|
||||||
|
): string {
|
||||||
|
if (startUrl) return startUrl;
|
||||||
|
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||||
|
return step.op === "navigate";
|
||||||
|
});
|
||||||
|
if (navigate) return navigate.url;
|
||||||
|
for (const draft of drafts) {
|
||||||
|
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||||
|
}
|
||||||
|
return states?.[0]?.url ?? "about:blank";
|
||||||
|
}
|
||||||
|
|
||||||
|
export type { FillCommit, NavigationCause };
|
||||||
|
/** Test seam */
|
||||||
|
export { resetStateIdCounterForTests };
|
||||||
@@ -33,19 +33,4 @@ describe("RefStore", () => {
|
|||||||
expect(s.resolve("@e11")).toBe(11);
|
expect(s.resolve("@e11")).toBe(11);
|
||||||
expect(s.size()).toBe(2);
|
expect(s.size()).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("preserves the child CDP session as part of ref identity", () => {
|
|
||||||
const s = new RefStore();
|
|
||||||
s.set("e1", 42, {
|
|
||||||
tabId: 7,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
expect(s.resolveEntry("e1")).toMatchObject({
|
|
||||||
backendNodeId: 42,
|
|
||||||
tabId: 7,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
/**
|
/**
|
||||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||||
*
|
*
|
||||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||||
* `replace(...)` with the new ref → node address pairs. A node address
|
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||||
* includes the owning flat CDP session/frame when the element lives in
|
* `tool.click` consume the store via `resolve("@e1")`.
|
||||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
|
||||||
*
|
*
|
||||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||||
* returns `null`, never silently leaks. Storing values in different
|
* returns `null`, never silently leaks. Storing values in different
|
||||||
@@ -16,19 +15,10 @@ export type BackendNodeId = number;
|
|||||||
export interface RefEntry {
|
export interface RefEntry {
|
||||||
backendNodeId: BackendNodeId;
|
backendNodeId: BackendNodeId;
|
||||||
tabId: number | null;
|
tabId: number | null;
|
||||||
frameId?: string;
|
|
||||||
cdpSessionId?: string;
|
|
||||||
generation: number;
|
generation: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RefInput =
|
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||||
| BackendNodeId
|
|
||||||
| {
|
|
||||||
backendNodeId: BackendNodeId;
|
|
||||||
tabId: number;
|
|
||||||
frameId?: string;
|
|
||||||
cdpSessionId?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export class RefStore {
|
export class RefStore {
|
||||||
private readonly map = new Map<string, RefEntry>();
|
private readonly map = new Map<string, RefEntry>();
|
||||||
@@ -54,7 +44,7 @@ export class RefStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||||
* Used after every fresh `tool.snapshot`.
|
* Used after every fresh `tool.snapshot`.
|
||||||
*/
|
*/
|
||||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||||
@@ -63,20 +53,10 @@ export class RefStore {
|
|||||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
set(
|
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||||
ref: string,
|
|
||||||
id: BackendNodeId,
|
|
||||||
opts: {
|
|
||||||
tabId?: number;
|
|
||||||
frameId?: string;
|
|
||||||
cdpSessionId?: string;
|
|
||||||
} = {},
|
|
||||||
): void {
|
|
||||||
this.map.set(normaliseRef(ref), {
|
this.map.set(normaliseRef(ref), {
|
||||||
backendNodeId: id,
|
backendNodeId: id,
|
||||||
tabId: opts.tabId ?? null,
|
tabId: opts.tabId ?? null,
|
||||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
|
||||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -100,8 +80,6 @@ export class RefStore {
|
|||||||
return {
|
return {
|
||||||
backendNodeId: input.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
tabId: input.tabId,
|
tabId: input.tabId,
|
||||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
|
||||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
|
||||||
generation: this.generation,
|
generation: this.generation,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
|
|
||||||
|
|
||||||
const RECORDING_DOCUMENT_ID_PATTERN =
|
|
||||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
||||||
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
|
|
||||||
|
|
||||||
export function recordingDocumentMarkerValue(identity: string): string {
|
|
||||||
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
|
|
||||||
throw new TypeError("recording Document identity must be a random UUID");
|
|
||||||
}
|
|
||||||
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
|
|
||||||
export function readRecordingDocumentIdentity(
|
|
||||||
attrs: Readonly<Record<string, string>>,
|
|
||||||
): string | undefined {
|
|
||||||
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
|
|
||||||
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
|
|
||||||
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
|
|
||||||
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
|
|
||||||
}
|
|
||||||
@@ -612,14 +612,4 @@ describe("isInjectableContentScriptUrl", () => {
|
|||||||
).toBe(false);
|
).toBe(false);
|
||||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects the Edge Add-ons store", () => {
|
|
||||||
expect(
|
|
||||||
isInjectableContentScriptUrl(
|
|
||||||
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
|
||||||
),
|
|
||||||
).toBe(false);
|
|
||||||
// Non-store paths on the same host stay injectable.
|
|
||||||
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import type {
|
|||||||
RequestFrame,
|
RequestFrame,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
|
|
||||||
|
|
||||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
@@ -45,72 +44,9 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
|||||||
|
|
||||||
describe("ToolDispatcher", () => {
|
describe("ToolDispatcher", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
resetBrowserObservationForTests();
|
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("uses the configured recording runtime for start and stop", async () => {
|
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
|
||||||
const sessions = new SessionManager({
|
|
||||||
agentWindow: {
|
|
||||||
create: vi.fn(async () => 4242),
|
|
||||||
remove: vi.fn(),
|
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await sessions.start("aa11");
|
|
||||||
const tab = {
|
|
||||||
id: 7,
|
|
||||||
windowId: 4242,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: "https://example.com/start",
|
|
||||||
} as chrome.tabs.Tab;
|
|
||||||
const frameCoordinator = {
|
|
||||||
begin: vi.fn(),
|
|
||||||
armTab: vi.fn(async () => true),
|
|
||||||
sourceFor: vi.fn(() => null),
|
|
||||||
stop: vi.fn(async () => true),
|
|
||||||
cancel: vi.fn(),
|
|
||||||
};
|
|
||||||
const sendToTab = vi.fn(async () => ({ ok: true }));
|
|
||||||
setBrowserObservationAttachForTests(
|
|
||||||
() => () => {},
|
|
||||||
() => () => {},
|
|
||||||
);
|
|
||||||
const dispatcher = new ToolDispatcher({
|
|
||||||
transport,
|
|
||||||
sessions,
|
|
||||||
recording: {
|
|
||||||
tabsApi: {
|
|
||||||
get: vi.fn(async () => tab),
|
|
||||||
query: vi.fn(async () => [tab]),
|
|
||||||
},
|
|
||||||
frameCoordinator,
|
|
||||||
sendToTab,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
dispatcher.start();
|
|
||||||
|
|
||||||
deliver(
|
|
||||||
makeRequest("tool.record_start", {
|
|
||||||
session_id: "aa11",
|
|
||||||
url: "https://example.com/start",
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
|
||||||
|
|
||||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
|
||||||
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
|
||||||
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
|
||||||
|
|
||||||
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
|
||||||
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
|
||||||
|
|
||||||
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
|
||||||
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
@@ -267,55 +203,6 @@ describe("ToolDispatcher", () => {
|
|||||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
|
||||||
const remove = vi.fn(async () => {});
|
|
||||||
const sessions = new SessionManager({
|
|
||||||
agentWindow: {
|
|
||||||
create: vi.fn(async () => 4242),
|
|
||||||
remove,
|
|
||||||
ensureActiveTab: vi.fn(async () => {}),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await sessions.start("aa11");
|
|
||||||
let finishDetach: () => void = () => {};
|
|
||||||
const detach = new Promise<void>((resolve) => {
|
|
||||||
finishDetach = resolve;
|
|
||||||
});
|
|
||||||
const cdp = {
|
|
||||||
send: vi.fn(),
|
|
||||||
detachSession: vi.fn(() => detach),
|
|
||||||
ensureNetworkCapture: vi.fn(async () => {}),
|
|
||||||
networkEntriesSince: vi.fn(() => ({
|
|
||||||
tab_id: 7,
|
|
||||||
entries: [],
|
|
||||||
next_since: 0,
|
|
||||||
truncated: false,
|
|
||||||
})),
|
|
||||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
|
||||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
|
||||||
setUserAgentOverride: vi.fn(async () => {}),
|
|
||||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
|
||||||
};
|
|
||||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
|
||||||
dispatcher.start();
|
|
||||||
|
|
||||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
|
||||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
|
||||||
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
|
||||||
await flushMicrotasks();
|
|
||||||
finishDetach();
|
|
||||||
await flushMicrotasks();
|
|
||||||
|
|
||||||
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
|
||||||
expect(sent).toContainEqual({
|
|
||||||
id: "r-1",
|
|
||||||
error: expect.objectContaining({ code: "cancelled" }),
|
|
||||||
});
|
|
||||||
expect(sessions.has("aa11")).toBe(true);
|
|
||||||
expect(remove).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns not_found when stopping an unknown session", async () => {
|
it("returns not_found when stopping an unknown session", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
|
|||||||
@@ -1,54 +1,24 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { nodeContentRegion } from "../element-geometry";
|
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
describe("nodeContentRegion", () => {
|
describe("quadBoundingRect", () => {
|
||||||
it("rejects invalid content quads before using the box model fallback", async () => {
|
it("computes axis-aligned bounds from an 8-double quad", () => {
|
||||||
const send = async (_tabId: number, method: string) => {
|
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
||||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
x: 10,
|
||||||
if (method === "DOM.getBoxModel") {
|
y: 20,
|
||||||
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
width: 100,
|
||||||
}
|
height: 40,
|
||||||
throw new Error(`unexpected CDP call ${method}`);
|
|
||||||
};
|
|
||||||
|
|
||||||
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
|
||||||
[
|
|
||||||
{ x: 5, y: 10 },
|
|
||||||
{ x: 25, y: 10 },
|
|
||||||
{ x: 25, y: 30 },
|
|
||||||
{ x: 5, y: 30 },
|
|
||||||
],
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("preserves every visible content fragment", async () => {
|
|
||||||
const send = async () => ({
|
|
||||||
quads: [
|
|
||||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
|
||||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: send as CdpRunner["send"],
|
|
||||||
};
|
|
||||||
|
|
||||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
|
||||||
[
|
|
||||||
{ x: 10, y: 20 },
|
|
||||||
{ x: 60, y: 20 },
|
|
||||||
{ x: 60, y: 40 },
|
|
||||||
{ x: 10, y: 40 },
|
|
||||||
],
|
|
||||||
[
|
|
||||||
{ x: 15, y: 50 },
|
|
||||||
{ x: 90, y: 50 },
|
|
||||||
{ x: 90, y: 80 },
|
|
||||||
{ x: 15, y: 80 },
|
|
||||||
],
|
|
||||||
]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("returns null for degenerate quads", () => {
|
||||||
|
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
||||||
|
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("nodeBoundingRect", () => {
|
||||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||||
const calls: Array<{ method: string; params?: object }> = [];
|
const calls: Array<{ method: string; params?: object }> = [];
|
||||||
const send = async (_tabId: number, method: string, params?: object) => {
|
const send = async (_tabId: number, method: string, params?: object) => {
|
||||||
@@ -68,14 +38,12 @@ describe("nodeContentRegion", () => {
|
|||||||
};
|
};
|
||||||
const cdp = { send: send as CdpRunner["send"] };
|
const cdp = { send: send as CdpRunner["send"] };
|
||||||
|
|
||||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||||
[
|
x: 242,
|
||||||
{ x: 242, y: 468 },
|
y: 468,
|
||||||
{ x: 1011, y: 468 },
|
width: 769,
|
||||||
{ x: 1011, y: 648 },
|
height: 180,
|
||||||
{ x: 242, y: 648 },
|
});
|
||||||
],
|
|
||||||
]);
|
|
||||||
expect(calls.map((c) => c.method)).toEqual([
|
expect(calls.map((c) => c.method)).toEqual([
|
||||||
"DOM.getContentQuads",
|
"DOM.getContentQuads",
|
||||||
"DOM.getBoxModel",
|
"DOM.getBoxModel",
|
||||||
|
|||||||
@@ -1,286 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
|
||||||
import {
|
|
||||||
clipPolygon,
|
|
||||||
polygonArea,
|
|
||||||
polygonBounds,
|
|
||||||
projectAndClipRegion,
|
|
||||||
projectUnitPoint,
|
|
||||||
type Quad,
|
|
||||||
rectPolygon,
|
|
||||||
regionBounds,
|
|
||||||
} from "../geometry";
|
|
||||||
import type { CdpRunner } from "../shared";
|
|
||||||
|
|
||||||
describe("frame geometry projection", () => {
|
|
||||||
it("keeps region bounds separate from polygon area", () => {
|
|
||||||
const region = [
|
|
||||||
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
|
||||||
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
|
||||||
];
|
|
||||||
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
|
||||||
expect(polygonArea(region[0])).toBe(100);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("maps through the iframe content box instead of its border box", () => {
|
|
||||||
const contentQuad: Quad = [
|
|
||||||
{ x: 204, y: 306 },
|
|
||||||
{ x: 604, y: 306 },
|
|
||||||
{ x: 604, y: 506 },
|
|
||||||
{ x: 204, y: 506 },
|
|
||||||
];
|
|
||||||
const projected = projectAndClipRegion(
|
|
||||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
|
||||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
|
||||||
{ width: 1000, height: 800 },
|
|
||||||
);
|
|
||||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
|
||||||
const quad: Quad = [
|
|
||||||
{ x: 0, y: 0 },
|
|
||||||
{ x: 200, y: 20 },
|
|
||||||
{ x: 180, y: 140 },
|
|
||||||
{ x: 20, y: 100 },
|
|
||||||
];
|
|
||||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
|
||||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
|
||||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
|
||||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
|
||||||
const frameQuad: Quad = [
|
|
||||||
{ x: 50, y: 0 },
|
|
||||||
{ x: 100, y: 50 },
|
|
||||||
{ x: 50, y: 100 },
|
|
||||||
{ x: 0, y: 50 },
|
|
||||||
];
|
|
||||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
|
||||||
const send = vi.fn(async (_tabId, method, params) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
|
||||||
return backendNodeId === 20
|
|
||||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
|
||||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected root command ${method}`);
|
|
||||||
});
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: send as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child command ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
const projection = await resolveFrameProjection(
|
|
||||||
cdp,
|
|
||||||
{
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "same-process-parent",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
target: { tabId: 4 },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
frameId: "oopif",
|
|
||||||
parentFrameId: "same-process-parent",
|
|
||||||
ownerBackendNodeId: 20,
|
|
||||||
target: { tabId: 4, sessionId: "oopif-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"oopif",
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(projection?.edges).toEqual([
|
|
||||||
{
|
|
||||||
sourceViewport: { width: 200, height: 100 },
|
|
||||||
destinationQuad: [
|
|
||||||
{ x: 110, y: 70 },
|
|
||||||
{ x: 310, y: 70 },
|
|
||||||
{ x: 310, y: 170 },
|
|
||||||
{ x: 110, y: 170 },
|
|
||||||
],
|
|
||||||
destinationClips: [
|
|
||||||
[
|
|
||||||
{ x: 100, y: 50 },
|
|
||||||
{ x: 400, y: 50 },
|
|
||||||
{ x: 400, y: 250 },
|
|
||||||
{ x: 100, y: 250 },
|
|
||||||
],
|
|
||||||
],
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected root command ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child command ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
getFrameGraph: vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
})),
|
|
||||||
};
|
|
||||||
|
|
||||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
frameId: "child",
|
|
||||||
backendNodeId: 101,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(geometry).toMatchObject({
|
|
||||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
|
||||||
actionPoint: { x: 200, y: 150 },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
|
||||||
const getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method) => {
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected root command ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child command ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
getFrameGraph,
|
|
||||||
};
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
resolveNodeGeometry(
|
|
||||||
cdp,
|
|
||||||
4,
|
|
||||||
{
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
frameId: "child",
|
|
||||||
backendNodeId: 101,
|
|
||||||
},
|
|
||||||
{ scrollIntoView: true },
|
|
||||||
),
|
|
||||||
).resolves.not.toMatchObject({ code: expect.any(String) });
|
|
||||||
expect(getFrameGraph).toHaveBeenCalledOnce();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(async (_tabId, method) => {
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
|
||||||
}
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected ${method}`);
|
|
||||||
}) as CdpRunner["send"],
|
|
||||||
getFrameGraph: vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 10,
|
|
||||||
target: { tabId: 4 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
})),
|
|
||||||
};
|
|
||||||
|
|
||||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
|
||||||
target: { tabId: 4 },
|
|
||||||
frameId: "child",
|
|
||||||
backendNodeId: 101,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(geometry).toMatchObject({
|
|
||||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
|
||||||
actionPoint: { x: 200, y: 150 },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
|
||||||
const cdp: CdpRunner = {
|
|
||||||
send: vi.fn(),
|
|
||||||
sendToTarget: vi.fn(async (_target, method) => {
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"],
|
|
||||||
};
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
resolveNodeGeometry(cdp, 4, {
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
frameId: "child",
|
|
||||||
backendNodeId: 101,
|
|
||||||
}),
|
|
||||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import { RefStore } from "@/session-manager/ref-store";
|
|
||||||
import type { RequestHelpParams } from "@/transport/types";
|
import type { RequestHelpParams } from "@/transport/types";
|
||||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
|
|
||||||
@@ -51,10 +50,11 @@ function installHelpLifecycleChrome() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||||
const refStore = new RefStore();
|
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
id === sessionId
|
||||||
|
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||||
|
: null,
|
||||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
return mgr;
|
return mgr;
|
||||||
@@ -339,23 +339,26 @@ describe("handleRequestHelp", () => {
|
|||||||
|
|
||||||
ac.abort();
|
ac.abort();
|
||||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
expect(cdpSend).toHaveBeenCalledWith(
|
||||||
expect.anything(),
|
5,
|
||||||
"DOM.removeAttribute",
|
"DOM.querySelectorAll",
|
||||||
expect.anything(),
|
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||||
|
);
|
||||||
|
expect(cdpSend).toHaveBeenCalledWith(
|
||||||
|
6,
|
||||||
|
"DOM.querySelectorAll",
|
||||||
|
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
it("tags ref targets via CDP and reports them matched", async () => {
|
||||||
const refStore = new RefStore();
|
|
||||||
refStore.set("e1", 42, { tabId: 5 });
|
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore,
|
refStore: { resolve: () => 42 },
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
@@ -363,17 +366,7 @@ describe("handleRequestHelp", () => {
|
|||||||
} as unknown as SessionManager;
|
} as unknown as SessionManager;
|
||||||
const deps = baseDeps({
|
const deps = baseDeps({
|
||||||
cdp: {
|
cdp: {
|
||||||
send: vi.fn(async (_tabId, method) => {
|
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected ${method}`);
|
|
||||||
}),
|
|
||||||
} as unknown as RequestHelpDeps["cdp"],
|
} as unknown as RequestHelpDeps["cdp"],
|
||||||
});
|
});
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -382,97 +375,14 @@ describe("handleRequestHelp", () => {
|
|||||||
deps,
|
deps,
|
||||||
);
|
);
|
||||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||||
expect(sentMsg.selectors).toEqual([]);
|
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
|
||||||
expect(res).toMatchObject({
|
expect(res).toMatchObject({
|
||||||
outcome: "continued",
|
outcome: "continued",
|
||||||
tab_id: 5,
|
tab_id: 5,
|
||||||
resolved_targets: [
|
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||||
{
|
|
||||||
matched: true,
|
|
||||||
ref: "@e1",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
|
||||||
const mgr = {
|
|
||||||
get: (id: string) =>
|
|
||||||
id === "abcd"
|
|
||||||
? {
|
|
||||||
sessionId: "abcd",
|
|
||||||
agentWindowId: 99,
|
|
||||||
refStore: {
|
|
||||||
resolveEntry: () => ({
|
|
||||||
backendNodeId: 42,
|
|
||||||
tabId: 5,
|
|
||||||
frameId: "child",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
generation: 1,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
borrowedTabs: new Map(),
|
|
||||||
}
|
|
||||||
: null,
|
|
||||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
|
||||||
} as unknown as SessionManager;
|
|
||||||
const send = vi.fn(async (_tabId, method) => {
|
|
||||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "DOM.getBoxModel") {
|
|
||||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected root ${method}`);
|
|
||||||
});
|
|
||||||
const sendToTarget = vi.fn(async (_target, method) => {
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child ${method}`);
|
|
||||||
});
|
|
||||||
const deps = baseDeps({
|
|
||||||
cdp: {
|
|
||||||
send,
|
|
||||||
sendToTarget,
|
|
||||||
getFrameGraph: vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 5 } },
|
|
||||||
{
|
|
||||||
frameId: "child",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 99,
|
|
||||||
target: { tabId: 5, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
})),
|
|
||||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
|
||||||
});
|
|
||||||
|
|
||||||
const res = await handleRequestHelp(
|
|
||||||
mgr,
|
|
||||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
|
||||||
deps,
|
|
||||||
);
|
|
||||||
|
|
||||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
|
||||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
|
||||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
|
||||||
expect(sendToTarget).toHaveBeenCalledWith(
|
|
||||||
{ tabId: 5, sessionId: "child-session" },
|
|
||||||
"DOM.getContentQuads",
|
|
||||||
{ backendNodeId: 42 },
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||||
const deps = baseDeps();
|
const deps = baseDeps();
|
||||||
const res = await handleRequestHelp(
|
const res = await handleRequestHelp(
|
||||||
@@ -488,15 +398,16 @@ describe("handleRequestHelp", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||||
const refStore = new RefStore();
|
|
||||||
refStore.set("e1", 42, { tabId: 4 });
|
|
||||||
const mgr = {
|
const mgr = {
|
||||||
get: (id: string) =>
|
get: (id: string) =>
|
||||||
id === "abcd"
|
id === "abcd"
|
||||||
? {
|
? {
|
||||||
sessionId: "abcd",
|
sessionId: "abcd",
|
||||||
agentWindowId: 99,
|
agentWindowId: 99,
|
||||||
refStore,
|
refStore: {
|
||||||
|
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||||
|
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||||
|
},
|
||||||
borrowedTabs: new Map(),
|
borrowedTabs: new Map(),
|
||||||
}
|
}
|
||||||
: null,
|
: null,
|
||||||
|
|||||||
@@ -30,9 +30,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||||
sent.push({ tabId, method, params });
|
sent.push({ tabId, method, params });
|
||||||
const h = handlers[method];
|
const h = handlers[method];
|
||||||
if (!h && method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
|
||||||
}
|
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -167,63 +164,6 @@ describe("handleClick", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e3", 1234, {
|
|
||||||
tabId: 4,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"Input.dispatchMouseEvent": () => ({}),
|
|
||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
|
||||||
"DOM.getBoxModel": () => ({
|
|
||||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
|
||||||
rootFrameId: "main",
|
|
||||||
frames: [
|
|
||||||
{ frameId: "main", target: { tabId: 4 } },
|
|
||||||
{
|
|
||||||
frameId: "child-frame",
|
|
||||||
parentFrameId: "main",
|
|
||||||
ownerBackendNodeId: 99,
|
|
||||||
target: { tabId: 4, sessionId: "child-session" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}));
|
|
||||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
|
||||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
|
||||||
targetCalls.push({ sessionId: target.sessionId, method });
|
|
||||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
|
||||||
if (method === "DOM.getContentQuads") {
|
|
||||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
|
||||||
}
|
|
||||||
if (method === "Page.getLayoutMetrics") {
|
|
||||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
|
||||||
}
|
|
||||||
throw new Error(`unexpected child CDP call ${method}`);
|
|
||||||
}) as CdpRunner["sendToTarget"];
|
|
||||||
|
|
||||||
const res = await handleClick(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", ref: "@e3" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
|
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
|
||||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
|
||||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
|
||||||
expect(targetCalls).toEqual([
|
|
||||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
|
||||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
|
||||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
|
||||||
]);
|
|
||||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||||
const order: string[] = [];
|
const order: string[] = [];
|
||||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,11 +1,19 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RECORD_FINISH, RECORD_START, RECORD_STEP, RECORD_STOP } from "@/lib/record-bridge";
|
import {
|
||||||
|
RECORD_FINISH,
|
||||||
|
RECORD_QUERY,
|
||||||
|
RECORD_START,
|
||||||
|
RECORD_STEP,
|
||||||
|
RECORD_STOP,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
import { resetStateIdCounterForTests } from "@/lib/trace-reducer";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type { CdpRunner } from "@/tools/shared";
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type { RecordedTrace, RecordStopResult, TraceV3 } from "@/transport/types";
|
import type { RecordedTrace, RecordStopResult, TraceV3 } from "@/transport/types";
|
||||||
import {
|
import {
|
||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
|
attachRecordQueryListener,
|
||||||
attachRecordStepListener,
|
attachRecordStepListener,
|
||||||
handleRecordStart,
|
handleRecordStart,
|
||||||
handleRecordStop,
|
handleRecordStop,
|
||||||
@@ -15,13 +23,7 @@ import {
|
|||||||
const AGENT_WINDOW_ID = 100;
|
const AGENT_WINDOW_ID = 100;
|
||||||
const TAB_ID = 4;
|
const TAB_ID = 4;
|
||||||
const START_URL = "https://example.com/";
|
const START_URL = "https://example.com/";
|
||||||
const RECORD_START_V3 = {
|
const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const };
|
||||||
session_id: "abcd",
|
|
||||||
url: START_URL,
|
|
||||||
trace_version: 3 as const,
|
|
||||||
supports_tab_switch_steps: true,
|
|
||||||
};
|
|
||||||
const stepSequenceByProducer = new Map<string, number>();
|
|
||||||
|
|
||||||
function asTraceV3(trace: RecordedTrace): TraceV3 {
|
function asTraceV3(trace: RecordedTrace): TraceV3 {
|
||||||
if (!("version" in trace)) {
|
if (!("version" in trace)) {
|
||||||
@@ -53,7 +55,6 @@ function chromeEvent<T extends (...args: never[]) => unknown>() {
|
|||||||
|
|
||||||
function installChrome() {
|
function installChrome() {
|
||||||
const runtimeOnMessage = chromeEvent<RuntimeListener>();
|
const runtimeOnMessage = chromeEvent<RuntimeListener>();
|
||||||
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
|
|
||||||
const webNavigationOnCompleted =
|
const webNavigationOnCompleted =
|
||||||
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
|
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
|
||||||
const webNavigationOnCommitted =
|
const webNavigationOnCommitted =
|
||||||
@@ -63,7 +64,7 @@ function installChrome() {
|
|||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
runtime: { onMessage: runtimeOnMessage },
|
runtime: { onMessage: runtimeOnMessage },
|
||||||
tabs: {
|
tabs: {
|
||||||
onActivated: tabsOnActivated,
|
onActivated: chromeEvent(),
|
||||||
onCreated: chromeEvent(),
|
onCreated: chromeEvent(),
|
||||||
onUpdated: chromeEvent(),
|
onUpdated: chromeEvent(),
|
||||||
},
|
},
|
||||||
@@ -72,7 +73,7 @@ function installChrome() {
|
|||||||
onCommitted: webNavigationOnCommitted,
|
onCommitted: webNavigationOnCommitted,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted };
|
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
|
||||||
}
|
}
|
||||||
|
|
||||||
function fakeManager() {
|
function fakeManager() {
|
||||||
@@ -160,9 +161,9 @@ function makeFakeCdp(
|
|||||||
return { result: { value: { idleMs, readyState: "complete" } } };
|
return { result: { value: { idleMs, readyState: "complete" } } };
|
||||||
},
|
},
|
||||||
"Accessibility.enable": () => ({}),
|
"Accessibility.enable": () => ({}),
|
||||||
"Accessibility.getFullAXTree": (_params, _tabId) => {
|
"Accessibility.getFullAXTree": (_params, tabId) => {
|
||||||
if (failing) throw new Error("Execution context was destroyed");
|
if (failing) throw new Error("Execution context was destroyed");
|
||||||
const tabTree = options?.treesByTab?.[_tabId];
|
const tabTree = options?.treesByTab?.[tabId];
|
||||||
if (tabTree) return tabTree;
|
if (tabTree) return tabTree;
|
||||||
if (script.length === 0) return axTree("Example Domain", ["Submit"]);
|
if (script.length === 0) return axTree("Example Domain", ["Submit"]);
|
||||||
return script.length === 1 ? script[0] : script.shift();
|
return script.length === 1 ? script[0] : script.shift();
|
||||||
@@ -227,13 +228,8 @@ function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
|
|||||||
(query.windowId === undefined || tab.windowId === query.windowId) &&
|
(query.windowId === undefined || tab.windowId === query.windowId) &&
|
||||||
(query.active === undefined || tab.active === query.active),
|
(query.active === undefined || tab.active === query.active),
|
||||||
),
|
),
|
||||||
activate(tabId: number) {
|
close(tabId: number) {
|
||||||
for (const tab of byId.values()) tab.active = tab.id === tabId;
|
byId.delete(tabId);
|
||||||
},
|
|
||||||
goTo(tabId: number, url: string, title: string) {
|
|
||||||
const tab = byId.get(tabId);
|
|
||||||
if (!tab) throw new Error(`tab ${tabId} closed`);
|
|
||||||
Object.assign(tab, { url, title });
|
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -247,26 +243,18 @@ function runtimeOnMessageEmit(
|
|||||||
chromeApi: ReturnType<typeof installChrome>,
|
chromeApi: ReturnType<typeof installChrome>,
|
||||||
requestId: string,
|
requestId: string,
|
||||||
step: unknown,
|
step: unknown,
|
||||||
tabId = TAB_ID,
|
): void {
|
||||||
active = true,
|
|
||||||
): ReturnType<typeof vi.fn> {
|
|
||||||
const producerId = `test-producer-${tabId}`;
|
|
||||||
const producerKey = `${requestId}:${producerId}`;
|
|
||||||
const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1;
|
|
||||||
stepSequenceByProducer.set(producerKey, sequence);
|
|
||||||
const sendResponse = vi.fn();
|
|
||||||
chromeApi.runtimeOnMessage.emit(
|
chromeApi.runtimeOnMessage.emit(
|
||||||
{ type: RECORD_STEP, requestId, producerId, sequence, step },
|
{ type: RECORD_STEP, requestId, step },
|
||||||
{ tab: { id: tabId, active } } as chrome.runtime.MessageSender,
|
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
||||||
sendResponse,
|
() => {},
|
||||||
);
|
);
|
||||||
return sendResponse;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("recorded user steps reach the exported trace", () => {
|
describe("recorded user steps reach the exported trace", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
stepSequenceByProducer.clear();
|
|
||||||
resetBrowserObservationForTests();
|
resetBrowserObservationForTests();
|
||||||
|
resetStateIdCounterForTests();
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -296,14 +284,15 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
{
|
{
|
||||||
type: RECORD_STEP,
|
type: RECORD_STEP,
|
||||||
requestId,
|
requestId,
|
||||||
producerId: "test-producer-4",
|
|
||||||
sequence: 1,
|
|
||||||
step: {
|
step: {
|
||||||
op: "click",
|
op: "click",
|
||||||
page_url: START_URL,
|
page_url: START_URL,
|
||||||
target: { role: "button", name: "Submit", tag: "button" },
|
target: { role: "button", name: "Submit", tag: "button" },
|
||||||
geometry: {
|
geometry: {
|
||||||
rect: { x: 0, y: 0, w: 10, h: 10 },
|
rect: { x: 0, y: 0, w: 10, h: 10 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
tag: "button",
|
tag: "button",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -339,37 +328,21 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
await handleRecordStart(manager, RECORD_START_V3, { tabsApi, sendToTab, cdp: makeFakeCdp() });
|
await handleRecordStart(manager, RECORD_START_V3, { tabsApi, sendToTab, cdp: makeFakeCdp() });
|
||||||
attachRecordStepListener({ tabsApi, sendToTab });
|
attachRecordStepListener({ tabsApi, sendToTab });
|
||||||
|
|
||||||
let sequence = 0;
|
|
||||||
const emit = (step: unknown) => {
|
const emit = (step: unknown) => {
|
||||||
sequence += 1;
|
|
||||||
runtimeOnMessage.emit(
|
runtimeOnMessage.emit(
|
||||||
{ type: RECORD_STEP, requestId, producerId: "test-producer-4", sequence, step },
|
{ type: RECORD_STEP, requestId, step },
|
||||||
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
||||||
() => {},
|
() => {},
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
const fill = {
|
emit({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
page_url: START_URL,
|
page_url: START_URL,
|
||||||
target: { role: "textbox", name: "Search", tag: "input" },
|
target: { role: "textbox", name: "Search", tag: "input" },
|
||||||
value: "hello",
|
value: "hello",
|
||||||
commit: "enter",
|
commit: "enter",
|
||||||
};
|
});
|
||||||
emit(fill);
|
|
||||||
const duplicateAck = vi.fn();
|
|
||||||
runtimeOnMessage.emit(
|
|
||||||
{
|
|
||||||
type: RECORD_STEP,
|
|
||||||
requestId,
|
|
||||||
producerId: "test-producer-4",
|
|
||||||
sequence: 1,
|
|
||||||
step: fill,
|
|
||||||
},
|
|
||||||
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
|
||||||
duplicateAck,
|
|
||||||
);
|
|
||||||
expect(duplicateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
|
|
||||||
emit({
|
emit({
|
||||||
op: "click",
|
op: "click",
|
||||||
page_url: START_URL,
|
page_url: START_URL,
|
||||||
@@ -438,13 +411,13 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
});
|
});
|
||||||
attachRecordStepListener({ tabsApi, sendToTab });
|
attachRecordStepListener({ tabsApi, sendToTab });
|
||||||
|
|
||||||
const commit = (url: string, transitionType: string, transitionQualifiers: string[] = []) => {
|
const commit = (url: string, transitionType: string) => {
|
||||||
chromeApi.webNavigationOnCommitted.emit({
|
chromeApi.webNavigationOnCommitted.emit({
|
||||||
tabId: TAB_ID,
|
tabId: TAB_ID,
|
||||||
frameId: 0,
|
frameId: 0,
|
||||||
url,
|
url,
|
||||||
transitionType,
|
transitionType,
|
||||||
transitionQualifiers,
|
transitionQualifiers: [],
|
||||||
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -455,7 +428,7 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
await settleWait();
|
await settleWait();
|
||||||
|
|
||||||
tabsApi.goTo("https://iwiki.woa.com/dashboard", "工作台 - 腾讯iWiki");
|
tabsApi.goTo("https://iwiki.woa.com/dashboard", "工作台 - 腾讯iWiki");
|
||||||
commit("https://iwiki.woa.com/dashboard", "link", ["server_redirect"]);
|
commit("https://iwiki.woa.com/dashboard", "link");
|
||||||
await settleWait();
|
await settleWait();
|
||||||
|
|
||||||
runtimeOnMessageEmit(chromeApi, requestId, {
|
runtimeOnMessageEmit(chromeApi, requestId, {
|
||||||
@@ -1106,52 +1079,61 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
expect(asTraceV3((stopped as RecordStopResult).trace).stopped_by).toBe("user_finish");
|
expect(asTraceV3((stopped as RecordStopResult).trace).stopped_by).toBe("user_finish");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns a shared finish failure to a concurrent CLI stop", async () => {
|
it("retries STOP only on armed tabs whose final delivery did not ack", async () => {
|
||||||
const chromeApi = installChrome();
|
const chromeApi = installChrome();
|
||||||
const manager = fakeManager();
|
const manager = fakeManager();
|
||||||
const tabsApi = makeTabsApi();
|
const firstTab = {
|
||||||
let requestId = "";
|
id: TAB_ID,
|
||||||
let releaseStop!: () => void;
|
windowId: AGENT_WINDOW_ID,
|
||||||
const stopReleased = new Promise<void>((resolve) => {
|
active: true,
|
||||||
releaseStop = resolve;
|
status: "complete",
|
||||||
});
|
url: START_URL,
|
||||||
let announceStop!: () => void;
|
title: "First",
|
||||||
const stopStarted = new Promise<void>((resolve) => {
|
} as chrome.tabs.Tab;
|
||||||
announceStop = resolve;
|
const secondTab = {
|
||||||
});
|
id: 5,
|
||||||
const sendToTab = vi.fn(async (_tabId: number, msg: unknown) => {
|
windowId: AGENT_WINDOW_ID,
|
||||||
const typed = msg as { type?: string; requestId?: string };
|
active: false,
|
||||||
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
|
status: "complete",
|
||||||
|
url: "https://example.com/second",
|
||||||
|
title: "Second",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
|
||||||
|
const stopTabs: number[] = [];
|
||||||
|
let secondStopAttempts = 0;
|
||||||
|
const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
|
||||||
|
const typed = msg as { type?: string };
|
||||||
if (typed.type === RECORD_STOP) {
|
if (typed.type === RECORD_STOP) {
|
||||||
announceStop();
|
stopTabs.push(tabId);
|
||||||
await stopReleased;
|
if (tabId === 5 && secondStopAttempts++ === 0) {
|
||||||
return { ok: false, error: "final capture failed" };
|
return { ok: false, error: "failed to deliver final fill" };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
});
|
});
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
await handleRecordStart(manager, RECORD_START_V3, deps);
|
||||||
attachRecordFinishListener(deps);
|
attachRecordQueryListener(deps);
|
||||||
chromeApi.runtimeOnMessage.emit(
|
await new Promise<void>((resolve) => {
|
||||||
{ type: RECORD_FINISH, requestId },
|
chromeApi.runtimeOnMessage.emit(
|
||||||
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
{ type: RECORD_QUERY },
|
||||||
() => {},
|
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||||
);
|
() => resolve(),
|
||||||
await stopStarted;
|
);
|
||||||
|
});
|
||||||
|
|
||||||
const cliStop = handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
releaseStop();
|
expect(firstStop).toMatchObject({ code: "protocol_error" });
|
||||||
const result = await Promise.race([
|
expect(stopTabs).toEqual([TAB_ID, 5]);
|
||||||
cliStop,
|
|
||||||
new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 500)),
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(result).not.toBe("timed_out");
|
const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
expect(result).toMatchObject({ code: "protocol_error" });
|
const retryTrace = (retry as RecordStopResult).trace;
|
||||||
|
expect("version" in retryTrace && retryTrace.version).toBe(3);
|
||||||
|
expect(stopTabs).toEqual([TAB_ID, 5, 5]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("defaults to legacy trace v2 without starting observation capture", async () => {
|
it("defaults to trace v2 when trace_version is omitted", async () => {
|
||||||
const { runtimeOnMessage } = installChrome();
|
const { runtimeOnMessage } = installChrome();
|
||||||
const manager = fakeManager();
|
const manager = fakeManager();
|
||||||
const tabsApi = makeTabsApi();
|
const tabsApi = makeTabsApi();
|
||||||
@@ -1169,23 +1151,6 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
{
|
{
|
||||||
type: RECORD_STEP,
|
type: RECORD_STEP,
|
||||||
requestId,
|
requestId,
|
||||||
producerId: "test-producer-4",
|
|
||||||
sequence: 1,
|
|
||||||
step: {
|
|
||||||
op: "hover",
|
|
||||||
page_url: START_URL,
|
|
||||||
target: { role: "button", name: "Submit", tag: "button" },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{ tab: { id: TAB_ID } } as chrome.runtime.MessageSender,
|
|
||||||
() => {},
|
|
||||||
);
|
|
||||||
runtimeOnMessage.emit(
|
|
||||||
{
|
|
||||||
type: RECORD_STEP,
|
|
||||||
requestId,
|
|
||||||
producerId: "test-producer-4",
|
|
||||||
sequence: 2,
|
|
||||||
step: {
|
step: {
|
||||||
op: "click",
|
op: "click",
|
||||||
page_url: START_URL,
|
page_url: START_URL,
|
||||||
@@ -1200,8 +1165,6 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
const trace = (stopped as RecordStopResult).trace;
|
const trace = (stopped as RecordStopResult).trace;
|
||||||
expect("version" in trace).toBe(false);
|
expect("version" in trace).toBe(false);
|
||||||
expect("pages" in trace && trace.pages.length).toBeGreaterThan(0);
|
expect("pages" in trace && trace.pages.length).toBeGreaterThan(0);
|
||||||
// `hover` has no v2 counterpart on peers that negotiate v2.
|
|
||||||
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["click"]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects unsupported trace_version values", async () => {
|
it("rejects unsupported trace_version values", async () => {
|
||||||
@@ -1214,37 +1177,142 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
expect(result).toMatchObject({ code: "invalid_params" });
|
expect(result).toMatchObject({ code: "invalid_params" });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("records tab transitions and binds actions to each tab's observation", async () => {
|
it("does not let a closed armed tab block finish", async () => {
|
||||||
const chromeApi = installChrome();
|
const chromeApi = installChrome();
|
||||||
const manager = fakeManager();
|
const manager = fakeManager();
|
||||||
const secondUrl = "https://example.com/second";
|
const firstTab = {
|
||||||
const tabsApi = makeMultiTabsApi([
|
id: TAB_ID,
|
||||||
{
|
windowId: AGENT_WINDOW_ID,
|
||||||
id: TAB_ID,
|
active: true,
|
||||||
windowId: AGENT_WINDOW_ID,
|
status: "complete",
|
||||||
active: true,
|
url: START_URL,
|
||||||
status: "complete",
|
} as chrome.tabs.Tab;
|
||||||
url: START_URL,
|
const secondTab = {
|
||||||
title: "First tab",
|
id: 5,
|
||||||
} as chrome.tabs.Tab,
|
windowId: AGENT_WINDOW_ID,
|
||||||
{
|
active: false,
|
||||||
id: 5,
|
status: "complete",
|
||||||
windowId: AGENT_WINDOW_ID,
|
url: "https://example.com/second",
|
||||||
active: false,
|
} as chrome.tabs.Tab;
|
||||||
status: "complete",
|
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
|
||||||
url: secondUrl,
|
const stopTabs: number[] = [];
|
||||||
title: "Second tab",
|
const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
|
||||||
} as chrome.tabs.Tab,
|
if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId);
|
||||||
]);
|
return { ok: true };
|
||||||
const cdp = makeFakeCdp(undefined, {
|
|
||||||
treesByTab: {
|
|
||||||
[TAB_ID]: axTree("First tab", ["First action"]),
|
|
||||||
5: axTree("Second tab", ["Second action"]),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||||
|
|
||||||
|
await handleRecordStart(manager, RECORD_START_V3, deps);
|
||||||
|
attachRecordQueryListener(deps);
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
chromeApi.runtimeOnMessage.emit(
|
||||||
|
{ type: RECORD_QUERY },
|
||||||
|
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||||
|
() => resolve(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
tabsApi.close(5);
|
||||||
|
|
||||||
|
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
|
|
||||||
|
const stoppedTrace = (stopped as RecordStopResult).trace;
|
||||||
|
expect("version" in stoppedTrace && stoppedTrace.version).toBe(3);
|
||||||
|
expect(stopTabs).toEqual([TAB_ID]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => {
|
||||||
|
const chromeApi = installChrome();
|
||||||
|
const manager = fakeManager();
|
||||||
|
const firstTab = {
|
||||||
|
id: TAB_ID,
|
||||||
|
windowId: AGENT_WINDOW_ID,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
|
url: START_URL,
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const secondTab = {
|
||||||
|
id: 5,
|
||||||
|
windowId: AGENT_WINDOW_ID,
|
||||||
|
active: false,
|
||||||
|
status: "complete",
|
||||||
|
url: "https://example.com/second",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
|
||||||
|
let releaseSecondStart!: () => void;
|
||||||
|
const secondStartGate = new Promise<void>((resolve) => {
|
||||||
|
releaseSecondStart = resolve;
|
||||||
|
});
|
||||||
|
let announceSecondStart!: () => void;
|
||||||
|
const secondStartBegan = new Promise<void>((resolve) => {
|
||||||
|
announceSecondStart = resolve;
|
||||||
|
});
|
||||||
|
const stopTabs: number[] = [];
|
||||||
|
const sendToTab = vi.fn(async (tabId: number, msg: unknown) => {
|
||||||
|
const type = (msg as { type?: string }).type;
|
||||||
|
if (type === RECORD_START && tabId === 5) {
|
||||||
|
announceSecondStart();
|
||||||
|
await secondStartGate;
|
||||||
|
}
|
||||||
|
if (type === RECORD_STOP) stopTabs.push(tabId);
|
||||||
|
return { ok: true };
|
||||||
|
});
|
||||||
|
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||||
|
|
||||||
|
await handleRecordStart(manager, RECORD_START_V3, deps);
|
||||||
|
attachRecordQueryListener(deps);
|
||||||
|
chromeApi.runtimeOnMessage.emit(
|
||||||
|
{ type: RECORD_QUERY },
|
||||||
|
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||||
|
() => {},
|
||||||
|
);
|
||||||
|
await secondStartBegan;
|
||||||
|
|
||||||
|
const stopping = handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||||
|
releaseSecondStart();
|
||||||
|
const stopped = await stopping;
|
||||||
|
|
||||||
|
const stoppedTrace = (stopped as RecordStopResult).trace;
|
||||||
|
expect("version" in stoppedTrace && stoppedTrace.version).toBe(3);
|
||||||
|
expect(stopTabs).toEqual([TAB_ID, 5]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drains a navigation callback received while finish is settling", async () => {
|
||||||
|
const chromeApi = installChrome();
|
||||||
|
const manager = fakeManager();
|
||||||
|
const firstTab = {
|
||||||
|
id: TAB_ID,
|
||||||
|
windowId: AGENT_WINDOW_ID,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
|
url: START_URL,
|
||||||
|
title: "Editor",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const secondUrl = "https://example.com/late-navigation";
|
||||||
|
const secondTab = {
|
||||||
|
id: 5,
|
||||||
|
windowId: AGENT_WINDOW_ID,
|
||||||
|
active: false,
|
||||||
|
status: "complete",
|
||||||
|
url: secondUrl,
|
||||||
|
title: "Late page",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
let releaseLookup!: () => void;
|
||||||
|
const lookupGate = new Promise<void>((resolve) => {
|
||||||
|
releaseLookup = resolve;
|
||||||
|
});
|
||||||
|
const baseTabsApi = makeMultiTabsApi([firstTab, secondTab]);
|
||||||
|
const tabsApi = {
|
||||||
|
...baseTabsApi,
|
||||||
|
get: async (tabId: number) => {
|
||||||
|
if (tabId === 5) await lookupGate;
|
||||||
|
return baseTabsApi.get(tabId);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const cdp = makeFakeCdp([axTree("Editor", ["Save"]), axTree("Saved")]);
|
||||||
let requestId = "";
|
let requestId = "";
|
||||||
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
|
const sendToTab = vi.fn(async (_tabId: number, msg: unknown) => {
|
||||||
const typed = message as { type?: string; requestId?: string };
|
const typed = msg as { type?: string; requestId?: string };
|
||||||
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
|
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
});
|
});
|
||||||
@@ -1252,352 +1320,80 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
await handleRecordStart(manager, RECORD_START_V3, deps);
|
||||||
attachRecordStepListener(deps);
|
attachRecordStepListener(deps);
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
runtimeOnMessageEmit(
|
|
||||||
chromeApi,
|
|
||||||
requestId,
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
page_url: secondUrl,
|
|
||||||
target: { role: "button", name: "Second action", tag: "button" },
|
|
||||||
},
|
|
||||||
5,
|
|
||||||
);
|
|
||||||
await settleWait();
|
|
||||||
|
|
||||||
tabsApi.activate(TAB_ID);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
runtimeOnMessageEmit(chromeApi, requestId, {
|
runtimeOnMessageEmit(chromeApi, requestId, {
|
||||||
op: "click",
|
op: "click",
|
||||||
page_url: START_URL,
|
page_url: START_URL,
|
||||||
target: { role: "button", name: "First action", tag: "button" },
|
target: { role: "button", name: "Save", tag: "button" },
|
||||||
});
|
});
|
||||||
await settleWait();
|
|
||||||
|
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const stopping = handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
|
||||||
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
|
|
||||||
const switches = trace.steps.filter((step) => step.op === "switch_tab");
|
|
||||||
const clicks = trace.steps.filter((step) => step.op === "click");
|
|
||||||
|
|
||||||
expect(switches).toHaveLength(2);
|
|
||||||
expect(clicks).toHaveLength(2);
|
|
||||||
expect(stateUrl(switches[0]!.state)).toBe(START_URL);
|
|
||||||
expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl);
|
|
||||||
expect(stateUrl(clicks[0]!.state)).toBe(secondUrl);
|
|
||||||
expect(stateUrl(switches[1]!.result.state)).toBe(START_URL);
|
|
||||||
expect(stateUrl(clicks[1]!.state)).toBe(START_URL);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps switch_tab internal when the v3 caller did not advertise support", async () => {
|
|
||||||
const chromeApi = installChrome();
|
|
||||||
const manager = fakeManager();
|
|
||||||
const tabsApi = makeMultiTabsApi([
|
|
||||||
{
|
|
||||||
id: TAB_ID,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: START_URL,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
{
|
|
||||||
id: 5,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: false,
|
|
||||||
status: "complete",
|
|
||||||
url: "https://example.com/second",
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
]);
|
|
||||||
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
|
|
||||||
|
|
||||||
await handleRecordStart(
|
|
||||||
manager,
|
|
||||||
{ session_id: "abcd", url: START_URL, trace_version: 3 },
|
|
||||||
deps,
|
|
||||||
);
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
|
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
|
||||||
expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => {
|
|
||||||
const chromeApi = installChrome();
|
|
||||||
const manager = fakeManager();
|
|
||||||
const tabsApi = makeMultiTabsApi(
|
|
||||||
[TAB_ID, 5, 6].map(
|
|
||||||
(id) =>
|
|
||||||
({
|
|
||||||
id,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: id === TAB_ID,
|
|
||||||
status: "complete",
|
|
||||||
url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`,
|
|
||||||
}) as chrome.tabs.Tab,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
let releaseSecond!: () => void;
|
|
||||||
const secondGate = new Promise<void>((resolve) => {
|
|
||||||
releaseSecond = resolve;
|
|
||||||
});
|
|
||||||
let announceSecond!: () => void;
|
|
||||||
const secondStarted = new Promise<void>((resolve) => {
|
|
||||||
announceSecond = resolve;
|
|
||||||
});
|
|
||||||
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
|
|
||||||
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
|
|
||||||
announceSecond();
|
|
||||||
await secondGate;
|
|
||||||
}
|
|
||||||
return { ok: true };
|
|
||||||
});
|
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await secondStarted;
|
|
||||||
tabsApi.activate(6);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
releaseSecond();
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
||||||
|
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
|
||||||
const switches = trace.steps.filter((step) => step.op === "switch_tab");
|
|
||||||
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
|
|
||||||
expect(switches).toHaveLength(1);
|
|
||||||
expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not retry an in-flight tab rearm after recording stops", async () => {
|
|
||||||
const chromeApi = installChrome();
|
|
||||||
const manager = fakeManager();
|
|
||||||
const tabsApi = makeMultiTabsApi([
|
|
||||||
{
|
|
||||||
id: TAB_ID,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: START_URL,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
{
|
|
||||||
id: 5,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: false,
|
|
||||||
status: "complete",
|
|
||||||
url: "https://example.com/second",
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
]);
|
|
||||||
let failRearm!: () => void;
|
|
||||||
const rearmGate = new Promise<never>((_resolve, reject) => {
|
|
||||||
failRearm = () => reject(new Error("rearm failed after stop"));
|
|
||||||
});
|
|
||||||
let announceRearm!: () => void;
|
|
||||||
const rearmStarted = new Promise<void>((resolve) => {
|
|
||||||
announceRearm = resolve;
|
|
||||||
});
|
|
||||||
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
|
|
||||||
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
|
|
||||||
announceRearm();
|
|
||||||
await rearmGate;
|
|
||||||
}
|
|
||||||
return { ok: true };
|
|
||||||
});
|
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await rearmStarted;
|
|
||||||
|
|
||||||
const stoppedPromise = handleRecordStop(manager, { session_id: "abcd" }, deps);
|
|
||||||
await vi.waitFor(() => {
|
|
||||||
expect(sendToTab).toHaveBeenCalledWith(5, expect.objectContaining({ type: RECORD_STOP }));
|
|
||||||
});
|
|
||||||
failRearm();
|
|
||||||
const stopped = await stoppedPromise;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
||||||
|
|
||||||
const startsForSecondTab = sendToTab.mock.calls.filter(
|
|
||||||
([tabId, message]) => tabId === 5 && (message as { type?: string }).type === RECORD_START,
|
|
||||||
);
|
|
||||||
expect(startsForSecondTab).toHaveLength(1);
|
|
||||||
expect(asTraceV3((stopped as RecordStopResult).trace).steps).toHaveLength(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps pending action settles scoped to their tab during an immediate switch", async () => {
|
|
||||||
const chromeApi = installChrome();
|
|
||||||
const manager = fakeManager();
|
|
||||||
const secondUrl = "https://example.com/second";
|
|
||||||
const tabsApi = makeMultiTabsApi([
|
|
||||||
{
|
|
||||||
id: TAB_ID,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: START_URL,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
{
|
|
||||||
id: 5,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: false,
|
|
||||||
status: "complete",
|
|
||||||
url: secondUrl,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
]);
|
|
||||||
let requestId = "";
|
|
||||||
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
|
|
||||||
const typed = message as { type?: string; requestId?: string };
|
|
||||||
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
|
|
||||||
return { ok: true };
|
|
||||||
});
|
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
|
||||||
attachRecordStepListener(deps);
|
|
||||||
runtimeOnMessageEmit(chromeApi, requestId, {
|
|
||||||
op: "click",
|
|
||||||
page_url: START_URL,
|
|
||||||
target: { role: "button", name: "First action", tag: "button" },
|
|
||||||
});
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
runtimeOnMessageEmit(
|
|
||||||
chromeApi,
|
|
||||||
requestId,
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
page_url: secondUrl,
|
|
||||||
target: { role: "button", name: "Second action", tag: "button" },
|
|
||||||
},
|
|
||||||
5,
|
|
||||||
);
|
|
||||||
await settleWait();
|
|
||||||
|
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
|
||||||
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
|
|
||||||
|
|
||||||
expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]);
|
|
||||||
expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL);
|
|
||||||
expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl);
|
|
||||||
expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("tracks the same navigation URL independently for each tab", async () => {
|
|
||||||
const chromeApi = installChrome();
|
|
||||||
const manager = fakeManager();
|
|
||||||
const tabsApi = makeMultiTabsApi([
|
|
||||||
{
|
|
||||||
id: TAB_ID,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: START_URL,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
{
|
|
||||||
id: 5,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: false,
|
|
||||||
status: "complete",
|
|
||||||
url: "https://example.com/second",
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
]);
|
|
||||||
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
|
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
tabsApi.goTo(5, START_URL, "Same URL in second tab");
|
|
||||||
chromeApi.webNavigationOnCommitted.emit({
|
chromeApi.webNavigationOnCommitted.emit({
|
||||||
tabId: 5,
|
tabId: 5,
|
||||||
frameId: 0,
|
frameId: 0,
|
||||||
|
url: secondUrl,
|
||||||
|
transitionType: "link",
|
||||||
|
transitionQualifiers: [],
|
||||||
|
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
||||||
|
const earlyStop = await Promise.race([
|
||||||
|
stopping.then((result) => ({ result })),
|
||||||
|
new Promise<null>((resolve) => setTimeout(() => resolve(null), 500)),
|
||||||
|
]);
|
||||||
|
releaseLookup();
|
||||||
|
const stopped = earlyStop?.result ?? (await stopping);
|
||||||
|
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
||||||
|
|
||||||
|
expect(trace.steps.some((step) => step.op === "navigate" && step.to === secondUrl)).toBe(true);
|
||||||
|
const navigation = trace.steps.find((step) => step.op === "navigate" && step.to === secondUrl);
|
||||||
|
expect(navigation?.result.state).toBeTruthy();
|
||||||
|
}, 15_000);
|
||||||
|
|
||||||
|
it("captures a new-tab navigation from the emitting tab", async () => {
|
||||||
|
const chromeApi = installChrome();
|
||||||
|
const manager = fakeManager();
|
||||||
|
const firstTab = {
|
||||||
|
id: TAB_ID,
|
||||||
|
windowId: AGENT_WINDOW_ID,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
url: START_URL,
|
url: START_URL,
|
||||||
transitionType: "typed",
|
title: "Old tab",
|
||||||
transitionQualifiers: ["from_address_bar"],
|
} as chrome.tabs.Tab;
|
||||||
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
const secondUrl = "https://example.com/new-tab";
|
||||||
await settleWait();
|
const secondTab = {
|
||||||
|
id: 5,
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
windowId: AGENT_WINDOW_ID,
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
active: false,
|
||||||
expect(
|
status: "complete",
|
||||||
trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL),
|
url: secondUrl,
|
||||||
).toHaveLength(1);
|
title: "New tab",
|
||||||
});
|
} as chrome.tabs.Tab;
|
||||||
|
const tabsApi = makeMultiTabsApi([firstTab, secondTab]);
|
||||||
it("drops late input and frame navigation from an inactive tab", async () => {
|
const cdp = makeFakeCdp(undefined, {
|
||||||
const chromeApi = installChrome();
|
treesByTab: {
|
||||||
const manager = fakeManager();
|
[TAB_ID]: axTree("Old tab", ["Old action"]),
|
||||||
const secondUrl = "https://example.com/second";
|
5: axTree("New tab", ["New action"]),
|
||||||
const tabsApi = makeMultiTabsApi([
|
},
|
||||||
{
|
|
||||||
id: TAB_ID,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: true,
|
|
||||||
status: "complete",
|
|
||||||
url: START_URL,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
{
|
|
||||||
id: 5,
|
|
||||||
windowId: AGENT_WINDOW_ID,
|
|
||||||
active: false,
|
|
||||||
status: "complete",
|
|
||||||
url: secondUrl,
|
|
||||||
} as chrome.tabs.Tab,
|
|
||||||
]);
|
|
||||||
let requestId = "";
|
|
||||||
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
|
|
||||||
const typed = message as { type?: string; requestId?: string };
|
|
||||||
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
|
|
||||||
return { ok: true };
|
|
||||||
});
|
});
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||||
|
const deps = { tabsApi, sendToTab, cdp };
|
||||||
|
|
||||||
await handleRecordStart(manager, RECORD_START_V3, deps);
|
await handleRecordStart(manager, RECORD_START_V3, deps);
|
||||||
attachRecordStepListener(deps);
|
|
||||||
tabsApi.activate(5);
|
|
||||||
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
|
|
||||||
await settleWait();
|
|
||||||
const lateAck = runtimeOnMessageEmit(
|
|
||||||
chromeApi,
|
|
||||||
requestId,
|
|
||||||
{
|
|
||||||
op: "fill",
|
|
||||||
page_url: START_URL,
|
|
||||||
target: { role: "textbox", name: "Draft", tag: "input" },
|
|
||||||
value: "late",
|
|
||||||
},
|
|
||||||
TAB_ID,
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
|
|
||||||
chromeApi.webNavigationOnCommitted.emit({
|
|
||||||
tabId: TAB_ID,
|
|
||||||
frameId: 0,
|
|
||||||
url: "https://example.com/late",
|
|
||||||
transitionType: "link",
|
|
||||||
transitionQualifiers: [],
|
|
||||||
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
|
||||||
chromeApi.webNavigationOnCommitted.emit({
|
chromeApi.webNavigationOnCommitted.emit({
|
||||||
tabId: 5,
|
tabId: 5,
|
||||||
frameId: 7,
|
frameId: 0,
|
||||||
url: "https://frame.example/late",
|
url: secondUrl,
|
||||||
transitionType: "link",
|
transitionType: "link",
|
||||||
transitionQualifiers: [],
|
transitionQualifiers: [],
|
||||||
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
await settleWait();
|
||||||
|
|
||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
const trace = asTraceV3((stopped as RecordStopResult).trace);
|
||||||
expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]);
|
const navigation = trace.steps.find((step) => step.op === "navigate" && step.to === secondUrl);
|
||||||
});
|
const landed = trace.states.find((state) => state.id === navigation?.result.state);
|
||||||
|
|
||||||
|
expect(landed?.url).toBe(secondUrl);
|
||||||
|
expect(landed?.body).toContain("New tab");
|
||||||
|
expect(landed?.body).not.toContain("Old action");
|
||||||
|
}, 15_000);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -66,52 +66,6 @@ function makeApis(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("handleSessionStop with auto-return", () => {
|
describe("handleSessionStop with auto-return", () => {
|
||||||
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
|
||||||
const aw = fakeAgentWindow([100]);
|
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
|
||||||
await sm.start("aa11");
|
|
||||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
|
||||||
const controller = new AbortController();
|
|
||||||
controller.abort();
|
|
||||||
|
|
||||||
const res = await handleSessionStop(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11" },
|
|
||||||
{ cdp, signal: controller.signal },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(res).toMatchObject({ code: "cancelled" });
|
|
||||||
expect(sm.has("aa11")).toBe(true);
|
|
||||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
|
||||||
expect(aw.remove).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
|
||||||
const aw = fakeAgentWindow([100]);
|
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
|
||||||
await sm.start("aa11");
|
|
||||||
const controller = new AbortController();
|
|
||||||
let finishDetach: () => void = () => {};
|
|
||||||
const detach = new Promise<void>((resolve) => {
|
|
||||||
finishDetach = resolve;
|
|
||||||
});
|
|
||||||
const cdp = { detachSession: vi.fn(() => detach) };
|
|
||||||
|
|
||||||
const stopping = handleSessionStop(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11" },
|
|
||||||
{ cdp, signal: controller.signal },
|
|
||||||
);
|
|
||||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
|
||||||
controller.abort();
|
|
||||||
finishDetach();
|
|
||||||
|
|
||||||
const res = await stopping;
|
|
||||||
expect(res).toMatchObject({ code: "cancelled" });
|
|
||||||
expect(sm.has("aa11")).toBe(true);
|
|
||||||
expect(aw.remove).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||||
const aw = fakeAgentWindow([100]);
|
const aw = fakeAgentWindow([100]);
|
||||||
const sm = new SessionManager({ agentWindow: aw });
|
const sm = new SessionManager({ agentWindow: aw });
|
||||||
|
|||||||
@@ -46,25 +46,6 @@ describe("lookupSnapshotRef", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("resolveSnapshotRef", () => {
|
describe("resolveSnapshotRef", () => {
|
||||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
const ctx = await sm.start("aa11");
|
|
||||||
ctx.refStore.set("e3", 1234, {
|
|
||||||
tabId: 4,
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
});
|
|
||||||
|
|
||||||
const expected = {
|
|
||||||
backendNodeId: 1234,
|
|
||||||
refKey: "e3",
|
|
||||||
frameId: "child-frame",
|
|
||||||
cdpSessionId: "child-session",
|
|
||||||
};
|
|
||||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
|
||||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -250,10 +250,9 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
|||||||
// URL injection-eligibility test
|
// URL injection-eligibility test
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const EXTENSION_STORE_RES = [
|
const CHROME_WEB_STORE_RES = [
|
||||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||||
/^https:\/\/chromewebstore\.google\.com/i,
|
/^https:\/\/chromewebstore\.google\.com/i,
|
||||||
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -265,15 +264,14 @@ const EXTENSION_STORE_RES = [
|
|||||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||||
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
||||||
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
|
||||||
*
|
*
|
||||||
* Mirrors the implicit scheme list documented in
|
* Mirrors the implicit scheme list documented in
|
||||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||||
*/
|
*/
|
||||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||||
if (!url) return false;
|
if (!url) return false;
|
||||||
for (const re of EXTENSION_STORE_RES) {
|
for (const re of CHROME_WEB_STORE_RES) {
|
||||||
if (re.test(url)) return false;
|
if (re.test(url)) return false;
|
||||||
}
|
}
|
||||||
return /^https?:\/\//i.test(url);
|
return /^https?:\/\//i.test(url);
|
||||||
|
|||||||
@@ -1,9 +1,101 @@
|
|||||||
export {
|
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||||
type CaptureVomObservationOptions,
|
import type { CdpAxNode } from "./observation";
|
||||||
captureVomObservation,
|
import { buildVomScene } from "./observation";
|
||||||
} from "./observation";
|
import type { CdpRunner } from "./shared";
|
||||||
export {
|
import {
|
||||||
type CaptureVomFrame,
|
type CapturedNode,
|
||||||
type CaptureVomMatchNode,
|
type CapturedSurfaceProbe,
|
||||||
type CaptureVomObservationResult,
|
type CapturedViewModel,
|
||||||
} from "./vom/record-safe-observation";
|
captureViewModel,
|
||||||
|
collectOverlayExcludedBackendIds,
|
||||||
|
} from "./vom/capture";
|
||||||
|
|
||||||
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationResult {
|
||||||
|
text: string;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
truncated: boolean;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||||
|
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LayoutMetricsViewportReply {
|
||||||
|
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fallbackCapturedViewModel(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
let viewport = { width: 0, height: 0 };
|
||||||
|
try {
|
||||||
|
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
viewport = {
|
||||||
|
width: vpSrc.clientWidth ?? 0,
|
||||||
|
height: vpSrc.clientHeight ?? 0,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// viewport stays zero-sized
|
||||||
|
}
|
||||||
|
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||||
|
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureForVom(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
conditionalSurfaceProbe: boolean,
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
try {
|
||||||
|
return await captureViewModel(cdp, tabId, {
|
||||||
|
conditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return fallbackCapturedViewModel(cdp, tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
|
||||||
|
export async function captureVomObservation(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
url: string | undefined,
|
||||||
|
opts: CaptureVomObservationOptions = {},
|
||||||
|
): Promise<CaptureVomObservationResult> {
|
||||||
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
|
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
|
||||||
|
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
|
||||||
|
const axNodes = result.nodes ?? [];
|
||||||
|
const captured = await captureForVom(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
opts.conditionalSurfaceProbe ?? false,
|
||||||
|
opts.hoverProbeBypassOverlay,
|
||||||
|
);
|
||||||
|
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
|
||||||
|
const rendered = renderVom(scene, {
|
||||||
|
maxDepth: opts.maxDepth,
|
||||||
|
maxTokens: opts.maxTokens,
|
||||||
|
redactValues: opts.redactValues,
|
||||||
|
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
text: rendered.text,
|
||||||
|
refs: rendered.refs,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
captured: captured.nodes,
|
||||||
|
surfaceProbes: captured.surfaceProbes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,12 +51,7 @@ import {
|
|||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
handleSnapshot,
|
handleSnapshot,
|
||||||
} from "./observation";
|
} from "./observation";
|
||||||
import {
|
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||||
handleRecordAwait,
|
|
||||||
handleRecordStart,
|
|
||||||
handleRecordStop,
|
|
||||||
type RecordRuntimeDeps,
|
|
||||||
} from "./record";
|
|
||||||
import {
|
import {
|
||||||
handleSessionStart,
|
handleSessionStart,
|
||||||
handleSessionStop,
|
handleSessionStop,
|
||||||
@@ -104,7 +99,6 @@ export interface DispatcherDeps {
|
|||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
cdp?: DispatcherCdpRunner;
|
cdp?: DispatcherCdpRunner;
|
||||||
recording?: RecordRuntimeDeps;
|
|
||||||
/**
|
/**
|
||||||
* Invoked whenever a dispatched RPC may have changed the live
|
* Invoked whenever a dispatched RPC may have changed the live
|
||||||
* session set (currently `tool.session_start` and
|
* session set (currently `tool.session_start` and
|
||||||
@@ -140,7 +134,6 @@ export class ToolDispatcher {
|
|||||||
private readonly transport: Transport;
|
private readonly transport: Transport;
|
||||||
private readonly sessions: SessionManager;
|
private readonly sessions: SessionManager;
|
||||||
private readonly cdp?: DispatcherCdpRunner;
|
private readonly cdp?: DispatcherCdpRunner;
|
||||||
private readonly recording?: RecordRuntimeDeps;
|
|
||||||
private readonly onSessionsChanged?: () => void;
|
private readonly onSessionsChanged?: () => void;
|
||||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
@@ -160,7 +153,6 @@ export class ToolDispatcher {
|
|||||||
this.transport = deps.transport;
|
this.transport = deps.transport;
|
||||||
this.sessions = deps.sessions;
|
this.sessions = deps.sessions;
|
||||||
this.cdp = deps.cdp;
|
this.cdp = deps.cdp;
|
||||||
this.recording = deps.recording;
|
|
||||||
this.onSessionsChanged = deps.onSessionsChanged;
|
this.onSessionsChanged = deps.onSessionsChanged;
|
||||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||||
this.approveBorrow = deps.approveBorrow;
|
this.approveBorrow = deps.approveBorrow;
|
||||||
@@ -298,7 +290,6 @@ export class ToolDispatcher {
|
|||||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
signal,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
case "tool.tab_list":
|
case "tool.tab_list":
|
||||||
@@ -383,11 +374,7 @@ export class ToolDispatcher {
|
|||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsCaptureApi,
|
tabsApi: chromeTabsCaptureApi,
|
||||||
// Active hover probing is opt-in. A held hover latch still
|
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||||
// suppresses it, because probing would move the cursor off
|
|
||||||
// the element the caller is deliberately holding.
|
|
||||||
conditionalSurfaceProbe:
|
|
||||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
|
||||||
hoverProbeBypassOverlay: bypassOverlay,
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
@@ -543,26 +530,44 @@ export class ToolDispatcher {
|
|||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
case "tool.record_start":
|
case "tool.record_start":
|
||||||
return this.recording
|
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||||
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
tabsApi: chromeTabsApi,
|
||||||
...this.recording,
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
signal,
|
bypassOverlay: async (tabId, enabled) => {
|
||||||
})
|
try {
|
||||||
: recordingRuntimeUnavailable();
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||||
|
signal,
|
||||||
|
});
|
||||||
case "tool.record_stop":
|
case "tool.record_stop":
|
||||||
return this.recording
|
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||||
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
tabsApi: chromeTabsApi,
|
||||||
...this.recording,
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
signal,
|
bypassOverlay: async (tabId, enabled) => {
|
||||||
})
|
try {
|
||||||
: recordingRuntimeUnavailable();
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
});
|
||||||
case "tool.record_await":
|
case "tool.record_await":
|
||||||
return this.recording
|
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||||
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
tabsApi: chromeTabsApi,
|
||||||
...this.recording,
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
signal,
|
signal,
|
||||||
})
|
});
|
||||||
: recordingRuntimeUnavailable();
|
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
code: "unknown_method",
|
code: "unknown_method",
|
||||||
@@ -698,13 +703,6 @@ function isRpcError(v: unknown): v is RpcError {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function recordingRuntimeUnavailable(): RpcError {
|
|
||||||
return {
|
|
||||||
code: "protocol_error",
|
|
||||||
message: "recording runtime is unavailable",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
|
|||||||
@@ -1,15 +1,9 @@
|
|||||||
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
// Shared CDP helpers for scrolling nodes into view and computing
|
||||||
// belongs to frame-geometry.ts.
|
// viewport-space bounding rectangles. Used by interaction tools (click
|
||||||
|
// centre) and observation tools (element screenshot clip).
|
||||||
|
|
||||||
import type { RpcError } from "@/transport/types";
|
import type { RpcError } from "@/transport/types";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import {
|
|
||||||
parseCdpQuad,
|
|
||||||
polygonBounds,
|
|
||||||
type Region,
|
|
||||||
rectPolygon,
|
|
||||||
type ViewportRect,
|
|
||||||
} from "./geometry";
|
|
||||||
import { type CdpRunner, isRpcError } from "./shared";
|
import { type CdpRunner, isRpcError } from "./shared";
|
||||||
|
|
||||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||||
@@ -19,6 +13,45 @@ function elementNotVisibleError(): RpcError {
|
|||||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
||||||
|
export interface ViewportRect {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the axis-aligned bounding box of an 8-double CDP content
|
||||||
|
* quad / box-model polygon. Exported for unit tests.
|
||||||
|
*/
|
||||||
|
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
||||||
|
if (quad.length !== 8) return null;
|
||||||
|
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
||||||
|
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
||||||
|
const minX = Math.min(...xs);
|
||||||
|
const maxX = Math.max(...xs);
|
||||||
|
const minY = Math.min(...ys);
|
||||||
|
const maxY = Math.max(...ys);
|
||||||
|
const width = maxX - minX;
|
||||||
|
const height = maxY - minY;
|
||||||
|
if (width <= 0 || height <= 0) return null;
|
||||||
|
return { x: minX, y: minY, width, height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectToQuad(rect: ViewportRect): number[] {
|
||||||
|
return [
|
||||||
|
rect.x,
|
||||||
|
rect.y,
|
||||||
|
rect.x + rect.width,
|
||||||
|
rect.y,
|
||||||
|
rect.x + rect.width,
|
||||||
|
rect.y + rect.height,
|
||||||
|
rect.x,
|
||||||
|
rect.y + rect.height,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||||
* `Runtime.callFunctionOn` against the live JS object.
|
* `Runtime.callFunctionOn` against the live JS object.
|
||||||
@@ -79,24 +112,48 @@ export async function scrollNodeIntoView(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function validCdpRegion(quads: number[][] | undefined): Region {
|
/**
|
||||||
return (quads ?? []).flatMap((raw) => {
|
* Viewport-space bounding box for `backendNodeId`. Prefers
|
||||||
const polygon = parseCdpQuad(raw);
|
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
||||||
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
* visible descendant client rects for zero-size overflow containers.
|
||||||
});
|
*/
|
||||||
}
|
export async function nodeBoundingRect(
|
||||||
|
|
||||||
async function resolveVisibleContentRegion(
|
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
): Promise<Region | RpcError> {
|
): Promise<ViewportRect | RpcError> {
|
||||||
|
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||||
|
if (isRpcError(polygon)) return polygon;
|
||||||
|
// visibleContentPolygon only returns quads with positive area.
|
||||||
|
return quadBoundingRect(polygon)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute the centroid of a CDP content quad. Quads are reported as
|
||||||
|
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
||||||
|
*/
|
||||||
|
export function quadCentre(quad: number[]): { x: number; y: number } {
|
||||||
|
const x = (quad[0] + quad[4]) / 2;
|
||||||
|
const y = (quad[1] + quad[5]) / 2;
|
||||||
|
return { x, y };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
||||||
|
export function boxCentre(box: number[]): { x: number; y: number } {
|
||||||
|
return quadCentre(box);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function visibleContentPolygon(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<number[] | RpcError> {
|
||||||
try {
|
try {
|
||||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const visible = validCdpRegion(quads.quads);
|
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||||
if (visible.length > 0) return visible;
|
if (quad) return quad;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||||
}
|
}
|
||||||
@@ -105,34 +162,15 @@ async function resolveVisibleContentRegion(
|
|||||||
backendNodeId,
|
backendNodeId,
|
||||||
});
|
});
|
||||||
const content = box.model?.content;
|
const content = box.model?.content;
|
||||||
const fallback = validCdpRegion(content ? [content] : undefined);
|
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||||
if (fallback.length > 0) return fallback;
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||||
}
|
}
|
||||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||||
if (descendantRect) {
|
if (descendantRect) return rectToQuad(descendantRect);
|
||||||
return [
|
|
||||||
rectPolygon({
|
|
||||||
x: descendantRect.x,
|
|
||||||
y: descendantRect.y,
|
|
||||||
w: descendantRect.width,
|
|
||||||
h: descendantRect.height,
|
|
||||||
}),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
return elementNotVisibleError();
|
return elementNotVisibleError();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Resolve every visible content region in the node's target-local viewport. */
|
|
||||||
export async function nodeContentRegion(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
backendNodeId: number,
|
|
||||||
): Promise<Region | RpcError> {
|
|
||||||
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function descendantBoundingRect(
|
async function descendantBoundingRect(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -203,3 +241,20 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
|||||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||||
return { x, y, width, height };
|
return { x, y, width, height };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the click point for `backendNodeId`. Prefers
|
||||||
|
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
||||||
|
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
||||||
|
* Returns `permission_denied` with "element not visible" when every
|
||||||
|
* geometry path comes back empty / fails.
|
||||||
|
*/
|
||||||
|
export async function nodeCentre(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
): Promise<{ x: number; y: number } | RpcError> {
|
||||||
|
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||||
|
if (isRpcError(polygon)) return polygon;
|
||||||
|
return quadCentre(polygon);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,290 +0,0 @@
|
|||||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import type { RpcError } from "@/transport/types";
|
|
||||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
|
||||||
import {
|
|
||||||
clipPolygon,
|
|
||||||
type GeometryProjection,
|
|
||||||
type Point,
|
|
||||||
type Polygon,
|
|
||||||
type ProjectiveEdge,
|
|
||||||
parseCdpQuad,
|
|
||||||
polygonArea,
|
|
||||||
polygonCentroid,
|
|
||||||
projectRegionToViewport,
|
|
||||||
type Quad,
|
|
||||||
type Region,
|
|
||||||
rectPolygon,
|
|
||||||
regionBounds,
|
|
||||||
type Size,
|
|
||||||
type ViewportRect,
|
|
||||||
} from "./geometry";
|
|
||||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
|
||||||
|
|
||||||
export interface NodeAddress {
|
|
||||||
target: CdpTarget;
|
|
||||||
backendNodeId: number;
|
|
||||||
frameId?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ResolvedNodeGeometry {
|
|
||||||
topVisibleRegions: Region;
|
|
||||||
topBounds: ViewportRect;
|
|
||||||
actionPoint: Point;
|
|
||||||
}
|
|
||||||
|
|
||||||
function geometryError(message: string): RpcError {
|
|
||||||
return { code: "cdp_failed", message };
|
|
||||||
}
|
|
||||||
|
|
||||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
|
||||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
|
||||||
}
|
|
||||||
|
|
||||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
|
||||||
const seen = new Set<string>();
|
|
||||||
let current = frame;
|
|
||||||
while (current.parentFrameId) {
|
|
||||||
if (seen.has(current.frameId)) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return current;
|
|
||||||
}
|
|
||||||
|
|
||||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
|
||||||
const byId = frameMap(graph);
|
|
||||||
const path: CdpFrame[] = [];
|
|
||||||
const seen = new Set<string>();
|
|
||||||
let current = byId.get(frameId);
|
|
||||||
if (!current) return null;
|
|
||||||
while (current.parentFrameId) {
|
|
||||||
if (seen.has(current.frameId)) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
path.push(current);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent) return null;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
|
||||||
const metrics = await sendToCdpTarget<{
|
|
||||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
|
||||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
|
||||||
const width = viewport.clientWidth ?? 0;
|
|
||||||
const height = viewport.clientHeight ?? 0;
|
|
||||||
return width > 0 && height > 0 ? { width, height } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function ownerContentQuad(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
parent: CdpFrame,
|
|
||||||
ownerBackendNodeId: number,
|
|
||||||
): Promise<Quad | null> {
|
|
||||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
|
||||||
cdp,
|
|
||||||
parent.target,
|
|
||||||
"DOM.getBoxModel",
|
|
||||||
{ backendNodeId: ownerBackendNodeId },
|
|
||||||
);
|
|
||||||
return parseCdpQuad(result.model?.content);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function sameTargetFrameClips(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
byId: Map<string, CdpFrame>,
|
|
||||||
frame: CdpFrame,
|
|
||||||
targetRoot: CdpFrame,
|
|
||||||
): Promise<Polygon[] | null> {
|
|
||||||
const clips: Polygon[] = [];
|
|
||||||
let current = frame;
|
|
||||||
const seen = new Set<string>();
|
|
||||||
while (current.frameId !== targetRoot.frameId) {
|
|
||||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
|
||||||
seen.add(current.frameId);
|
|
||||||
const parent = byId.get(current.parentFrameId);
|
|
||||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
|
||||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
|
||||||
if (!clip) return null;
|
|
||||||
clips.push(clip);
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
return clips;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function resolveFrameProjection(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
graph: CdpFrameGraph,
|
|
||||||
frameId: string,
|
|
||||||
): Promise<GeometryProjection | null> {
|
|
||||||
const byId = frameMap(graph);
|
|
||||||
const frame = byId.get(frameId);
|
|
||||||
if (!frame) return null;
|
|
||||||
let targetRoot = targetRootFrame(byId, frame);
|
|
||||||
if (!targetRoot) return null;
|
|
||||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
|
||||||
if (!sourceViewport) return null;
|
|
||||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
|
||||||
if (!sourceClips) return null;
|
|
||||||
|
|
||||||
const edges: ProjectiveEdge[] = [];
|
|
||||||
while (targetRoot.parentFrameId) {
|
|
||||||
const parent = byId.get(targetRoot.parentFrameId);
|
|
||||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
|
||||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
|
||||||
if (!destinationQuad) return null;
|
|
||||||
const source =
|
|
||||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
|
||||||
if (!source) return null;
|
|
||||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
|
||||||
if (!parentTargetRoot) return null;
|
|
||||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
|
||||||
if (!destinationClips) return null;
|
|
||||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
|
||||||
targetRoot = parentTargetRoot;
|
|
||||||
}
|
|
||||||
|
|
||||||
const topViewport =
|
|
||||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
|
||||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
|
||||||
if (!cdp.getFrameGraph) return null;
|
|
||||||
try {
|
|
||||||
return await cdp.getFrameGraph(tabId);
|
|
||||||
} catch (error) {
|
|
||||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function scrollFrameOwners(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
graph: CdpFrameGraph,
|
|
||||||
frameId: string,
|
|
||||||
): Promise<RpcError | null> {
|
|
||||||
const byId = frameMap(graph);
|
|
||||||
const ancestry = frameAncestry(graph, frameId);
|
|
||||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
|
||||||
for (const child of [...ancestry].reverse()) {
|
|
||||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
|
||||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
|
||||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
|
||||||
}
|
|
||||||
const error = await scrollNodeIntoView(
|
|
||||||
cdpRunnerForTarget(cdp, parent.target),
|
|
||||||
tabId,
|
|
||||||
child.ownerBackendNodeId,
|
|
||||||
);
|
|
||||||
if (error) return error;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function scrollElementWithFrameGraph(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
target: CdpTarget,
|
|
||||||
backendNodeId: number,
|
|
||||||
frameId: string | undefined,
|
|
||||||
graph: CdpFrameGraph | null,
|
|
||||||
): Promise<RpcError | null> {
|
|
||||||
if (frameId) {
|
|
||||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
|
||||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
|
||||||
if (error) return error;
|
|
||||||
} else if (target.sessionId) {
|
|
||||||
return geometryError("an OOPIF node address requires frameId");
|
|
||||||
}
|
|
||||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function scrollElementAndFramesIntoView(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
target: CdpTarget,
|
|
||||||
backendNodeId: number,
|
|
||||||
frameId?: string,
|
|
||||||
): Promise<RpcError | null> {
|
|
||||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
|
||||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
|
||||||
}
|
|
||||||
|
|
||||||
function largestRegion(regions: Region): Polygon | null {
|
|
||||||
let largest: { polygon: Polygon; area: number } | null = null;
|
|
||||||
for (const polygon of regions) {
|
|
||||||
const area = polygonArea(polygon);
|
|
||||||
if (area <= 0) continue;
|
|
||||||
if (!largest || area > largest.area) largest = { polygon, area };
|
|
||||||
}
|
|
||||||
return largest?.polygon ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function resolveNodeGeometry(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
address: NodeAddress,
|
|
||||||
options: { scrollIntoView?: boolean } = {},
|
|
||||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
|
||||||
try {
|
|
||||||
if (address.target.sessionId && !address.frameId) {
|
|
||||||
return geometryError("an OOPIF node address requires frameId");
|
|
||||||
}
|
|
||||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
|
||||||
if (address.frameId && !graph) {
|
|
||||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.scrollIntoView) {
|
|
||||||
const scrollError = await scrollElementWithFrameGraph(
|
|
||||||
cdp,
|
|
||||||
tabId,
|
|
||||||
address.target,
|
|
||||||
address.backendNodeId,
|
|
||||||
address.frameId,
|
|
||||||
graph,
|
|
||||||
);
|
|
||||||
if (scrollError) return scrollError;
|
|
||||||
}
|
|
||||||
|
|
||||||
const localRegion = await nodeContentRegion(
|
|
||||||
cdpRunnerForTarget(cdp, address.target),
|
|
||||||
tabId,
|
|
||||||
address.backendNodeId,
|
|
||||||
);
|
|
||||||
if (isRpcError(localRegion)) return localRegion;
|
|
||||||
|
|
||||||
let topVisibleRegions: Region;
|
|
||||||
if (address.frameId && graph) {
|
|
||||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
|
||||||
if (!projection)
|
|
||||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
|
||||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
|
||||||
} else {
|
|
||||||
const viewport = await targetViewport(cdp, address.target);
|
|
||||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
|
||||||
topVisibleRegions = localRegion
|
|
||||||
.map((polygon) =>
|
|
||||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
|
||||||
)
|
|
||||||
.filter((polygon) => polygon.length >= 3);
|
|
||||||
}
|
|
||||||
|
|
||||||
const topBounds = regionBounds(topVisibleRegions);
|
|
||||||
const actionRegion = largestRegion(topVisibleRegions);
|
|
||||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
|
||||||
if (!topBounds || !actionPoint) {
|
|
||||||
return { code: "permission_denied", message: "element not visible" };
|
|
||||||
}
|
|
||||||
return { topVisibleRegions, topBounds, actionPoint };
|
|
||||||
} catch (error) {
|
|
||||||
return geometryError(error instanceof Error ? error.message : String(error));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,266 +0,0 @@
|
|||||||
export interface Point {
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type Polygon = Point[];
|
|
||||||
export type Region = Polygon[];
|
|
||||||
export type Quad = [Point, Point, Point, Point];
|
|
||||||
|
|
||||||
export interface Size {
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ViewportRect {
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ProjectiveEdge {
|
|
||||||
sourceViewport: Size;
|
|
||||||
destinationQuad: Quad;
|
|
||||||
destinationClips?: Polygon[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface GeometryProjection {
|
|
||||||
sourceClips: Polygon[];
|
|
||||||
edges: ProjectiveEdge[];
|
|
||||||
topViewport: Size;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
|
||||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
|
||||||
return [
|
|
||||||
{ x: raw[0], y: raw[1] },
|
|
||||||
{ x: raw[2], y: raw[3] },
|
|
||||||
{ x: raw[4], y: raw[5] },
|
|
||||||
{ x: raw[6], y: raw[7] },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
|
||||||
return [
|
|
||||||
{ x: rect.x, y: rect.y },
|
|
||||||
{ x: rect.x + rect.w, y: rect.y },
|
|
||||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
|
||||||
{ x: rect.x, y: rect.y + rect.h },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
export function viewportPolygon(viewport: Size): Quad {
|
|
||||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
|
||||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
|
||||||
const [p0, p1, p2, p3] = quad;
|
|
||||||
const dx1 = p1.x - p2.x;
|
|
||||||
const dx2 = p3.x - p2.x;
|
|
||||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
|
||||||
const dy1 = p1.y - p2.y;
|
|
||||||
const dy2 = p3.y - p2.y;
|
|
||||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
|
||||||
|
|
||||||
let a: number;
|
|
||||||
let b: number;
|
|
||||||
let c: number;
|
|
||||||
let d: number;
|
|
||||||
let e: number;
|
|
||||||
let f: number;
|
|
||||||
let g = 0;
|
|
||||||
let h = 0;
|
|
||||||
|
|
||||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
|
||||||
a = p1.x - p0.x;
|
|
||||||
b = p3.x - p0.x;
|
|
||||||
c = p0.x;
|
|
||||||
d = p1.y - p0.y;
|
|
||||||
e = p3.y - p0.y;
|
|
||||||
f = p0.y;
|
|
||||||
} else {
|
|
||||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
|
||||||
if (Math.abs(denominator) < 1e-9) {
|
|
||||||
return {
|
|
||||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
|
||||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
|
||||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
|
||||||
a = p1.x - p0.x + g * p1.x;
|
|
||||||
b = p3.x - p0.x + h * p3.x;
|
|
||||||
c = p0.x;
|
|
||||||
d = p1.y - p0.y + g * p1.y;
|
|
||||||
e = p3.y - p0.y + h * p3.y;
|
|
||||||
f = p0.y;
|
|
||||||
}
|
|
||||||
|
|
||||||
const scale = g * point.x + h * point.y + 1;
|
|
||||||
return {
|
|
||||||
x: (a * point.x + b * point.y + c) / scale,
|
|
||||||
y: (d * point.x + e * point.y + f) / scale,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
|
||||||
const { width, height } = edge.sourceViewport;
|
|
||||||
if (width <= 0 || height <= 0) return [];
|
|
||||||
return polygon.map((point) =>
|
|
||||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function polygonSignedArea(points: Polygon): number {
|
|
||||||
let area = 0;
|
|
||||||
for (let index = 0; index < points.length; index += 1) {
|
|
||||||
const current = points[index];
|
|
||||||
const next = points[(index + 1) % points.length];
|
|
||||||
area += current.x * next.y - next.x * current.y;
|
|
||||||
}
|
|
||||||
return area / 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
|
||||||
const subjectX = end.x - start.x;
|
|
||||||
const subjectY = end.y - start.y;
|
|
||||||
const clipX = clipEnd.x - clipStart.x;
|
|
||||||
const clipY = clipEnd.y - clipStart.y;
|
|
||||||
const denominator = subjectX * clipY - subjectY * clipX;
|
|
||||||
if (Math.abs(denominator) < 1e-9) return end;
|
|
||||||
const offsetX = clipStart.x - start.x;
|
|
||||||
const offsetY = clipStart.y - start.y;
|
|
||||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
|
||||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
|
||||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
|
||||||
if (subject.length < 3 || clip.length < 3) return [];
|
|
||||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
|
||||||
let output = [...subject];
|
|
||||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
|
||||||
const clipStart = clip[edge];
|
|
||||||
const clipEnd = clip[(edge + 1) % clip.length];
|
|
||||||
const input = output;
|
|
||||||
output = [];
|
|
||||||
const inside = (point: Point) =>
|
|
||||||
orientation *
|
|
||||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
|
||||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
|
||||||
-1e-9;
|
|
||||||
let start = input[input.length - 1];
|
|
||||||
for (const end of input) {
|
|
||||||
const startInside = inside(start);
|
|
||||||
const endInside = inside(end);
|
|
||||||
if (endInside) {
|
|
||||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
|
||||||
output.push(end);
|
|
||||||
} else if (startInside) {
|
|
||||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
|
||||||
}
|
|
||||||
start = end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
|
||||||
if (points.length < 3) return null;
|
|
||||||
const xs = points.map((point) => point.x);
|
|
||||||
const ys = points.map((point) => point.y);
|
|
||||||
const x = Math.min(...xs);
|
|
||||||
const y = Math.min(...ys);
|
|
||||||
const right = Math.max(...xs);
|
|
||||||
const bottom = Math.max(...ys);
|
|
||||||
if (right <= x || bottom <= y) return null;
|
|
||||||
return { x, y, width: right - x, height: bottom - y };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function regionBounds(region: Region): ViewportRect | null {
|
|
||||||
return polygonBounds(region.flat());
|
|
||||||
}
|
|
||||||
|
|
||||||
export function polygonArea(points: Polygon): number {
|
|
||||||
return Math.abs(polygonSignedArea(points));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function polygonCentroid(points: Polygon): Point | null {
|
|
||||||
if (points.length < 3) return null;
|
|
||||||
let twiceArea = 0;
|
|
||||||
let x = 0;
|
|
||||||
let y = 0;
|
|
||||||
for (let index = 0; index < points.length; index += 1) {
|
|
||||||
const current = points[index];
|
|
||||||
const next = points[(index + 1) % points.length];
|
|
||||||
const cross = current.x * next.y - next.x * current.y;
|
|
||||||
twiceArea += cross;
|
|
||||||
x += (current.x + next.x) * cross;
|
|
||||||
y += (current.y + next.y) * cross;
|
|
||||||
}
|
|
||||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
|
||||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectAndClipRegion(
|
|
||||||
region: Region,
|
|
||||||
edges: ProjectiveEdge[],
|
|
||||||
topViewport: Size,
|
|
||||||
): Region {
|
|
||||||
let projected = region;
|
|
||||||
for (const edge of edges) {
|
|
||||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
|
||||||
projected = projected
|
|
||||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
|
||||||
.filter((polygon) => polygon.length >= 3)
|
|
||||||
.map((polygon) =>
|
|
||||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
|
||||||
(current, clip) => clipPolygon(current, clip),
|
|
||||||
projectPolygon(polygon, edge),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.filter((polygon) => polygon.length >= 3);
|
|
||||||
}
|
|
||||||
const topClip = viewportPolygon(topViewport);
|
|
||||||
return projected
|
|
||||||
.map((polygon) => clipPolygon(polygon, topClip))
|
|
||||||
.filter((polygon) => polygon.length >= 3);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
|
||||||
const clipped = region
|
|
||||||
.map((polygon) =>
|
|
||||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
|
||||||
)
|
|
||||||
.filter((polygon) => polygon.length >= 3);
|
|
||||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function projectRectToViewport(
|
|
||||||
rect: { x: number; y: number; w: number; h: number } | null,
|
|
||||||
projection: GeometryProjection,
|
|
||||||
): ViewportRect | null {
|
|
||||||
if (!rect) return null;
|
|
||||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function childFrameProjection(
|
|
||||||
parent: GeometryProjection,
|
|
||||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
|
||||||
): GeometryProjection {
|
|
||||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
|
||||||
return {
|
|
||||||
sourceClips: [],
|
|
||||||
edges: [
|
|
||||||
{
|
|
||||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
|
||||||
destinationQuad,
|
|
||||||
destinationClips: parent.sourceClips,
|
|
||||||
},
|
|
||||||
...parent.edges,
|
|
||||||
],
|
|
||||||
topViewport: parent.topViewport,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import {
|
|||||||
HELP_REQUEST,
|
HELP_REQUEST,
|
||||||
type HelpCancelMessage,
|
type HelpCancelMessage,
|
||||||
type HelpFinishMessage,
|
type HelpFinishMessage,
|
||||||
type HelpHighlightRect,
|
|
||||||
type HelpQueryResponse,
|
type HelpQueryResponse,
|
||||||
type HelpRequestMessage,
|
type HelpRequestMessage,
|
||||||
isHelpAckMessage,
|
isHelpAckMessage,
|
||||||
@@ -26,7 +25,6 @@ import type {
|
|||||||
ResolvedTarget,
|
ResolvedTarget,
|
||||||
RpcError,
|
RpcError,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { resolveNodeGeometry } from "./frame-geometry";
|
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
@@ -39,6 +37,7 @@ import {
|
|||||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||||
|
|
||||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||||
|
const HELP_ATTR = "data-bsk-help";
|
||||||
const HELP_SEND_RETRIES = 3;
|
const HELP_SEND_RETRIES = 3;
|
||||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||||
@@ -75,7 +74,6 @@ interface ActiveHelpRequest {
|
|||||||
title?: string;
|
title?: string;
|
||||||
targets: HelpTarget[];
|
targets: HelpTarget[];
|
||||||
selectors: string[];
|
selectors: string[];
|
||||||
rects: HelpHighlightRect[];
|
|
||||||
timeoutMs: number;
|
timeoutMs: number;
|
||||||
notificationId: string;
|
notificationId: string;
|
||||||
resolvedTargets?: ResolvedTarget[];
|
resolvedTargets?: ResolvedTarget[];
|
||||||
@@ -136,7 +134,6 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
|||||||
...(help.title ? { title: help.title } : {}),
|
...(help.title ? { title: help.title } : {}),
|
||||||
displayMode: isPrimaryTab ? "full" : "compact",
|
displayMode: isPrimaryTab ? "full" : "compact",
|
||||||
selectors: isPrimaryTab ? help.selectors : [],
|
selectors: isPrimaryTab ? help.selectors : [],
|
||||||
rects: isPrimaryTab ? help.rects : [],
|
|
||||||
timeoutMs: help.timeoutMs,
|
timeoutMs: help.timeoutMs,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -168,6 +165,33 @@ async function sendHelpRequestWithAck(
|
|||||||
throw lastError ?? new Error("failed to show help overlay");
|
throw lastError ?? new Error("failed to show help overlay");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function tagRefTarget(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
backendNodeId: number,
|
||||||
|
index: number,
|
||||||
|
): Promise<string | null> {
|
||||||
|
let objectId: string | undefined;
|
||||||
|
try {
|
||||||
|
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||||
|
backendNodeId,
|
||||||
|
});
|
||||||
|
objectId = resolved.object?.objectId;
|
||||||
|
if (!objectId) return null;
|
||||||
|
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||||
|
});
|
||||||
|
return `[${HELP_ATTR}="${index}"]`;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (objectId) {
|
||||||
|
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function selectorExists(
|
async function selectorExists(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
@@ -185,22 +209,36 @@ async function selectorExists(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||||
|
if (!cdp) return;
|
||||||
|
try {
|
||||||
|
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||||
|
depth: 0,
|
||||||
|
});
|
||||||
|
const rootId = doc.root?.nodeId;
|
||||||
|
if (rootId === undefined) return;
|
||||||
|
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||||
|
nodeId: rootId,
|
||||||
|
selector: `[${HELP_ATTR}]`,
|
||||||
|
});
|
||||||
|
for (const nodeId of nodeIds ?? []) {
|
||||||
|
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function resolveHelpTargets(
|
async function resolveHelpTargets(
|
||||||
ctx: ReturnType<SessionManager["get"]>,
|
ctx: ReturnType<SessionManager["get"]>,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
targets: HelpTarget[],
|
targets: HelpTarget[],
|
||||||
deps: RequestHelpDeps,
|
deps: RequestHelpDeps,
|
||||||
options: { scrollIntoView: boolean },
|
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||||
): Promise<{
|
|
||||||
selectors: string[];
|
|
||||||
rects: HelpHighlightRect[];
|
|
||||||
resolvedTargets?: ResolvedTarget[];
|
|
||||||
}> {
|
|
||||||
const selectors: string[] = [];
|
const selectors: string[] = [];
|
||||||
const rects: HelpHighlightRect[] = [];
|
|
||||||
const resolved: ResolvedTarget[] = [];
|
const resolved: ResolvedTarget[] = [];
|
||||||
let rootNodeId: number | undefined;
|
let rootNodeId: number | undefined;
|
||||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
if (deps.cdp && targets.length > 0) {
|
||||||
try {
|
try {
|
||||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||||
depth: 0,
|
depth: 0,
|
||||||
@@ -224,55 +262,28 @@ async function resolveHelpTargets(
|
|||||||
resolved.push({ matched, selector: tgt.selector });
|
resolved.push({ matched, selector: tgt.selector });
|
||||||
} else if (tgt.ref) {
|
} else if (tgt.ref) {
|
||||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||||
let rect: HelpHighlightRect | null = null;
|
const backendNodeId = looked?.backendNodeId ?? null;
|
||||||
if (looked && deps.cdp) {
|
let sel: string | null = null;
|
||||||
const target = {
|
if (backendNodeId !== null && deps.cdp) {
|
||||||
tabId,
|
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
|
||||||
};
|
|
||||||
const geometry = await resolveNodeGeometry(
|
|
||||||
deps.cdp,
|
|
||||||
tabId,
|
|
||||||
{
|
|
||||||
target,
|
|
||||||
backendNodeId: looked.backendNodeId,
|
|
||||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
|
||||||
},
|
|
||||||
{ scrollIntoView: options.scrollIntoView },
|
|
||||||
);
|
|
||||||
if (!isRpcError(geometry)) {
|
|
||||||
rect = {
|
|
||||||
top: geometry.topBounds.y,
|
|
||||||
left: geometry.topBounds.x,
|
|
||||||
width: geometry.topBounds.width,
|
|
||||||
height: geometry.topBounds.height,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (rect) rects.push(rect);
|
if (sel) selectors.push(sel);
|
||||||
resolved.push({
|
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||||
matched: rect !== null,
|
|
||||||
ref: tgt.ref,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refreshHelpTargets(
|
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||||
help: ActiveHelpRequest,
|
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||||
): Promise<void> {
|
|
||||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
|
||||||
help.ctx,
|
help.ctx,
|
||||||
help.primaryTabId,
|
help.primaryTabId,
|
||||||
help.targets,
|
help.targets,
|
||||||
help.deps,
|
help.deps,
|
||||||
options,
|
|
||||||
);
|
);
|
||||||
help.selectors = selectors;
|
help.selectors = selectors;
|
||||||
help.rects = rects;
|
|
||||||
help.resolvedTargets = resolvedTargets;
|
help.resolvedTargets = resolvedTargets;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -283,7 +294,12 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
|||||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
[...tabsToCancel].map((tabId) =>
|
[...tabsToCancel].map((tabId) =>
|
||||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
Promise.all([
|
||||||
|
clearRefTags(help.deps.cdp, tabId),
|
||||||
|
help.deps
|
||||||
|
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||||
|
.catch(() => {}),
|
||||||
|
]),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -460,9 +476,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (tabId === help.primaryTabId) {
|
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
|
||||||
}
|
|
||||||
help.overlayTabIds.add(tabId);
|
help.overlayTabIds.add(tabId);
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
@@ -472,7 +486,6 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
|||||||
...(help.title ? { title: help.title } : {}),
|
...(help.title ? { title: help.title } : {}),
|
||||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
|
||||||
timeoutMs: help.timeoutMs,
|
timeoutMs: help.timeoutMs,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -673,13 +686,7 @@ export async function handleRequestHelp(
|
|||||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||||
const tabId = target.tabId;
|
const tabId = target.tabId;
|
||||||
const initialTargets = params.targets ?? [];
|
const initialTargets = params.targets ?? [];
|
||||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||||
ctx,
|
|
||||||
tabId,
|
|
||||||
initialTargets,
|
|
||||||
deps,
|
|
||||||
{ scrollIntoView: true },
|
|
||||||
);
|
|
||||||
|
|
||||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||||
await deps.activateTab(tabId).catch(() => {});
|
await deps.activateTab(tabId).catch(() => {});
|
||||||
@@ -722,7 +729,6 @@ export async function handleRequestHelp(
|
|||||||
...(params.title ? { title: params.title } : {}),
|
...(params.title ? { title: params.title } : {}),
|
||||||
targets: initialTargets,
|
targets: initialTargets,
|
||||||
selectors,
|
selectors,
|
||||||
rects,
|
|
||||||
timeoutMs,
|
timeoutMs,
|
||||||
notificationId,
|
notificationId,
|
||||||
resolvedTargets,
|
resolvedTargets,
|
||||||
|
|||||||
@@ -3,15 +3,14 @@
|
|||||||
//
|
//
|
||||||
// All interaction tools:
|
// All interaction tools:
|
||||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||||
// or `selector` (DOM.querySelector + describeNode).
|
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||||
// 3. Scroll the node and its frame owners into view, project its live
|
// 3. Scroll the node into view, then dispatch the appropriate
|
||||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
// `Input.*` CDP events.
|
||||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||||
// commands.
|
// commands.
|
||||||
|
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
@@ -29,13 +28,17 @@ import type {
|
|||||||
SelectResult,
|
SelectResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import { backendNodeToObject } from "./element-geometry";
|
import {
|
||||||
|
backendNodeToObject,
|
||||||
|
boxCentre,
|
||||||
|
nodeCentre,
|
||||||
|
quadCentre,
|
||||||
|
scrollNodeIntoView,
|
||||||
|
} from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
type ChromeTabsApi,
|
type ChromeTabsApi,
|
||||||
cdpRunnerForTarget,
|
|
||||||
chromeTabsApi,
|
chromeTabsApi,
|
||||||
enforceAgentWindow,
|
enforceAgentWindow,
|
||||||
isRpcError,
|
isRpcError,
|
||||||
@@ -134,16 +137,7 @@ async function resolveBackendNode(
|
|||||||
target: { tabId: number },
|
target: { tabId: number },
|
||||||
params: { ref?: string; selector?: string },
|
params: { ref?: string; selector?: string },
|
||||||
toolName: string,
|
toolName: string,
|
||||||
): Promise<
|
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||||
| {
|
|
||||||
backendNodeId: number;
|
|
||||||
cdpTarget: CdpTarget;
|
|
||||||
frameId?: string;
|
|
||||||
usedRef?: string;
|
|
||||||
usedSelector?: string;
|
|
||||||
}
|
|
||||||
| RpcError
|
|
||||||
> {
|
|
||||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||||
if (hasRef && hasSelector) {
|
if (hasRef && hasSelector) {
|
||||||
@@ -161,15 +155,7 @@ async function resolveBackendNode(
|
|||||||
if (hasRef) {
|
if (hasRef) {
|
||||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||||
if (isRpcError(resolved)) return resolved;
|
if (isRpcError(resolved)) return resolved;
|
||||||
return {
|
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||||
backendNodeId: resolved.backendNodeId,
|
|
||||||
cdpTarget: {
|
|
||||||
tabId: target.tabId,
|
|
||||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
|
||||||
},
|
|
||||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
|
||||||
usedRef: resolved.refKey,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
// selector path
|
// selector path
|
||||||
try {
|
try {
|
||||||
@@ -206,7 +192,7 @@ async function resolveBackendNode(
|
|||||||
message: "DOM.describeNode returned no backendNodeId",
|
message: "DOM.describeNode returned no backendNodeId",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
return { backendNodeId, usedSelector: params.selector };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -242,19 +228,19 @@ export async function handleClick(
|
|||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
try {
|
||||||
const geometry = await resolveNodeGeometry(
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
deps.cdp,
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
target.tabId,
|
if (scrollErr) return scrollErr;
|
||||||
{
|
} catch (err) {
|
||||||
target: node.cdpTarget,
|
return {
|
||||||
backendNodeId: node.backendNodeId,
|
code: "cdp_failed",
|
||||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
},
|
};
|
||||||
{ scrollIntoView: true },
|
}
|
||||||
);
|
|
||||||
if (isRpcError(geometry)) return geometry;
|
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
const centre = geometry.actionPoint;
|
if (isRpcError(centre)) return centre;
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "click aborted" };
|
return { code: "cancelled", message: "click aborted" };
|
||||||
@@ -367,19 +353,19 @@ export async function handleHover(
|
|||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
|
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
try {
|
||||||
const geometry = await resolveNodeGeometry(
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
deps.cdp,
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
target.tabId,
|
if (scrollErr) return scrollErr;
|
||||||
{
|
} catch (err) {
|
||||||
target: node.cdpTarget,
|
return {
|
||||||
backendNodeId: node.backendNodeId,
|
code: "cdp_failed",
|
||||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
},
|
};
|
||||||
{ scrollIntoView: true },
|
}
|
||||||
);
|
|
||||||
if (isRpcError(geometry)) return geometry;
|
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
const centre = geometry.actionPoint;
|
if (isRpcError(centre)) return centre;
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "hover aborted" };
|
return { code: "cancelled", message: "hover aborted" };
|
||||||
@@ -599,11 +585,10 @@ export async function handleFill(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{
|
{
|
||||||
@@ -617,18 +602,12 @@ export async function handleFill(
|
|||||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
node.cdpTarget,
|
|
||||||
node.backendNodeId,
|
|
||||||
node.frameId,
|
|
||||||
);
|
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -640,7 +619,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
@@ -649,7 +628,7 @@ export async function handleFill(
|
|||||||
if (clearBefore) {
|
if (clearBefore) {
|
||||||
// Clear input/textarea value or wipe contenteditable innerText,
|
// Clear input/textarea value or wipe contenteditable innerText,
|
||||||
// then fire `input` so frameworks observe the empty state.
|
// then fire `input` so frameworks observe the empty state.
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
if (this.isContentEditable) { this.textContent = ''; }
|
if (this.isContentEditable) { this.textContent = ''; }
|
||||||
@@ -676,7 +655,7 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
functionDeclaration: `function() {
|
functionDeclaration: `function() {
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||||
@@ -874,21 +853,14 @@ export async function handlePress(
|
|||||||
if (params.ref || params.selector) {
|
if (params.ref || params.selector) {
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
node.cdpTarget,
|
|
||||||
node.backendNodeId,
|
|
||||||
node.frameId,
|
|
||||||
);
|
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "press aborted" };
|
return { code: "cancelled", message: "press aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -993,11 +965,10 @@ export async function handleSelect(
|
|||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{ backendNodeId: node.backendNodeId },
|
{ backendNodeId: node.backendNodeId },
|
||||||
@@ -1020,18 +991,12 @@ export async function handleSelect(
|
|||||||
"single-select <select> requires exactly one value",
|
"single-select <select> requires exactly one value",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
node.cdpTarget,
|
|
||||||
node.backendNodeId,
|
|
||||||
node.frameId,
|
|
||||||
);
|
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -1043,12 +1008,12 @@ export async function handleSelect(
|
|||||||
return { code: "cancelled", message: "select aborted" };
|
return { code: "cancelled", message: "select aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const evaluated = await nodeCdp.send<{
|
const evaluated = await deps.cdp.send<{
|
||||||
result?: { value?: SelectMutationResult | null };
|
result?: { value?: SelectMutationResult | null };
|
||||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||||
objectId,
|
objectId,
|
||||||
@@ -1107,6 +1072,9 @@ export async function handleSelect(
|
|||||||
|
|
||||||
export const __testing__ = {
|
export const __testing__ = {
|
||||||
DEFAULT_TIMEOUT_MS,
|
DEFAULT_TIMEOUT_MS,
|
||||||
|
quadCentre,
|
||||||
|
boxCentre,
|
||||||
resolveBackendNode,
|
resolveBackendNode,
|
||||||
|
nodeCentre,
|
||||||
isFillable,
|
isFillable,
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
+270
-293
@@ -15,26 +15,31 @@ import {
|
|||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStepAck,
|
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||||
import {
|
import {
|
||||||
type RecordFrameCoordinator,
|
applyTargetMatching,
|
||||||
type RecordingCaptureScope,
|
buildTraceV3,
|
||||||
recordFrameCoordinator,
|
cancelPendingSettles,
|
||||||
} from "@/lib/recording/frame-coordinator";
|
captureAndRegisterObservation,
|
||||||
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
|
clearPendingRedirectLanding,
|
||||||
import {
|
createObservationState,
|
||||||
appendRecordedPayload,
|
flushPendingRedirectLanding,
|
||||||
observeRecordedNavigation,
|
flushPendingSettles,
|
||||||
type RecordingStepBuffer,
|
inferMissingPostStates,
|
||||||
} from "@/lib/recording/step-buffer";
|
type RecordingObservationState,
|
||||||
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
|
rememberStepOnPage,
|
||||||
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
scheduleDraftSettle,
|
||||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
scheduleRedirectLandingFlush,
|
||||||
|
settleUnsettledDrafts,
|
||||||
|
} from "@/lib/record-observation";
|
||||||
|
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||||
|
import { buildTraceV2 } from "@/lib/trace-reducer-v2";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type {
|
import type {
|
||||||
|
DraftTraceStep,
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
RecordedTrace,
|
RecordedTrace,
|
||||||
@@ -45,7 +50,7 @@ import type {
|
|||||||
RpcError,
|
RpcError,
|
||||||
StopReason,
|
StopReason,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { TRACE_VERSION_V3 } from "@/transport/types";
|
import { TRACE_VERSION } from "@/transport/types";
|
||||||
import { handleNavigate } from "./navigation";
|
import { handleNavigate } from "./navigation";
|
||||||
import {
|
import {
|
||||||
type CdpRunner,
|
type CdpRunner,
|
||||||
@@ -58,22 +63,30 @@ import {
|
|||||||
|
|
||||||
interface ActiveRecording {
|
interface ActiveRecording {
|
||||||
requestId: string;
|
requestId: string;
|
||||||
tabs: RecordingTabCoordinator;
|
tabId: number;
|
||||||
agentWindowId: number;
|
agentWindowId: number;
|
||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
steps: RecordingDraftStep[];
|
steps: DraftTraceStep[];
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
startedAtMs: number;
|
startedAtMs: number;
|
||||||
traceVersion: 2 | 3;
|
traceVersion: 2 | 3;
|
||||||
supportsTabSwitchSteps: boolean;
|
|
||||||
finishPromise: Promise<RecordedTrace>;
|
finishPromise: Promise<RecordedTrace>;
|
||||||
resolveFinish: (trace: RecordedTrace) => void;
|
resolveFinish: (trace: RecordedTrace) => void;
|
||||||
rejectFinish: (err: Error) => void;
|
rejectFinish: (err: Error) => void;
|
||||||
settled: boolean;
|
settled: boolean;
|
||||||
finishAttempt: Promise<RecordedTrace | null> | null;
|
finishing: boolean;
|
||||||
observation: RecordingObservationRuntime | null;
|
currentUrl?: string;
|
||||||
|
pendingNavigation: boolean;
|
||||||
|
pendingNavigationDeadline?: number;
|
||||||
|
observation: RecordingObservationState;
|
||||||
stoppedBy: StopReason;
|
stoppedBy: StopReason;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
/** Tabs whose content capture acknowledged START and still need STOP. */
|
||||||
|
armedTabIds: Set<number>;
|
||||||
|
/** Rearms already sending START when finish begins. */
|
||||||
|
rearmCallbacks: Set<Promise<void>>;
|
||||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||||
navigationCallbacks: Set<Promise<void>>;
|
navigationCallbacks: Set<Promise<void>>;
|
||||||
/** Synchronous intake gate closed only after finish drains to stability. */
|
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||||
@@ -83,21 +96,16 @@ interface ActiveRecording {
|
|||||||
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||||
*/
|
*/
|
||||||
actionQueue: Promise<void>;
|
actionQueue: Promise<void>;
|
||||||
/** Last accepted sequence for each content-script document producer. */
|
/**
|
||||||
lastStepSequenceByProducer: Map<string, number>;
|
* CDP runner captured at `record_start`. The message listeners are attached
|
||||||
|
* once at service-worker startup, where no runner exists yet, so observation
|
||||||
|
* capture must go through the recording instead of the listener deps.
|
||||||
|
*/
|
||||||
|
cdp?: CdpRunner;
|
||||||
}
|
}
|
||||||
|
|
||||||
function enqueueRecordingAction(
|
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
|
||||||
recording: ActiveRecording,
|
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
|
||||||
task: () => Promise<void>,
|
|
||||||
): Promise<void> {
|
|
||||||
const queued = recording.actionQueue.then(task, task);
|
|
||||||
recording.actionQueue = queued.catch(() => {});
|
|
||||||
return queued;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isRecordingFinishing(recording: ActiveRecording): boolean {
|
|
||||||
return recording.settled || recording.finishAttempt !== null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordings = new Map<string, ActiveRecording>();
|
const recordings = new Map<string, ActiveRecording>();
|
||||||
@@ -180,11 +188,9 @@ async function sendRecordStartWithAck(
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
msg: RecordStartMessage,
|
msg: RecordStartMessage,
|
||||||
sendToTab: RecordDeps["sendToTab"],
|
sendToTab: RecordDeps["sendToTab"],
|
||||||
cancelled?: () => boolean,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
let lastError: unknown;
|
let lastError: unknown;
|
||||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||||
if (cancelled?.()) throw new Error("record start cancelled");
|
|
||||||
try {
|
try {
|
||||||
const response = await sendToTab(tabId, msg);
|
const response = await sendToTab(tabId, msg);
|
||||||
if (isRecordStartAck(response)) return;
|
if (isRecordStartAck(response)) return;
|
||||||
@@ -192,7 +198,6 @@ async function sendRecordStartWithAck(
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
lastError = err;
|
lastError = err;
|
||||||
}
|
}
|
||||||
if (cancelled?.()) throw new Error("record start cancelled");
|
|
||||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
@@ -202,28 +207,25 @@ async function sendRecordStartWithAck(
|
|||||||
|
|
||||||
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
||||||
if (params.trace_version === undefined) return 2;
|
if (params.trace_version === undefined) return 2;
|
||||||
if (params.trace_version === TRACE_VERSION_V3) return 3;
|
if (params.trace_version === TRACE_VERSION) return 3;
|
||||||
return {
|
return {
|
||||||
code: "invalid_params",
|
code: "invalid_params",
|
||||||
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
|
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION} (v3)`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
||||||
if (recording.traceVersion === 3 && recording.observation) {
|
if (recording.traceVersion === 3) {
|
||||||
return recording.observation.buildTrace({
|
return buildTraceV3({
|
||||||
drafts: recording.steps,
|
obs: recording.observation,
|
||||||
|
steps: recording.steps,
|
||||||
startedAt: recording.startedAt,
|
startedAt: recording.startedAt,
|
||||||
purpose: recording.purpose,
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
startUrl: recording.startUrl,
|
startUrl: recording.startUrl,
|
||||||
stoppedBy: recording.stoppedBy,
|
stoppedBy: recording.stoppedBy,
|
||||||
bskVersion: BSK_TRACE_VERSION,
|
bskVersion: BSK_TRACE_VERSION,
|
||||||
includeTabSwitches: recording.supportsTabSwitchSteps,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (recording.traceVersion === 3) {
|
|
||||||
throw new Error("trace v3 observation runtime is unavailable");
|
|
||||||
}
|
|
||||||
return buildTraceV2({
|
return buildTraceV2({
|
||||||
steps: recording.steps,
|
steps: recording.steps,
|
||||||
startedAt: recording.startedAt,
|
startedAt: recording.startedAt,
|
||||||
@@ -232,49 +234,33 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function stepBufferFor(
|
|
||||||
recording: ActiveRecording,
|
|
||||||
tabId: number,
|
|
||||||
fallbackUrl?: string,
|
|
||||||
): RecordingStepBuffer {
|
|
||||||
return {
|
|
||||||
steps: recording.steps,
|
|
||||||
navigation: recording.tabs.navigation(tabId, fallbackUrl),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function activateRecordingTab(
|
|
||||||
recording: ActiveRecording,
|
|
||||||
targetTabId: number,
|
|
||||||
): Promise<void> {
|
|
||||||
const previousTabId = recording.tabs.currentTabId;
|
|
||||||
if (targetTabId === previousTabId) return;
|
|
||||||
|
|
||||||
const transition = recording.observation
|
|
||||||
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
|
|
||||||
: {};
|
|
||||||
const { targetUrl, ...stateLinks } = transition;
|
|
||||||
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
|
|
||||||
op: "switch_tab",
|
|
||||||
...stateLinks,
|
|
||||||
};
|
|
||||||
recording.steps.push(draft);
|
|
||||||
recording.observation?.bindTabTransition(draft, recording.steps.length);
|
|
||||||
recording.tabs.commit(targetTabId, targetUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function processRecordedStep(
|
async function processRecordedStep(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
draftIndex: number,
|
draftIndex: number,
|
||||||
tabId: number,
|
|
||||||
producerId?: string,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!recording.observation) return;
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
try {
|
if (!cdp) return;
|
||||||
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
|
const draft = recording.steps[draftIndex];
|
||||||
} catch (err) {
|
if (!draft) return;
|
||||||
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
|
const stepId = draftIndex + 1;
|
||||||
|
|
||||||
|
if (draft.op === "navigate") {
|
||||||
|
draft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
} else {
|
||||||
|
applyTargetMatching(recording.observation, draft, stepId);
|
||||||
}
|
}
|
||||||
|
rememberStepOnPage(recording.observation, draft, stepId);
|
||||||
|
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
stepId,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
@@ -284,25 +270,16 @@ export interface RecordDeps {
|
|||||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
): Promise<unknown>;
|
): Promise<unknown>;
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
frameCoordinator?: Pick<
|
|
||||||
RecordFrameCoordinator,
|
|
||||||
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
|
|
||||||
>;
|
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
|
|
||||||
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
|
|
||||||
};
|
|
||||||
|
|
||||||
let defaultDeps: RecordDeps | null = null;
|
let defaultDeps: RecordDeps | null = null;
|
||||||
function getDefaultDeps(): RecordDeps {
|
function getDefaultDeps(): RecordDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
defaultDeps = {
|
defaultDeps = {
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
frameCoordinator: recordFrameCoordinator,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return defaultDeps;
|
return defaultDeps;
|
||||||
@@ -363,55 +340,28 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
|||||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: RecordStepAck) => void,
|
_sendResponse: () => void,
|
||||||
) => {
|
) => {
|
||||||
if (!isRecordStepMessage(message)) return false;
|
if (!isRecordStepMessage(message)) return false;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.requestId !== message.requestId) continue;
|
if (recording.requestId !== message.requestId) continue;
|
||||||
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
|
|
||||||
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
|
|
||||||
: undefined;
|
|
||||||
if (deps.frameCoordinator && !source) return false;
|
|
||||||
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
|
|
||||||
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
|
|
||||||
const producerKey = source
|
|
||||||
? `${source.tabId}:${source.documentId}:${source.producerId}`
|
|
||||||
: `${sourceTabId}:${message.producerId}`;
|
|
||||||
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
|
|
||||||
if (message.sequence < expectedSequence) {
|
|
||||||
sendResponse({ ok: true, sequence: message.sequence });
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (message.sequence > expectedSequence) {
|
|
||||||
sendResponse({
|
|
||||||
ok: false,
|
|
||||||
expectedSequence,
|
|
||||||
error: `out-of-order recorded step ${message.sequence}`,
|
|
||||||
});
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
|
|
||||||
enqueueRecordingAction(recording, async () => {
|
enqueueRecordingAction(recording, async () => {
|
||||||
if (sourceTabId !== recording.tabs.currentTabId) {
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
if (!sourceWasActive) return;
|
// Flush before appending so a coalesced OAuth navigate is ordered
|
||||||
if (sourceTabId !== recording.tabs.activeTabId)
|
// ahead of the click that landed on the destination page. The flush
|
||||||
recording.tabs.noteActivation(sourceTabId);
|
// itself waits for that navigate's observation before returning.
|
||||||
await activateRecordingTab(recording, sourceTabId);
|
await flushPendingRedirectLanding(
|
||||||
}
|
recording.observation,
|
||||||
await recording.observation?.flushRedirects(sourceTabId);
|
recording.steps,
|
||||||
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
|
cdp,
|
||||||
const draftIndex = appendRecordedPayload(
|
recording.tabId,
|
||||||
stepBufferFor(recording, sourceTabId, message.step.page_url),
|
deps.tabsApi,
|
||||||
message.step,
|
{ cancelled: () => recording.settled },
|
||||||
targetHint,
|
|
||||||
);
|
);
|
||||||
if (draftIndex !== null) {
|
const draftIndex = appendRecordedPayload(recording, message.step);
|
||||||
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
|
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||||
}
|
|
||||||
});
|
});
|
||||||
sendResponse({ ok: true, sequence: message.sequence });
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@@ -438,7 +388,9 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
|
|
||||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
|
if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
|
||||||
|
return recording;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -475,7 +427,7 @@ async function clearRearmTimersForRecording(
|
|||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
clearRearmTimer(recording.tabs.currentTabId);
|
clearRearmTimer(recording.tabId);
|
||||||
try {
|
try {
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||||
for (const tab of tabs) {
|
for (const tab of tabs) {
|
||||||
@@ -490,33 +442,25 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
|
|
||||||
throw new Error("failed to flush one or more recording documents");
|
|
||||||
}
|
|
||||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||||
let tabIds = [recording.tabs.currentTabId];
|
let failed = false;
|
||||||
try {
|
for (const tabId of [...recording.armedTabIds]) {
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
try {
|
||||||
tabIds = [
|
await deps.tabsApi.get(tabId);
|
||||||
...new Set([
|
} catch {
|
||||||
recording.tabs.currentTabId,
|
// A closed tab no longer has content capture to flush.
|
||||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
recording.armedTabIds.delete(tabId);
|
||||||
]),
|
continue;
|
||||||
];
|
}
|
||||||
} catch {
|
|
||||||
// Fall back to the current recording tab.
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const tabId of tabIds) {
|
|
||||||
try {
|
try {
|
||||||
const response = await deps.sendToTab(tabId, stopMsg);
|
const response = await deps.sendToTab(tabId, stopMsg);
|
||||||
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
|
if (!isRecordStartAck(response)) {
|
||||||
throw new Error("content script did not confirm recorded steps");
|
throw new Error("content script did not confirm recorded steps");
|
||||||
}
|
}
|
||||||
|
recording.armedTabIds.delete(tabId);
|
||||||
} catch {
|
} catch {
|
||||||
if (tabId === recording.tabs.currentTabId) {
|
failed = true;
|
||||||
throw new Error("failed to flush recorded steps");
|
continue;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (deps.bypassOverlay) {
|
if (deps.bypassOverlay) {
|
||||||
try {
|
try {
|
||||||
@@ -526,54 +470,53 @@ async function stopRecordingOnAllAgentTabs(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (failed || recording.armedTabIds.size > 0) {
|
||||||
|
throw new Error("failed to flush recorded steps");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rearmRecording(
|
async function rearmRecording(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
targetTabId: number,
|
targetTabId: number,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
activation?: TabActivation,
|
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
|
let resolveTracked!: () => void;
|
||||||
|
const tracked = new Promise<void>((resolve) => {
|
||||||
|
resolveTracked = resolve;
|
||||||
|
});
|
||||||
|
recording.rearmCallbacks.add(tracked);
|
||||||
|
|
||||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||||
// already hides the control chrome while activeRecord is set.
|
// already hides the control chrome while activeRecord is set.
|
||||||
const isFinishing = () => isRecordingFinishing(recording);
|
try {
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||||
if (isFinishing()) return false;
|
if (recording.settled || recording.finishing) return false;
|
||||||
const startMsg: RecordStartMessage = {
|
const startMsg: RecordStartMessage = {
|
||||||
type: RECORD_START,
|
type: RECORD_START,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
startedAtMs: recording.startedAtMs,
|
startedAtMs: recording.startedAtMs,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
if (deps.frameCoordinator) {
|
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||||
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
|
recording.armedTabIds.add(targetTabId);
|
||||||
if (!frameStarted) throw new Error("recording document did not start");
|
recording.tabId = targetTabId;
|
||||||
}
|
return true;
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
|
} catch {
|
||||||
if (isFinishing()) return false;
|
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||||
if (activation) {
|
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||||
if (!recording.tabs.isLatest(activation)) return true;
|
}
|
||||||
await enqueueRecordingAction(recording, async () => {
|
|
||||||
if (isFinishing() || !recording.tabs.isLatest(activation)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await activateRecordingTab(recording, targetTabId);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
if (isFinishing()) return false;
|
|
||||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
|
||||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
recording.rearmCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
|
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||||
const existing = rearmTimers.get(tabId);
|
const existing = rearmTimers.get(tabId);
|
||||||
if (existing) clearTimeout(existing);
|
if (existing) clearTimeout(existing);
|
||||||
rearmTimers.set(
|
rearmTimers.set(
|
||||||
@@ -582,7 +525,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabAc
|
|||||||
rearmTimers.delete(tabId);
|
rearmTimers.delete(tabId);
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const current = await findRecordingForTab(tabId, deps);
|
const current = await findRecordingForTab(tabId, deps);
|
||||||
if (current) await rearmRecording(current, tabId, deps, activation);
|
if (current) await rearmRecording(current, tabId, deps);
|
||||||
})();
|
})();
|
||||||
}, RECORD_REARM_DEBOUNCE_MS),
|
}, RECORD_REARM_DEBOUNCE_MS),
|
||||||
);
|
);
|
||||||
@@ -594,20 +537,14 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
|||||||
const windowId = tab.windowId;
|
const windowId = tab.windowId;
|
||||||
if (tabId === undefined || windowId === undefined) return;
|
if (tabId === undefined || windowId === undefined) return;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
|
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
||||||
scheduleRearmForTab(tabId, deps);
|
scheduleRearmForTab(tabId, deps);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||||
for (const recording of recordings.values()) {
|
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||||
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
|
|
||||||
continue;
|
|
||||||
const activation = recording.tabs.noteActivation(activeInfo.tabId);
|
|
||||||
scheduleRearmForTab(activeInfo.tabId, deps, activation);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
chrome.tabs.onCreated.addListener(onCreated);
|
chrome.tabs.onCreated.addListener(onCreated);
|
||||||
@@ -627,39 +564,80 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
transitionQualifiers?: string[],
|
transitionQualifiers?: string[],
|
||||||
) => {
|
) => {
|
||||||
if (!url) return;
|
if (!url) return;
|
||||||
const candidates = [...recordings.values()].filter(
|
const direct = findRecordingByTabId(tabId);
|
||||||
(recording) =>
|
const candidates = direct
|
||||||
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
|
? direct.acceptingNavigation
|
||||||
);
|
? [direct]
|
||||||
for (const recording of candidates) {
|
: []
|
||||||
const queued = enqueueRecordingAction(recording, async () => {
|
: [...recordings.values()].filter(
|
||||||
if (tabId !== recording.tabs.currentTabId) {
|
(recording) => !recording.settled && recording.acceptingNavigation,
|
||||||
await activateRecordingTab(recording, tabId);
|
|
||||||
}
|
|
||||||
const result = observeRecordedNavigation(
|
|
||||||
stepBufferFor(recording, tabId, url),
|
|
||||||
url,
|
|
||||||
causedByAction,
|
|
||||||
transitionType,
|
|
||||||
transitionQualifiers,
|
|
||||||
);
|
);
|
||||||
if (result.kind === "coalesce_redirect") {
|
if (candidates.length === 0) return;
|
||||||
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
|
|
||||||
|
let resolveTracked!: () => void;
|
||||||
|
const tracked = new Promise<void>((resolve) => {
|
||||||
|
resolveTracked = resolve;
|
||||||
|
});
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const recording = await findRecordingForTab(tabId, deps);
|
||||||
|
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
recording,
|
||||||
|
url,
|
||||||
|
causedByAction,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (!cdp) return;
|
||||||
|
|
||||||
if (result.kind === "noop") return;
|
if (result.kind === "coalesce_redirect") {
|
||||||
recording.observation?.clearRedirect(tabId);
|
scheduleRedirectLandingFlush(
|
||||||
if (result.kind === "appended") {
|
recording.observation,
|
||||||
await processRecordedStep(recording, result.index, tabId);
|
recording.steps,
|
||||||
} else {
|
cdp,
|
||||||
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
|
tabId,
|
||||||
}
|
deps.tabsApi,
|
||||||
});
|
result.url,
|
||||||
const tracked = queued.catch(() => {});
|
{ cancelled: () => recording.settled },
|
||||||
recording.navigationCallbacks.add(tracked);
|
);
|
||||||
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (result.kind === "noop") return;
|
||||||
|
|
||||||
|
// A concrete navigation supersedes any in-flight redirect coalesce.
|
||||||
|
clearPendingRedirectLanding(recording.observation);
|
||||||
|
|
||||||
|
const draftIndex = result.index;
|
||||||
|
const lastDraft = recording.steps[draftIndex];
|
||||||
|
if (!lastDraft) return;
|
||||||
|
|
||||||
|
if (result.kind === "appended") {
|
||||||
|
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
|
||||||
|
}
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
draftIndex + 1,
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
|
}
|
||||||
|
})();
|
||||||
};
|
};
|
||||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||||
void (async () => {
|
void (async () => {
|
||||||
@@ -705,14 +683,34 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
|
|
||||||
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||||
|
|
||||||
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
|
async function drainRecordingToStability(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||||
await Promise.all([...recording.navigationCallbacks]);
|
await Promise.all([...recording.navigationCallbacks]);
|
||||||
const actionTail = recording.actionQueue;
|
const actionTail = recording.actionQueue;
|
||||||
await actionTail;
|
await actionTail;
|
||||||
await recording.observation?.flush();
|
await flushPendingRedirectLanding(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
await flushPendingSettles(recording.observation);
|
||||||
|
const redirectTail = recording.observation.redirectFlushQueue;
|
||||||
|
const settleTail = recording.observation.settleQueue;
|
||||||
|
await Promise.all([redirectTail, settleTail]);
|
||||||
|
|
||||||
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
|
if (
|
||||||
|
recording.navigationCallbacks.size === 0 &&
|
||||||
|
recording.actionQueue === actionTail &&
|
||||||
|
recording.observation.redirectFlushQueue === redirectTail &&
|
||||||
|
recording.observation.settleQueue === settleTail
|
||||||
|
) {
|
||||||
// No event or promise continuation can interleave with this synchronous
|
// No event or promise continuation can interleave with this synchronous
|
||||||
// check-and-close, so work accepted before the cutoff is fully drained.
|
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||||
recording.acceptingNavigation = false;
|
recording.acceptingNavigation = false;
|
||||||
@@ -741,8 +739,7 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
|
await rearmRecording(recording, tabId, deps);
|
||||||
await rearmRecording(recording, tabId, deps, activation);
|
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
@@ -776,38 +773,37 @@ async function finishRecording(
|
|||||||
): Promise<RecordedTrace | null> {
|
): Promise<RecordedTrace | null> {
|
||||||
const recording = recordings.get(sessionId);
|
const recording = recordings.get(sessionId);
|
||||||
if (!recording || recording.settled) return null;
|
if (!recording || recording.settled) return null;
|
||||||
if (recording.finishAttempt) return recording.finishAttempt;
|
if (recording.finishing) return recording.finishPromise;
|
||||||
|
recording.finishing = true;
|
||||||
recording.stoppedBy = stoppedBy;
|
recording.stoppedBy = stoppedBy;
|
||||||
|
|
||||||
const attempt = finishRecordingAttempt(sessionId, recording, deps);
|
|
||||||
recording.finishAttempt = attempt;
|
|
||||||
try {
|
|
||||||
return await attempt;
|
|
||||||
} finally {
|
|
||||||
if (!recording.settled) {
|
|
||||||
recording.finishAttempt = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function finishRecordingAttempt(
|
|
||||||
sessionId: string,
|
|
||||||
recording: ActiveRecording,
|
|
||||||
deps: RecordDeps,
|
|
||||||
): Promise<RecordedTrace | null> {
|
|
||||||
await clearRearmTimersForRecording(recording, deps);
|
await clearRearmTimersForRecording(recording, deps);
|
||||||
|
await Promise.all([...recording.rearmCallbacks]);
|
||||||
try {
|
try {
|
||||||
// Disposing content capture may commit a final dirty fill. Flush content
|
// Disposing content capture may commit a final dirty fill. Flush content
|
||||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||||
// and the observation queues are drained.
|
// and the observation queues are drained.
|
||||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||||
} catch {
|
} catch {
|
||||||
|
recording.finishing = false;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (!(await drainRecordingToStability(recording))) {
|
if (!(await drainRecordingToStability(recording, deps))) {
|
||||||
|
recording.finishing = false;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
|
cancelPendingSettles(recording.observation);
|
||||||
|
inferMissingPostStates(recording.steps);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (cdp) {
|
||||||
|
await settleUnsettledDrafts(
|
||||||
|
recording.observation,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recordings.delete(sessionId);
|
recordings.delete(sessionId);
|
||||||
@@ -824,12 +820,6 @@ export async function handleRecordStart(
|
|||||||
): Promise<RecordStartResult | RpcError> {
|
): Promise<RecordStartResult | RpcError> {
|
||||||
const traceVersionOrErr = negotiatedTraceVersion(params);
|
const traceVersionOrErr = negotiatedTraceVersion(params);
|
||||||
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
||||||
if (traceVersionOrErr === 3 && !deps.cdp) {
|
|
||||||
return {
|
|
||||||
code: "protocol_error",
|
|
||||||
message: "trace v3 recording requires an active CDP connection",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
@@ -859,7 +849,7 @@ export async function handleRecordStart(
|
|||||||
const redactValues = params.redact_values ?? false;
|
const redactValues = params.redact_values ?? false;
|
||||||
recordings.set(params.session_id, {
|
recordings.set(params.session_id, {
|
||||||
requestId,
|
requestId,
|
||||||
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
|
tabId: target.tabId,
|
||||||
agentWindowId: ctx.agentWindowId,
|
agentWindowId: ctx.agentWindowId,
|
||||||
startUrl: navigateUrl,
|
startUrl: navigateUrl,
|
||||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||||
@@ -867,44 +857,30 @@ export async function handleRecordStart(
|
|||||||
startedAt: new Date(startedAtMs).toISOString(),
|
startedAt: new Date(startedAtMs).toISOString(),
|
||||||
startedAtMs,
|
startedAtMs,
|
||||||
traceVersion: traceVersionOrErr,
|
traceVersion: traceVersionOrErr,
|
||||||
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
|
|
||||||
finishPromise,
|
finishPromise,
|
||||||
resolveFinish,
|
resolveFinish,
|
||||||
rejectFinish,
|
rejectFinish,
|
||||||
settled: false,
|
settled: false,
|
||||||
finishAttempt: null,
|
finishing: false,
|
||||||
observation:
|
currentUrl: navigateUrl,
|
||||||
traceVersionOrErr === 3 && deps.cdp
|
pendingNavigation: false,
|
||||||
? new RecordingObservationRuntime({
|
pendingNavigationDeadline: undefined,
|
||||||
cdp: deps.cdp,
|
observation: createObservationState({ maxPageTokens, redactValues }),
|
||||||
tabsApi: deps.tabsApi,
|
|
||||||
maxTokens: maxPageTokens,
|
|
||||||
redactValues,
|
|
||||||
})
|
|
||||||
: null,
|
|
||||||
stoppedBy: "user_finish",
|
stoppedBy: "user_finish",
|
||||||
|
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues,
|
||||||
|
armedTabIds: new Set(),
|
||||||
|
rearmCallbacks: new Set(),
|
||||||
navigationCallbacks: new Set(),
|
navigationCallbacks: new Set(),
|
||||||
acceptingNavigation: true,
|
acceptingNavigation: true,
|
||||||
actionQueue: Promise.resolve(),
|
actionQueue: Promise.resolve(),
|
||||||
lastStepSequenceByProducer: new Map(),
|
...(deps.cdp ? { cdp: deps.cdp } : {}),
|
||||||
});
|
|
||||||
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
|
|
||||||
const recording = recordings.get(params.session_id);
|
|
||||||
if (!recording || recording.requestId !== requestId || recording.settled) return;
|
|
||||||
try {
|
|
||||||
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
|
|
||||||
} catch {
|
|
||||||
// The normal action-time capture remains available if a child Document
|
|
||||||
// is replaced while its readiness refresh is in flight.
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
// Observe navigations for the whole recording lifetime; attach before
|
// Observe navigations for the whole recording lifetime; attach before
|
||||||
// optional navigate so the destination load can rearm capture.
|
// optional navigate so the destination load can rearm capture.
|
||||||
ensureBrowserObservationListeners(deps);
|
ensureBrowserObservationListeners(deps);
|
||||||
|
|
||||||
const abortPending = async (notifyContent: boolean) => {
|
const abortPending = async (notifyContent: boolean) => {
|
||||||
recordings.get(params.session_id)?.observation?.cancel();
|
|
||||||
deps.frameCoordinator?.cancel(requestId);
|
|
||||||
recordings.delete(params.session_id);
|
recordings.delete(params.session_id);
|
||||||
releaseBrowserObservationListenersIfIdle();
|
releaseBrowserObservationListenersIfIdle();
|
||||||
if (notifyContent) {
|
if (notifyContent) {
|
||||||
@@ -986,7 +962,7 @@ export async function handleRecordStart(
|
|||||||
return cancelledError();
|
return cancelledError();
|
||||||
}
|
}
|
||||||
active.startUrl = startUrl;
|
active.startUrl = startUrl;
|
||||||
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
|
active.currentUrl = startUrl;
|
||||||
|
|
||||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||||
await abortPending(false);
|
await abortPending(false);
|
||||||
@@ -994,7 +970,7 @@ export async function handleRecordStart(
|
|||||||
code: "invalid_params",
|
code: "invalid_params",
|
||||||
message: params.url
|
message: params.url
|
||||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||||
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`,
|
: `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1021,11 +997,8 @@ export async function handleRecordStart(
|
|||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (deps.frameCoordinator) {
|
|
||||||
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
|
|
||||||
if (!frameStarted) throw new Error("top recording document did not start");
|
|
||||||
}
|
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||||
|
active.armedTabIds.add(target.tabId);
|
||||||
} catch {
|
} catch {
|
||||||
await abortPending(true);
|
await abortPending(true);
|
||||||
return {
|
return {
|
||||||
@@ -1035,11 +1008,17 @@ export async function handleRecordStart(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (active.observation) {
|
if (deps.cdp) {
|
||||||
const activeRecording = recordings.get(params.session_id);
|
const activeRecording = recordings.get(params.session_id);
|
||||||
if (activeRecording) {
|
if (activeRecording) {
|
||||||
try {
|
try {
|
||||||
await activeRecording.observation?.captureInitial(target.tabId);
|
await captureAndRegisterObservation(
|
||||||
|
activeRecording.observation,
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
startUrl,
|
||||||
|
);
|
||||||
} catch {
|
} catch {
|
||||||
// Proceed without initial observation; steps may be dropped by reducer.
|
// Proceed without initial observation; steps may be dropped by reducer.
|
||||||
}
|
}
|
||||||
@@ -1144,8 +1123,6 @@ export function clearRecordingForSession(sessionId: string): void {
|
|||||||
}
|
}
|
||||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||||
if (!recording.settled) {
|
if (!recording.settled) {
|
||||||
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
|
|
||||||
recording.observation?.cancel();
|
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recording.rejectFinish(new Error("recording cleared"));
|
recording.rejectFinish(new Error("recording cleared"));
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user