Compare commits

..
Author SHA1 Message Date
Ljy-0827 2853ff1b44 fix(record): rebase and dto policy 2026-08-22 17:01:16 +08:00
Ljy-0827 e90e70657f fix(recording): using unified geometry 2026-08-22 17:01:16 +08:00
“haonan”andCursor bd1f2e211e feat(record): capture Trace v3 states after pages settle
Wait for DOM quiet after actions, match targets to VOM refs, and reduce
drafts into deduped state-linked Trace v3 while keeping the v2 reducer.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-22 17:01:16 +08:00
“haonan” 8c76658e76 fix(protocol): align trace schemas with v2/v3 classify rules
RecordedTrace validation now follows classify_value — numeric version
selects v3, otherwise v2 — so mixed pages/states and v2 envelopes with
a version field fail Schema checks before Recall import. Keep standalone
trace_step as a v2|v3 union for tools that validate a single step.
2026-08-22 17:01:15 +08:00
“haonan” 825abde4cb fix(protocol): keep generic trace schema compatible with v2 and v3
Split standalone v2/v3 schemas and drop TraceV2 deny_unknown_fields so legacy traces with extra keys still parse; mix and version checks stay in RecordedTrace.
2026-08-22 17:01:15 +08:00
Ljy-0827 f1a97b16ab fix(protocol): compatible with dto 2026-08-22 17:01:15 +08:00
Ljy-0827 fa7ab71c35 fix(protocol): integrate with vom 2026-08-22 17:01:15 +08:00
“haonan” b473ed0ce8 test(protocol): align handshake fixtures with 1.0 floor and clarify v2/v3 naming 2026-08-22 17:01:15 +08:00
“haonan”andCursor ab5c6d414a fix(protocol): align Trace v2/v3 classification and hover steps
Require Trace v3 version const 3, reject mixed pages/states payloads, and
align TypeScript selection optionality with the generated schema. Trace v2
on main already records hover steps, so the frozen v2 schema must accept
them instead of dropping them during reduction.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-22 17:01:15 +08:00
“haonan”andCursor 954078380c feat(protocol): negotiate Trace v3 with v2 compatibility
Define Trace v3 wire types and generated schemas while keeping protocol
1.1 peers compatible with 1.0. Default record start still produces Trace
v2, and only an explicit trace_version=3 request enables the new shape.
Refs #90

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-22 17:01:15 +08:00
196 changed files with 2592 additions and 19821 deletions

No files matched your search

-177
View File
@@ -325,180 +325,3 @@ jobs:
echo "Chrome Web Store submission created successfully:" echo "Chrome Web Store submission created successfully:"
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE" jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
publish-edge-addons:
needs: [resolve, build-extension]
runs-on: ubuntu-latest
environment: edge-addons
permissions:
contents: read
steps:
- name: Download build artifact
uses: actions/download-artifact@v8
with:
name: extension-zip
path: dist
- name: Validate Microsoft Edge Add-ons configuration
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
run: |
set -euo pipefail
if [ -z "$API_KEY" ]; then
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
exit 1
fi
if [ -z "$CLIENT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
exit 1
fi
if [ -z "$PRODUCT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
exit 1
fi
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
exit 1
fi
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
if [ -z "$ARCHIVE" ]; then
echo "::error::No extension zip found in the downloaded artifact"
exit 1
fi
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
- name: Upload and publish Microsoft Edge Add-ons extension
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
operation_id_from_headers() {
local headers="$1"
local location
location="$(awk '
tolower($1) == "location:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
print
exit
}
' "$headers")"
location="${location%%\?*}"
printf '%s' "${location##*/}"
}
poll_operation() {
local operation_name="$1"
local status_url="$2"
local status="InProgress"
local http_status
for attempt in $(seq 1 30); do
sleep 5
http_status="$(curl --silent --show-error \
--output "$STATUS_RESPONSE" \
--write-out '%{http_code}' \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
"$status_url")"
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
exit 1
fi
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
echo "$operation_name status check $attempt/30: $status"
case "$status" in
Succeeded)
return 0
;;
InProgress)
;;
Failed)
echo "::error::Microsoft Edge Add-ons $operation_name failed"
jq . "$STATUS_RESPONSE"
exit 1
;;
*)
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
jq . "$STATUS_RESPONSE"
exit 1
;;
esac
done
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
jq . "$STATUS_RESPONSE"
exit 1
}
HTTP_STATUS="$(curl --silent --show-error \
--output "$UPLOAD_RESPONSE" \
--dump-header "$UPLOAD_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/zip" \
--data-binary "@$ARCHIVE" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
exit 1
fi
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
if [ -z "$UPLOAD_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
cat "$UPLOAD_HEADERS"
exit 1
fi
poll_operation \
"package upload" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
HTTP_STATUS="$(curl --silent --show-error \
--output "$PUBLISH_RESPONSE" \
--dump-header "$PUBLISH_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/json" \
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
"$API_ROOT/products/$PRODUCT_ID/submissions")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
exit 1
fi
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
if [ -z "$PUBLISH_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
cat "$PUBLISH_HEADERS"
exit 1
fi
poll_operation \
"submission" \
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
echo "Microsoft Edge Add-ons submission created successfully:"
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
-8
View File
@@ -7,18 +7,10 @@ dist/
.output/ .output/
.wxt/ .wxt/
.web-extension-id .web-extension-id
.pnpm-store/
# Git worktrees
.worktrees/
# Logs # Logs
*.log *.log
# Local browser evaluation artifacts
evals/browser/results/
evals/browser/agents.local.json
# macOS # macOS
.DS_Store .DS_Store
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]] [[package]]
name = "bsk" name = "bsk"
version = "0.1.11" version = "0.1.10"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bsk-protocol", "bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]] [[package]]
name = "bsk-protocol" name = "bsk-protocol"
version = "0.1.11" version = "0.1.10"
dependencies = [ dependencies = [
"schemars", "schemars",
"semver", "semver",
+1 -1
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"] default-members = ["crates/bsk-cli"]
[workspace.package] [workspace.package]
version = "0.1.11" version = "0.1.10"
edition = "2024" edition = "2024"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
-1
View File
@@ -208,7 +208,6 @@ The repository is a Cargo + pnpm workspace:
- `apps/extension` — browser extension - `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support - `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`) - `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
## License ## License
-1
View File
@@ -178,7 +178,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `apps/extension` — 浏览器扩展 - `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持 - `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`) - `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
## 许可证 ## 许可证
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@browser-skill/extension", "name": "@browser-skill/extension",
"version": "0.1.7", "version": "0.1.6",
"private": true, "private": true,
"type": "module", "type": "module",
"scripts": { "scripts": {
@@ -1,12 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest";
import type { RecordStepPayload } from "@/lib/record-bridge"; import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture"; import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
runtime: { runtime: {
sendMessage: vi.fn((message: { sequence?: number }) => sendMessage: vi.fn(() => Promise.resolve()),
Promise.resolve({ ok: true, sequence: message.sequence }),
),
}, },
}); });
@@ -48,6 +46,58 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
} }
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
});
describe("record-capture semantic", () => { describe("record-capture semantic", () => {
let steps: RecordStepPayload[]; let steps: RecordStepPayload[];
@@ -124,19 +174,6 @@ describe("record-capture semantic", () => {
]); ]);
}); });
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => { it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = ` document.body.innerHTML = `
<div id="chrome">page chrome</div> <div id="chrome">page chrome</div>
@@ -191,7 +228,6 @@ describe("record-capture semantic", () => {
expect(steps[0]).toMatchObject({ expect(steps[0]).toMatchObject({
op: "hover", op: "hover",
target: { role: "button", name: "Open user navigation menu" }, target: { role: "button", name: "Open user navigation menu" },
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
}); });
expect(steps[1]).toMatchObject({ expect(steps[1]).toMatchObject({
op: "click", op: "click",
@@ -1,102 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
@@ -1,36 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { RecordStepMessage } from "@/lib/record-bridge";
import { RecordStepDelivery } from "../record-step-delivery";
describe("RecordStepDelivery", () => {
it("retries from the first unacknowledged sequence before sending later steps", async () => {
const sent: RecordStepMessage[] = [];
const send = vi.fn(async (message: RecordStepMessage) => {
sent.push(message);
if (sent.length === 1) throw new Error("service worker unavailable");
return { ok: true, sequence: message.sequence };
});
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
await expect(delivery.flush()).resolves.toBe(true);
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
});
it("keeps unacknowledged steps pending for a later flush", async () => {
const send = vi
.fn<(message: RecordStepMessage) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
await expect(delivery.flush()).resolves.toBe(false);
await expect(delivery.flush()).resolves.toBe(true);
expect(send).toHaveBeenCalledTimes(3);
});
});
+141 -64
View File
@@ -12,7 +12,21 @@ import {
hasDirectHoverInteractiveSignal, hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal, hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy"; } from "@/lib/hover-trigger-policy";
import type { RecordStepPayload } from "@/lib/record-bridge"; import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/recording/draft-policy"; import { shouldRecordPress } from "@/lib/recording/draft-policy";
import { import {
closestHoverSurfaceCandidate, closestHoverSurfaceCandidate,
@@ -23,6 +37,28 @@ import {
isLikelyHoverSurfaceOwner, isLikelyHoverSurfaceOwner,
} from "./record-hover-surface"; } from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const knownRecordRequests = new Set<string>();
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const payload = { type: RECORD_STEP, requestId, step };
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
return false;
},
);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
void pending.then(() => {
sends.delete(pending);
if (sends.size === 0) pendingStepSends.delete(requestId);
});
}
export interface RecordCaptureController { export interface RecordCaptureController {
dispose(): void; dispose(): void;
} }
@@ -32,7 +68,6 @@ interface FillSession {
target: CaptureTargetDescriptor; target: CaptureTargetDescriptor;
baselineValue: string; baselineValue: string;
lastValue: string; lastValue: string;
pendingCommit?: "enter" | "suggestion" | "blur";
} }
interface HoverCandidate { interface HoverCandidate {
@@ -122,24 +157,6 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
return null; return null;
} }
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
const rect = el.getBoundingClientRect();
return {
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
tag: el.tagName.toLowerCase(),
};
}
function geometryForEventTarget(
target: EventTarget | null,
): RecordStepPayload["geometry"] | undefined {
if (!(target instanceof Element)) return undefined;
const clickable = target.closest(
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
);
return captureGeometry(clickable ?? target);
}
function fillableValue(el: FillableElement): string { function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value; return el.value;
@@ -359,7 +376,6 @@ function scheduleInputCompletionCommit(
export function startRecordCapture( export function startRecordCapture(
_requestId: string, _requestId: string,
sendStep: (step: RecordStepPayload) => void, sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController { ): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => { const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step }); sendStep({ page_url: location.href, ...step });
@@ -398,17 +414,14 @@ export function startRecordCapture(
emitStep({ emitStep({
op: "fill", op: "fill",
target: session.target, target: session.target,
geometry: captureGeometry(session.element),
value, value,
commit: session.pendingCommit ?? "blur",
...(isPassword ? { redacted: true } : {}), ...(isPassword ? { redacted: true } : {}),
}); });
}; };
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => { const commitFillSession = () => {
if (!fillSession || composing) return; if (!fillSession || composing) return;
const session = fillSession; const session = fillSession;
session.pendingCommit = commit;
fillSession = null; fillSession = null;
emitFill(session); emitFill(session);
committedValues.set(session.element, session.lastValue); committedValues.set(session.element, session.lastValue);
@@ -614,7 +627,6 @@ export function startRecordCapture(
emitStep({ emitStep({
op: "hover", op: "hover",
target: hover.target, target: hover.target,
geometry: captureGeometry(hover.element),
}); });
emittedHoverElements.add(hover.element); emittedHoverElements.add(hover.element);
}; };
@@ -669,7 +681,6 @@ export function startRecordCapture(
emitStep({ emitStep({
op: "click", op: "click",
target, target,
geometry: geometryForEventTarget(eventTarget(event)),
expects_navigation: true, expects_navigation: true,
}); });
}; };
@@ -771,7 +782,7 @@ export function startRecordCapture(
scheduleInputCompletionCommit( scheduleInputCompletionCommit(
sessionElement, sessionElement,
syncFillSessionValue, syncFillSessionValue,
() => commitFillSession("suggestion"), commitFillSession,
(el) => fillSession?.element === el, (el) => fillSession?.element === el,
); );
return; return;
@@ -831,7 +842,6 @@ export function startRecordCapture(
emitStep({ emitStep({
op: "select", op: "select",
target: desc, target: desc,
geometry: captureGeometry(target),
values, values,
labels, labels,
expects_navigation: true, expects_navigation: true,
@@ -872,7 +882,7 @@ export function startRecordCapture(
}; };
} }
if (fillable) { if (fillable) {
commitFillSession(event.key === "Enter" ? "enter" : "blur"); commitFillSession();
} }
const desc = describeEventTarget(target); const desc = describeEventTarget(target);
if (!desc && !event.key) return; if (!desc && !event.key) return;
@@ -880,7 +890,7 @@ export function startRecordCapture(
emitStep({ emitStep({
op: "press", op: "press",
key: event.key, key: event.key,
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}), ...(desc ? { target: desc } : {}),
...(modifiers.length ? { modifiers } : {}), ...(modifiers.length ? { modifiers } : {}),
expects_navigation: event.key === "Enter", expects_navigation: event.key === "Enter",
}); });
@@ -896,33 +906,23 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true); document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true); document.addEventListener("keydown", onKeyDown, true);
const captureNavigation = options.captureNavigation ?? true; const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
const urlObserver = captureNavigation urlObserver.observe(document, { subtree: true, childList: true });
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged(); const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as window.addEventListener("hashchange", onUrlEvent);
// the `commit` argument and stamp it onto the recorded fill step. window.addEventListener("popstate", onUrlEvent);
const onPageHide = () => commitFillSession(); window.addEventListener("pagehide", commitFillSession);
if (captureNavigation) {
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState; const originalPushState = history.pushState;
const originalReplaceState = history.replaceState; const originalReplaceState = history.replaceState;
if (captureNavigation) { history.pushState = function (...args: Parameters<History["pushState"]>) {
history.pushState = function (...args: Parameters<History["pushState"]>) { originalPushState.apply(this, args);
originalPushState.apply(this, args); emitNavigateIfChanged(navigationActionPending ? true : undefined);
emitNavigateIfChanged(navigationActionPending ? true : undefined); };
}; history.replaceState = function (...args: Parameters<History["replaceState"]>) {
history.replaceState = function (...args: Parameters<History["replaceState"]>) { originalReplaceState.apply(this, args);
originalReplaceState.apply(this, args); emitNavigateIfChanged(navigationActionPending ? true : undefined);
emitNavigateIfChanged(navigationActionPending ? true : undefined); };
};
}
return { return {
dispose() { dispose() {
@@ -936,16 +936,93 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true); document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true); document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true); document.removeEventListener("keydown", onKeyDown, true);
urlObserver?.disconnect(); urlObserver.disconnect();
if (captureNavigation) { window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("hashchange", onUrlEvent); window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent); window.removeEventListener("pagehide", commitFillSession);
} history.pushState = originalPushState;
window.removeEventListener("pagehide", onPageHide); history.replaceState = originalReplaceState;
if (captureNavigation) {
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
}
}, },
}; };
} }
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
void Promise.all(pending).then((delivered) => {
finishStop();
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
sendResponse(
succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
},
);
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -1,65 +0,0 @@
import {
isAcceptedRecordStepAck,
RECORD_STEP,
type RecordStepMessage,
type RecordStepPayload,
} from "@/lib/record-bridge";
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
export class RecordStepDelivery {
readonly #requestId: string;
readonly #producerId: string;
readonly #send: SendRecordStepMessage;
readonly #pending = new Map<number, RecordStepPayload>();
#nextSequence = 1;
#tail = Promise.resolve();
constructor(
requestId: string,
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
producerId: string = crypto.randomUUID(),
) {
this.#requestId = requestId;
this.#producerId = producerId;
this.#send = send;
}
enqueue(step: RecordStepPayload): void {
const sequence = this.#nextSequence;
this.#nextSequence += 1;
this.#pending.set(sequence, step);
void this.#schedule();
}
async flush(): Promise<boolean> {
await this.#tail;
await this.#schedule();
return this.#pending.size === 0;
}
#schedule(): Promise<void> {
const drain = () => this.#drain();
this.#tail = this.#tail.then(drain, drain);
return this.#tail;
}
async #drain(): Promise<void> {
for (const [sequence, step] of this.#pending) {
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: this.#requestId,
producerId: this.#producerId,
sequence,
step,
};
try {
const response = await this.#send(message);
if (!isAcceptedRecordStepAck(response, sequence)) return;
this.#pending.delete(sequence);
} catch {
return;
}
}
}
}
@@ -1,209 +0,0 @@
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+21 -25
View File
@@ -21,7 +21,6 @@ import {
type OverlayMode, type OverlayMode,
} from "@/lib/overlay-bridge"; } from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup"; import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler"; import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -39,7 +38,6 @@ import {
attachRecordFinishListener, attachRecordFinishListener,
attachRecordQueryListener, attachRecordQueryListener,
attachRecordStepListener, attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record"; } from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake"; import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport"; import type { Transport } from "@/transport/transport";
@@ -147,33 +145,10 @@ export default defineBackground(() => {
void sessionsLive.syncFromManager(); void sessionsLive.syncFromManager();
}, },
}); });
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({ const dispatcher = new ToolDispatcher({
transport, transport,
sessions, sessions,
cdp, cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged, onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed, onBrowserControlResumed,
approveBorrow: (ctx) => approveBorrow: (ctx) =>
@@ -195,6 +170,27 @@ export default defineBackground(() => {
}), }),
}); });
dispatcher.start(); dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener();
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") { if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({ attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked, onClicked: chrome.notifications.onClicked,
+45 -34
View File
@@ -11,6 +11,11 @@ import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline"; import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay"; import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import { import {
type CaptureSuppressAck, type CaptureSuppressAck,
type CaptureSuppressMessage, type CaptureSuppressMessage,
@@ -40,17 +45,11 @@ import {
} from "@/lib/overlay-bridge"; } from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import { import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH, RECORD_FINISH,
RECORD_QUERY, RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse, type RecordQueryResponse,
type RecordStartAck, type RecordStartAck,
type RecordStartMessage,
type RecordStopAck, type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge"; } from "@/lib/record-bridge";
import type { import type {
BorrowCancelMessage, BorrowCancelMessage,
@@ -70,6 +69,7 @@ export default defineContentScript({
if (window.top !== window) return; if (window.top !== window) return;
const overlays = new OverlayController(); const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null; let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null; let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null; let overlayHost: HTMLElement | null = null;
@@ -228,6 +228,8 @@ export default defineContentScript({
if (previousHelp) { if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled"); void sendHelpFinish(previousHelp.id, "cancelled");
} }
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null; activeRecordRequestId = null;
renderAll(); renderAll();
} }
@@ -257,9 +259,6 @@ export default defineContentScript({
| HelpRequestMessage | HelpRequestMessage
| HelpCancelMessage | HelpCancelMessage
| CaptureSuppressMessage | CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage | OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage | OverlayAgentStateMessage
| OverlayAutomationBypassMessage, | OverlayAutomationBypassMessage,
@@ -270,32 +269,41 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse); return captureSuppress.handleMessage(message, sendResponse);
} }
if (isRecordStartMessage(message)) { if (isRecordContentMessage(message)) {
activeRecordRequestId = message.requestId; const needsAsync = handleRecordContentMessage(
overlays.setAgentRecordRequest({ message,
id: message.requestId, {
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}), activeRequestId: activeRecordRequestId,
onFinish: () => { capture: recordCapture,
void chrome.runtime.sendMessage({ setActiveRequestId: (id) => {
type: RECORD_FINISH, activeRecordRequestId = id;
requestId: message.requestId, },
}); setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
}, },
}); sendResponse as unknown as
renderAll(); | ((response: RecordStartAck | RecordStopAck) => void)
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true }); | undefined,
return false; );
} return needsAsync;
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
} }
if ( if (
@@ -518,6 +526,9 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage); chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange); chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow); window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null; activeRecordRequestId = null;
}); });
}, },
@@ -200,9 +200,6 @@ describe("App", () => {
expect(copied).toContain("--browser 03c3e47f"); expect(copied).toContain("--browser 03c3e47f");
expect(copied).toContain('--purpose "发布 wiki 文档"'); expect(copied).toContain('--purpose "发布 wiki 文档"');
expect(copied).not.toMatch(/bsk record start[^\n]*--url/); expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
expect(copied).toContain("./trace");
expect(copied).toContain("trace.json");
expect(copied).toContain("states/");
// Button label stays static; a transient toast confirms the copy. // Button label stays static; a transient toast confirms the copy.
expect(copyButton.textContent).toContain("复制录制指令"); expect(copyButton.textContent).toContain("复制录制指令");
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制")); await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
@@ -247,41 +244,6 @@ describe("App", () => {
expect(copyButton.getAttribute("disabled")).not.toBeNull(); expect(copyButton.getAttribute("disabled")).not.toBeNull();
expect(screen.getByText("连接后可用")).toBeTruthy(); expect(screen.getByText("连接后可用")).toBeTruthy();
}); });
it("treats protocol drift as still connected and prompts an upgrade", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
state: "version_skew",
instanceId: "03c3e47f",
handshake: {
server: "bh",
version: mockDaemonVersion,
protocol_version: "1.0",
},
},
statusState: "version_skew",
setLabel,
setConnectionEnabled,
});
render(<App />);
expect(screen.getByText("已连接")).toBeTruthy();
expect(screen.getByText("可升级")).toBeTruthy();
expect(screen.queryByText("协议不一致")).toBeNull();
expect(screen.queryByText("Action needed")).toBeNull();
expect(screen.queryByText("兼容")).toBeNull();
const warning = screen.getByText(/协议版本不同,请及时升级/);
expect(warning.textContent).toContain("CLI 协议");
expect(warning.textContent).toContain("扩展协议");
openRecordView();
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
expect(copyButton.getAttribute("disabled")).toBeNull();
fireEvent.click(copyButton);
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
});
}); });
describe("control hints toggle", () => { describe("control hints toggle", () => {
+3 -4
View File
@@ -79,7 +79,6 @@ export function App() {
}, [copiedTick]); }, [copiedTick]);
const isSkewed = statusState === "version_skew"; const isSkewed = statusState === "version_skew";
const connectionLive = statusState === "connected" || isSkewed;
const daemonVersion = snapshot.handshake?.version ?? "—"; const daemonVersion = snapshot.handshake?.version ?? "—";
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—"; const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—"; const extensionVersion = snapshot.extensionVersion || "—";
@@ -91,7 +90,7 @@ export function App() {
setCopiedInstanceId(true); setCopiedInstanceId(true);
}; };
const recordReady = connectionLive && Boolean(snapshot.instanceId); const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
const recordPurpose = purposeDraft.trim(); const recordPurpose = purposeDraft.trim();
const recordStartUrl = startUrlDraft.trim(); const recordStartUrl = startUrlDraft.trim();
const recordCommand = snapshot.instanceId const recordCommand = snapshot.instanceId
@@ -207,12 +206,12 @@ export function App() {
</div> </div>
{isSkewed && ( {isSkewed && (
<p <p
className="mt-2 text-xs leading-snug text-muted-foreground" className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
data-slot="popup-version-skew-warning" data-slot="popup-version-skew-warning"
> >
{t("popup.versionSkewWarning", { {t("popup.versionSkewWarning", {
extensionProtocol: PROTOCOL_VERSION, extensionProtocol: PROTOCOL_VERSION,
cliProtocol: daemonProtocol, daemonProtocol,
})} })}
</p> </p>
)} )}
@@ -1,14 +0,0 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -26,20 +26,20 @@ function handshake(
} }
describe("computeConnectedState (protocol-based compat)", () => { describe("computeConnectedState (protocol-based compat)", () => {
it("returns connected when daemon protocol equals extension protocol", () => { it("returns connected when protocol strings match", () => {
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({ expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
kind: "connected", kind: "connected",
}); });
}); });
it("returns version_skew when daemon protocol minor is newer", () => { it("returns version_skew when daemon protocol minor is newer", () => {
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({ expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
kind: "version_skew", kind: "version_skew",
}); });
}); });
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => { it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({ expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
kind: "version_skew", kind: "version_skew",
}); });
}); });
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
}); });
it("rejects when extension is below daemon min_compatible_protocol", () => { it("rejects when extension is below daemon min_compatible_protocol", () => {
const result = computeConnectedState(handshake("1.1", "1.5")); const result = computeConnectedState(handshake("1.0", "1.5"));
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("min_compatible_protocol"); expect(result.reason).toContain("min_compatible_protocol");
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState({ const result = computeConnectedState({
server: "browser-skill-daemon", server: "browser-skill-daemon",
version: "0.1.0", version: "0.1.0",
protocol_version: "1.1", protocol_version: "1.0",
min_compatible_peer: "0.1.0", min_compatible_peer: "0.1.0",
}); });
expect(result).toEqual({ kind: "connected" }); expect(result).toEqual({ kind: "connected" });
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
}); });
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => { it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
const result = computeConnectedState(handshake("1.1", "not-a-protocol")); const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
expect(result.kind).toBe("rejected"); expect(result.kind).toBe("rejected");
if (result.kind === "rejected") { if (result.kind === "rejected") {
expect(result.reason).toContain("daemon min_compatible_protocol"); expect(result.reason).toContain("daemon min_compatible_protocol");
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
const second = transport.send.mock.calls[1]?.[0] as { id: string }; const second = transport.send.mock.calls[1]?.[0] as { id: string };
expect(second.id).not.toBe(first.id); expect(second.id).not.toBe(first.id);
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") }); transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
await Promise.resolve(); await Promise.resolve();
expect(controller.snapshot().state).not.toBe("connected"); expect(controller.snapshot().state).not.toBe("connected");
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") }); transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected")); await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
}); });
}); });
@@ -1,32 +0,0 @@
import { describe, expect, it } from "vitest";
import {
isAcceptedRecordStepAck,
isRecordStepMessage,
RECORD_STEP,
type RecordStepMessage,
} from "../record-bridge";
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: "rec-1",
producerId: "document-1",
sequence: 1,
step: { op: "click", target: { tag: "button", name: "Submit" } },
};
describe("record step delivery protocol", () => {
it("requires a producer and a positive integer sequence", () => {
expect(isRecordStepMessage(message)).toBe(true);
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
});
it("accepts only the acknowledgement for the sent sequence", () => {
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
false,
);
});
});
@@ -1,213 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -1,70 +0,0 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -11,7 +11,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues }); const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({ const state = session.registry.register({
url: URL, url: URL,
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]', rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
}); });
session.cursor.lastSettled = { session.cursor.lastSettled = {
stateId: state.id, stateId: state.id,
@@ -34,7 +34,7 @@ function sessionWithInput(redactValues = false): RecordingObservationSession {
return session; return session;
} }
function finalizedFillTrace(value: string, redactValues: boolean) { function finalizedFillBody(value: string, redactValues: boolean): string {
const session = sessionWithInput(redactValues); const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = { const draft: RecordingDraftStep = {
op: "fill", op: "fill",
@@ -53,20 +53,15 @@ function finalizedFillTrace(value: string, redactValues: boolean) {
startedAt: "2026-08-12T00:00:00.000Z", startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish", stoppedBy: "user_finish",
bskVersion: "test", bskVersion: "test",
redactValues, }).states[0]!.body;
});
}
function finalizedFillBody(value: string, redactValues: boolean): string {
return finalizedFillTrace(value, redactValues).states[0]!.body;
} }
describe("record observation annotations", () => { describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => { it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private"; const secret = "hunter2-private";
const dumped = JSON.stringify(finalizedFillTrace(secret, true)); const body = finalizedFillBody(secret, true);
expect(dumped).toContain("step 1: fill"); expect(body).toContain("step 1: fill");
expect(dumped).not.toContain(secret); expect(body).not.toContain(secret);
}); });
it("keeps ordinary fill details", () => { it("keeps ordinary fill details", () => {
@@ -78,7 +73,7 @@ describe("record observation annotations", () => {
const state = registry.register({ const state = registry.register({
url: "https://example.com/a\nb", url: "https://example.com/a\nb",
title: "hello\nworld", title: "hello\nworld",
vomText: "@vom 1", rawVomText: "@vom 1",
}); });
const trace = buildTraceV3({ const trace = buildTraceV3({
registry, registry,
@@ -96,18 +91,18 @@ describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => { it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry(); const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry(); const second = new RecordingStateRegistry();
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1"); expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1"); expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1"); expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
}); });
it("enriches metadata when a deduplicated observation becomes more complete", () => { it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry(); const registry = new RecordingStateRegistry();
registry.register({ url: URL, vomText: "same" }); registry.register({ url: URL, rawVomText: "same" });
const state = registry.register({ const state = registry.register({
url: URL, url: URL,
title: "Login", title: "Login",
vomText: "same", rawVomText: "same",
truncated: true, truncated: true,
}); });
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true }); expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
@@ -1,289 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
};
}
function runtime(): RecordingObservationRuntime {
return new RecordingObservationRuntime({
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
});
}
describe("RecordingObservationRuntime", () => {
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
captureRecordingObservation.mockReturnValueOnce(
new Promise((resolve) => {
release = resolve;
}),
);
const recording = runtime();
const first = recording.captureInitial(7);
const second = recording.captureInitial(7);
const flushed = recording.flush();
let flushCompleted = false;
void flushed.then(() => {
flushCompleted = true;
});
await Promise.resolve();
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
expect(flushCompleted).toBe(false);
release(observation());
await Promise.all([first, second, flushed]);
expect(flushCompleted).toBe(true);
});
it("cancels an in-flight initial capture", async () => {
captureRecordingObservation.mockImplementationOnce(
(input: { signal?: AbortSignal }) =>
new Promise((_, reject) => {
input.signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
}),
);
const recording = runtime();
const pending = recording.captureInitial(7);
recording.cancel();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
});
it("retries a failed initial capture when the recording settles at stop", async () => {
captureRecordingObservation
.mockRejectedValueOnce(new Error("document swapped"))
.mockResolvedValueOnce(observation());
const recording = runtime();
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
await recording.settleTrailing(7, []);
const trace = recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
});
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
expect(trace.states).toHaveLength(1);
});
it("keeps runtime matching data out of the serialized trace", async () => {
const captured = observation();
captured.index = new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "button",
rect: { x: 10, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
});
captureRecordingObservation.mockResolvedValueOnce(captured);
const recording = runtime();
await recording.captureInitial(7);
const dumped = JSON.stringify(
recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
}),
);
expect(dumped).toContain("RootWebArea");
expect(dumped).not.toContain("matchNodes");
expect(dumped).not.toContain("localRect");
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s
describe("recording-step-buffer", () => { describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => { it("stores semantic click without summary", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } }; const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, { appendRecordedPayload(buffer, {
op: "click", op: "click",
target: { tag: "button", role: "button", name: "发布" }, target: { tag: "button", role: "button", name: "发布" },
@@ -15,40 +15,7 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" }, captureTarget: { tag: "button", role: "button", name: "发布" },
}, },
]); ]);
expect(buffer.navigation.pendingNavigation).toBe(true); expect(buffer.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
op: "hover",
target: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
{
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
);
expect(buffer.steps).toEqual([
{
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
targetHint: {
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
},
]);
}); });
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => { it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
@@ -59,11 +26,9 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" }, captureTarget: { tag: "button", role: "button", name: "发布" },
}, },
], ],
navigation: { currentUrl: "https://example.com/a",
currentUrl: "https://example.com/a", pendingNavigation: true,
pendingNavigation: true, pendingNavigationDeadline: Date.now() + 5_000,
pendingNavigationDeadline: Date.now() + 5_000,
},
}; };
observeRecordedNavigation(buffer, "https://example.com/b", true); observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
@@ -85,11 +50,9 @@ describe("recording-step-buffer", () => {
values: ["tech"], values: ["tech"],
}, },
], ],
navigation: { currentUrl: "https://example.com/list",
currentUrl: "https://example.com/list", pendingNavigation: true,
pendingNavigation: true, pendingNavigationDeadline: Date.now() + 5_000,
pendingNavigationDeadline: Date.now() + 5_000,
},
}; };
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true); observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([ expect(buffer.steps).toEqual([
@@ -105,60 +68,13 @@ describe("recording-step-buffer", () => {
it("emits navigate for uncaused URL changes", () => { it("emits navigate for uncaused URL changes", () => {
const buffer = { const buffer = {
steps: [], steps: [],
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false }, currentUrl: "https://example.com/a",
pendingNavigation: false,
}; };
const result = observeRecordedNavigation(buffer, "https://example.com/b", false); observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({ expect(buffer.steps[0]).toMatchObject({
op: "navigate", op: "navigate",
url: "https://example.com/b", url: "https://example.com/b",
}); });
}); });
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
"https://passport.example/callback",
false,
"link",
["server_redirect"],
);
expect(hop1).toEqual({
kind: "coalesce_redirect",
url: "https://passport.example/callback",
});
expect(buffer.steps).toEqual([]);
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
"client_redirect",
]);
expect(hop2).toEqual({
kind: "coalesce_redirect",
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
});
}); });
@@ -1,39 +0,0 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -69,23 +69,6 @@ describe("matchObservationTarget", () => {
expect(target.ref).toBe("e2"); expect(target.ref).toBe("e2");
}); });
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => { it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({ const target = matchObservationTarget({
observation: observation( observation: observation(
@@ -98,21 +81,6 @@ describe("matchObservationTarget", () => {
expect(target).toEqual({ name: "发布", unmatched: true }); expect(target).toEqual({ name: "发布", unmatched: true });
}); });
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => { it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({ const target = matchObservationTarget({
observation: observation( observation: observation(
@@ -27,17 +27,6 @@ describe("shouldRecordPress", () => {
}); });
describe("reduceTraceSteps", () => { describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => { it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [ const drafts: RecordingDraftStep[] = [
{ {
@@ -171,7 +160,7 @@ describe("reduceTraceSteps", () => {
}); });
}); });
it("drops hover steps, which only exist in trace v3", () => { it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps( const { steps } = reduceTraceSteps(
[ [
{ {
@@ -187,7 +176,12 @@ describe("reduceTraceSteps", () => {
], ],
"https://example.com/app", "https://example.com/app",
); );
expect(steps.map((s) => s.op)).toEqual(["click"]); expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
}); });
it("resolveTraceStartUrl prefers explicit start URL", () => { it("resolveTraceStartUrl prefers explicit start URL", () => {
@@ -6,76 +6,6 @@ import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types"; import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => { describe("trace reducer v3", () => {
it("removes form literals when value redaction is requested", () => {
const reduced = reduceTraceStepsV3(
[
{
op: "fill",
value: "private@example.com",
preStateId: "s1",
postStateId: "s1",
},
{
op: "select",
values: ["private-account-id"],
labels: ["Private account"],
preStateId: "s1",
postStateId: "s1",
},
],
{ redactValues: true },
);
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
expect(reduced.steps[1]).toMatchObject({ op: "select" });
expect(reduced.steps[1]).not.toHaveProperty("selection");
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => { it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [ const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" }, { op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
@@ -106,7 +36,7 @@ describe("trace reducer v3", () => {
it("builds the wire model from protocol constants", () => { it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry(); const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" }); const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
const trace = buildTraceV3({ const trace = buildTraceV3({
registry, registry,
drafts: [ drafts: [
+1 -37
View File
@@ -16,10 +16,6 @@ export interface RecordStartAck {
ok: true; ok: true;
} }
export type RecordStepAck =
| { ok: true; sequence: number }
| { ok: false; expectedSequence: number; error: string };
export type RecordStopAck = { ok: true } | { ok: false; error: string }; export type RecordStopAck = { ok: true } | { ok: false; error: string };
export interface RecordQueryMessage { export interface RecordQueryMessage {
@@ -54,30 +50,17 @@ export interface RecordStepPayload {
labels?: string[]; labels?: string[];
url?: string; url?: string;
redacted?: boolean; redacted?: boolean;
commit?: "enter" | "suggestion" | "blur";
/** Page URL when the step was captured. */ /** Page URL when the step was captured. */
page_url?: string; page_url?: string;
/** Event-target bounds in the source frame's viewport coordinate space. */
geometry?: {
rect: { x: number; y: number; w: number; h: number };
tag: string;
};
/** Capture-only hint; never persisted unless converted to navigated_to. */ /** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean; expects_navigation?: boolean;
/** Whether an observed URL change was synchronously caused by the action. */ /** Whether an observed URL change was synchronously caused by the action. */
navigation_caused_by_action?: boolean; navigation_caused_by_action?: boolean;
/** Raw webNavigation transition metadata for cause mapping. */
transitionType?: string;
transitionQualifiers?: string[];
} }
export interface RecordStepMessage { export interface RecordStepMessage {
type: typeof RECORD_STEP; type: typeof RECORD_STEP;
requestId: string; requestId: string;
/** Stable for the lifetime of one content-script document. */
producerId: string;
/** Monotonic within one content-script producer and recording request. */
sequence: number;
step: RecordStepPayload; step: RecordStepPayload;
} }
@@ -129,27 +112,8 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage { export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false; if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>; const m = msg as Record<string, unknown>;
if ( if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
m.type !== RECORD_STEP ||
typeof m.requestId !== "string" ||
typeof m.producerId !== "string" ||
m.producerId.length === 0 ||
typeof m.sequence !== "number" ||
!Number.isSafeInteger(m.sequence) ||
m.sequence < 1
) {
return false;
}
const step = m.step; const step = m.step;
if (typeof step !== "object" || step === null) return false; if (typeof step !== "object" || step === null) return false;
return typeof (step as RecordStepPayload).op === "string"; return typeof (step as RecordStepPayload).op === "string";
} }
export function isAcceptedRecordStepAck(
value: unknown,
sequence: number,
): value is Extract<RecordStepAck, { ok: true }> {
if (typeof value !== "object" || value === null) return false;
const ack = value as Partial<RecordStepAck>;
return ack.ok === true && ack.sequence === sequence;
}
@@ -1,39 +0,0 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -1,65 +0,0 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -1,399 +0,0 @@
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -1,5 +0,0 @@
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
@@ -1,5 +1,4 @@
import type { RenderedRef } from "@browser-skill/vom"; import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { import {
type CaptureVomMatchNode, type CaptureVomMatchNode,
type CaptureVomObservationResult, type CaptureVomObservationResult,
@@ -29,11 +28,6 @@ export interface RegisteredObservation {
url: string; url: string;
} }
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string { function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`; return `${frameId}:${backendNodeId}`;
} }
@@ -46,12 +40,8 @@ export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>(); readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>(); readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>(); readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor( constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>(); const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) { for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId; const frameId = ref.frameId ?? input.rootFrameId;
@@ -61,16 +51,6 @@ export class ObservationNodeIndex {
frameRefs.push(ref); frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs); this.#refsByFrame.set(frameId, frameRefs);
} }
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) { for (const geometry of input.matchNodes) {
const { frameId } = geometry; const { frameId } = geometry;
const entry = { const entry = {
@@ -96,10 +76,6 @@ export class ObservationNodeIndex {
refs(frameId: string): readonly RenderedRef[] { refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? []; return this.#refsByFrame.get(frameId) ?? [];
} }
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
} }
async function readTabMeta( async function readTabMeta(
@@ -28,13 +28,7 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
} }
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft { function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return ( return draft.op !== "navigate" && draft.op !== "scroll";
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
} }
export class RecordingObservationSession { export class RecordingObservationSession {
@@ -80,7 +74,7 @@ export class RecordingObservationSession {
const state = this.registry.register({ const state = this.registry.register({
url: captured.url, url: captured.url,
title: captured.title, title: captured.title,
vomText: captured.vomText, rawVomText: captured.vomText,
truncated: captured.truncated, truncated: captured.truncated,
}); });
const observation: RegisteredObservation = { const observation: RegisteredObservation = {
@@ -1,277 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
import { buildTraceV3 } from "./trace-builder-v3";
import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
export class RecordingObservationRuntime {
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #registry = new RecordingStateRegistry();
readonly #annotations: StepAnnotation[] = [];
readonly #contexts = new Map<number, TabRecordingContext>();
readonly #maxTokens?: number;
readonly #redactValues: boolean;
constructor(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
maxTokens?: number;
redactValues?: boolean;
}) {
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#maxTokens = input.maxTokens;
this.#redactValues = input.redactValues ?? false;
}
#context(tabId: number): TabRecordingContext {
const existing = this.#contexts.get(tabId);
if (existing) return existing;
const session = new RecordingObservationSession({
registry: this.#registry,
annotations: this.#annotations,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
});
const context: TabRecordingContext = {
session,
settle: new SettleController({
session,
cdp: this.#cdp,
tabsApi: this.#tabsApi,
tabId,
}),
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
const context = this.#context(tabId);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
scheduleSettle(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
): void {
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
}
clearRedirect(tabId: number): void {
this.#contexts.get(tabId)?.settle.clearRedirect();
}
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// The trace may remain observation-free when CDP is unavailable.
}
}
await context.settle.settleTrailing(drafts);
inferMissingPostStates(drafts);
}
cancel(): void {
for (const context of this.#contexts.values()) {
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
buildTrace(input: {
drafts: RecordingDraftStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
drafts: input.drafts,
annotations: this.#annotations,
startedAt: input.startedAt,
purpose: input.purpose,
startUrl: input.startUrl,
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
@@ -212,7 +212,6 @@ export class SettleController {
url: finalUrl, url: finalUrl,
pageUrl: finalUrl, pageUrl: finalUrl,
cause: "browser", cause: "browser",
transitionQualifiers: ["server_redirect"],
preStateId: this.#session.cursor.lastSettled?.stateId, preStateId: this.#session.cursor.lastSettled?.stateId,
}; };
drafts.push(draft); drafts.push(draft);
@@ -2,8 +2,7 @@ export interface RecordedStateEntry {
id: string; id: string;
url: string; url: string;
title?: string; title?: string;
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */ rawVomText: string;
vomText: string;
truncated: boolean; truncated: boolean;
stepsHere: number[]; stepsHere: number[];
} }
@@ -20,10 +19,10 @@ export class RecordingStateRegistry {
register(input: { register(input: {
url: string; url: string;
title?: string; title?: string;
vomText: string; rawVomText: string;
truncated?: boolean; truncated?: boolean;
}): RecordedStateEntry { }): RecordedStateEntry {
const identity = stateIdentity(input.url, input.vomText); const identity = stateIdentity(input.url, input.rawVomText);
const existingId = this.#idByIdentity.get(identity); const existingId = this.#idByIdentity.get(identity);
if (existingId) { if (existingId) {
const existing = this.#entriesById.get(existingId)!; const existing = this.#entriesById.get(existingId)!;
@@ -36,7 +35,7 @@ export class RecordingStateRegistry {
id: `s${this.#nextId}`, id: `s${this.#nextId}`,
url: input.url, url: input.url,
...(input.title ? { title: input.title } : {}), ...(input.title ? { title: input.title } : {}),
vomText: input.vomText, rawVomText: input.rawVomText,
truncated: input.truncated ?? false, truncated: input.truncated ?? false,
stepsHere: [], stepsHere: [],
}; };
+56 -75
View File
@@ -1,13 +1,8 @@
import type { RecordStepPayload } from "../record-bridge"; import type { RecordStepPayload } from "../record-bridge";
import { hasRedirectQualifier } from "./navigation-policy"; import type { RecordingDraftStep } from "./types";
import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer { export interface RecordingStepBuffer {
steps: RecordingDraftStep[]; steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string; currentUrl?: string;
pendingNavigation: boolean; pendingNavigation: boolean;
pendingNavigationDeadline?: number; pendingNavigationDeadline?: number;
@@ -15,29 +10,33 @@ export interface RecordingNavigationCursor {
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000; const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep( function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): RecordingDraftStep | null {
const pageUrl = payload.page_url; const pageUrl = payload.page_url;
const common = {
...(pageUrl ? { pageUrl } : {}),
...(targetHint ? { targetHint } : {}),
};
switch (payload.op) { switch (payload.op) {
case "click": case "click":
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null; return payload.target
? {
op: "click",
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "hover": case "hover":
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null; return payload.target
? {
op: "hover",
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "fill": case "fill":
return payload.target return payload.target
? { ? {
op: "fill", op: "fill",
captureTarget: payload.target, captureTarget: payload.target,
value: payload.value ?? "", value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}), ...(payload.redacted ? { redacted: true } : {}),
...common, ...(pageUrl ? { pageUrl } : {}),
} }
: null; : null;
case "press": case "press":
@@ -47,7 +46,7 @@ function toDraftStep(
key: payload.key, key: payload.key,
...(payload.target ? { captureTarget: payload.target } : {}), ...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}), ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...common, ...(pageUrl ? { pageUrl } : {}),
} }
: null; : null;
case "select": case "select":
@@ -57,7 +56,7 @@ function toDraftStep(
captureTarget: payload.target, captureTarget: payload.target,
values: payload.values, values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}), ...(payload.labels?.length ? { labels: payload.labels } : {}),
...common, ...(pageUrl ? { pageUrl } : {}),
} }
: null; : null;
case "navigate": case "navigate":
@@ -65,91 +64,73 @@ function toDraftStep(
} }
} }
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number { function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) { for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[index]; const step = buffer.steps[i];
if (!step) continue; if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") { if (step.op === "click" || step.op === "press" || step.op === "select") {
step.navigatedTo = url; buffer.steps[i] = { ...step, navigatedTo: url };
return index; return true;
} }
break; break;
} }
return -1; return false;
} }
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation( export function observeRecordedNavigation(
buffer: RecordingStepBuffer, buffer: RecordingStepBuffer,
url: string, url: string,
causedByAction?: boolean, causedByAction?: boolean,
transitionType?: string, ): void {
transitionQualifiers?: string[], if (!url || url === buffer.currentUrl) return;
): NavigationObserveResult { buffer.currentUrl = url;
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent = const pendingIsCurrent =
navigation.pendingNavigation && buffer.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined || (buffer.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now()); buffer.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) { if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
navigation.pendingNavigation = false; buffer.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined; buffer.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url); if (!annotateLastStepNavigation(buffer, url)) {
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex }; buffer.steps.push({
} else { op: "navigate",
navigation.pendingNavigation = false; url,
navigation.pendingNavigationDeadline = undefined; pageUrl: url,
});
}
return;
} }
if (hasRedirectQualifier(transitionQualifiers)) { buffer.pendingNavigation = false;
return { kind: "coalesce_redirect", url }; buffer.pendingNavigationDeadline = undefined;
}
buffer.steps.push({ buffer.steps.push({
op: "navigate", op: "navigate",
url, url,
pageUrl: url, pageUrl: url,
transitionType,
transitionQualifiers,
}); });
return { kind: "appended", index: buffer.steps.length - 1 };
} }
export function appendRecordedPayload( export function appendRecordedPayload(
buffer: RecordingStepBuffer, buffer: RecordingStepBuffer,
payload: RecordStepPayload, payload: RecordStepPayload,
targetHint?: TargetMatchHint, ): void {
): number | null {
if (payload.op === "navigate") { if (payload.op === "navigate") {
if (!payload.url) return null; if (payload.url) {
const result = observeRecordedNavigation( observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
buffer, }
payload.url, return;
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
} }
const step = toDraftStep( const step = toDraftStep({
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl }, ...payload,
targetHint, page_url: payload.page_url ?? buffer.currentUrl,
); });
if (!step) return null; if (!step) return;
buffer.steps.push(step); buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") { if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.navigation.pendingNavigation = payload.expects_navigation === true; buffer.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation buffer.pendingNavigationDeadline = buffer.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS ? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined; : undefined;
} }
return buffer.steps.length - 1;
} }
@@ -1,61 +0,0 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -14,13 +14,8 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h); return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
} }
function candidateMatches( function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
candidate: IndexedObservationNode, return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
} }
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
@@ -58,17 +53,12 @@ export function matchObservationTarget(input: {
hint?: TargetMatchHint; hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor; fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 { }): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId; const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry; const geometry = input.hint?.geometry;
if (geometry) { if (geometry) {
const matches = input.observation.index const matches = input.observation.index
.candidates(frameId, geometry.tag) .candidates(frameId, geometry.tag)
.filter( .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!); if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter( const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback), (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
@@ -39,13 +39,8 @@ export function buildTraceV3(input: {
startUrl?: string; startUrl?: string;
stoppedBy: StopReason; stoppedBy: StopReason;
bskVersion: string; bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 { }): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, { const reduced = reduceTraceStepsV3(input.drafts);
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps); const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`])); const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>(); const annotationsByState = new Map<string, StepAnnotation[]>();
@@ -70,7 +65,7 @@ export function buildTraceV3(input: {
url: entry.url, url: entry.url,
title: entry.title, title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId), stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.vomText, vomText: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [], annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId, stepIdByDraftId: reduced.stepIdByDraftId,
}), }),
@@ -119,6 +119,14 @@ function toV2Step(
}, },
effectForNavigation(step.navigatedTo, urlToId), effectForNavigation(step.navigatedTo, urlToId),
); );
case "hover":
if (!step.captureTarget) return null;
return {
op: "hover",
id,
page,
target: step.captureTarget,
};
case "fill": case "fill":
if (!step.captureTarget) return null; if (!step.captureTarget) return null;
return { return {
@@ -153,11 +161,7 @@ function toV2Step(
}, },
effectForNavigation(step.navigatedTo, urlToId), effectForNavigation(step.navigatedTo, urlToId),
); );
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll": case "scroll":
case "switch_tab":
return null; return null;
} }
} }
@@ -1,6 +1,5 @@
import type { NavigationCause, StepV3 } from "@/transport/types"; import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy"; import { shouldIncludeDraft } from "./draft-policy";
import { hasRedirectQualifier } from "./navigation-policy";
import { unmatchedTarget } from "./target-matcher"; import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types"; import type { RecordingDraftStep } from "./types";
@@ -9,6 +8,7 @@ interface CollapsedDraft {
draftIds: number[]; draftIds: number[];
} }
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_CAUSES: Record<string, NavigationCause> = { const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed", typed: "user_typed",
generated: "user_typed", generated: "user_typed",
@@ -22,7 +22,7 @@ const TRANSITION_CAUSES: Record<string, NavigationCause> = {
}; };
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean { function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return hasRedirectQualifier(step.transitionQualifiers); return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
} }
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] { function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
@@ -58,26 +58,8 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
})); }));
} }
interface ReduceDraftOptions { function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null; if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId; const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId; const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null; if (!state || !resultState) return null;
@@ -99,14 +81,13 @@ function reduceDraft(
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
}; };
case "fill": case "fill":
const fillIsRedacted = options.redactValues || draft.redacted === true;
return { return {
op: "fill", op: "fill",
...common, ...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: fillIsRedacted ? "***" : draft.value, value: draft.value,
commit: draft.commit ?? "blur", commit: draft.commit ?? "blur",
...(fillIsRedacted ? { redacted: true } : {}), ...(draft.redacted ? { redacted: true } : {}),
}; };
case "press": case "press":
return { return {
@@ -123,7 +104,7 @@ function reduceDraft(
op: "select", op: "select",
...common, ...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}), selection: selection(draft.values, draft.labels),
}; };
case "scroll": case "scroll":
return { op: "scroll", ...common }; return { op: "scroll", ...common };
@@ -135,17 +116,11 @@ export interface ReducedTraceV3 {
stepIdByDraftId: Map<number, number>; stepIdByDraftId: Map<number, number>;
} }
export function reduceTraceStepsV3( export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
steps: RecordingDraftStep[],
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = []; const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>(); const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) { for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, { const step = reduceDraft(draft, output.length + 1);
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue; if (!step) continue;
output.push(step); output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id); for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
+2 -4
View File
@@ -15,9 +15,8 @@ export interface TargetGeometry {
export interface TargetMatchHint { export interface TargetMatchHint {
geometry?: TargetGeometry; geometry?: TargetGeometry;
/** Missing means top frame; null means the source Document could not be resolved. */ /** Missing means the current top frame, never an unrestricted frame search. */
frameId?: string | null; frameId?: string;
geometrySpace?: "top" | "local";
} }
export interface StepAnnotation { export interface StepAnnotation {
@@ -70,7 +69,6 @@ export type RecordingDraftStep =
DraftTarget & DraftTarget &
DraftNavigationEffect) DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink) | ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({ | ({
op: "navigate"; op: "navigate";
url: string; url: string;
@@ -1,22 +0,0 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -8,7 +8,6 @@ import type {
RequestFrame, RequestFrame,
} from "@/transport/types"; } from "@/transport/types";
import { ToolDispatcher } from "../dispatcher"; import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>; type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -45,72 +44,9 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => { describe("ToolDispatcher", () => {
afterEach(() => { afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals(); vi.unstubAllGlobals();
}); });
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => { it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport(); const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({ const sessions = new SessionManager({
@@ -1656,7 +1656,7 @@ describe("buildVomScene", () => {
const scene = buildVomScene(axNodes, captured); const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }), expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
); );
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"'); expect(rendered.text).toContain('@e1 button "close"');
@@ -1732,7 +1732,7 @@ describe("buildVomScene", () => {
}; };
const scene = buildVomScene(axNodes, captured); const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined(); expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
}); });
@@ -1801,9 +1801,9 @@ describe("buildVomScene", () => {
const scene = buildVomScene(axNodes, captured); const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual( expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }), expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
); );
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined(); expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
@@ -2131,7 +2131,7 @@ describe("buildVomScene", () => {
}); });
expect(scene.surfaces).toEqual([ expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
]); ]);
expect(renderVom(scene).text).toContain( expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]', '@e1 button "image" [hover first: My profile | Sign out]',
@@ -3155,98 +3155,6 @@ describe("handleSnapshot", () => {
}); });
}); });
it("decorates live observations with active controlled content", async () => {
const strings = [
"html",
"body",
"button",
"div",
"#text",
"role",
"tab",
"aria-selected",
"true",
"aria-controls",
"reviews-panel",
"id",
"static",
"auto",
"visible",
"1",
"A detailed review",
];
const index = (value: string) => strings.indexOf(value);
const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")];
const snapshot = {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [
[],
[],
[
index("role"),
index("tab"),
index("aria-selected"),
index("true"),
index("aria-controls"),
index("reviews-panel"),
],
[index("id"), index("reviews-panel")],
[],
],
nodeValue: [-1, -1, -1, -1, index("A detailed review")],
},
layout: {
nodeIndex: [0, 1, 2, 3, 4],
styles: [style, style, style, style, style],
bounds: [
[0, 0, 1000, 800],
[0, 0, 1000, 800],
[20, 20, 120, 40],
[20, 80, 500, 200],
[20, 80, 200, 20],
],
paintOrders: [0, 0, 1, 1, 1],
},
},
],
};
const ax: CdpAxNode[] = [
{
nodeId: "root",
backendDOMNodeId: 11,
role: { type: "role", value: "RootWebArea" },
childIds: ["reviews"],
},
{
nodeId: "reviews",
parentId: "root",
backendDOMNodeId: 12,
role: { type: "role", value: "tab" },
name: { type: "computedString", value: "Reviews" },
properties: [
{ name: "selected", value: { value: true } },
{ name: "controls", value: { value: "reviews-panel" } },
],
},
];
const result = await captureVomObservation(
makeOverlayDeps(ax, snapshot, VP_METRICS).cdp,
4,
"https://example.com",
);
expect(result.text).toContain('@e1 tab "Reviews"');
expect(result.text).toContain("[§ active: Reviews]");
expect(result.text).toContain("A detailed review");
});
it("does not leak form secrets through the record-safe observation payload", async () => { it("does not leak form secrets through the record-safe observation payload", async () => {
const secrets = { const secrets = {
email: "user@example.com", email: "user@example.com",
File diff suppressed because it is too large. Load diff
+37 -34
View File
@@ -51,12 +51,7 @@ import {
handleScreenshot, handleScreenshot,
handleSnapshot, handleSnapshot,
} from "./observation"; } from "./observation";
import { import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
handleRecordAwait,
handleRecordStart,
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { import {
handleSessionStart, handleSessionStart,
handleSessionStop, handleSessionStop,
@@ -104,7 +99,6 @@ export interface DispatcherDeps {
transport: Transport; transport: Transport;
sessions: SessionManager; sessions: SessionManager;
cdp?: DispatcherCdpRunner; cdp?: DispatcherCdpRunner;
recording?: RecordRuntimeDeps;
/** /**
* Invoked whenever a dispatched RPC may have changed the live * Invoked whenever a dispatched RPC may have changed the live
* session set (currently `tool.session_start` and * session set (currently `tool.session_start` and
@@ -140,7 +134,6 @@ export class ToolDispatcher {
private readonly transport: Transport; private readonly transport: Transport;
private readonly sessions: SessionManager; private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner; private readonly cdp?: DispatcherCdpRunner;
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void; private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void; private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover; private readonly approveBorrow?: BorrowConfirmationApprover;
@@ -160,7 +153,6 @@ export class ToolDispatcher {
this.transport = deps.transport; this.transport = deps.transport;
this.sessions = deps.sessions; this.sessions = deps.sessions;
this.cdp = deps.cdp; this.cdp = deps.cdp;
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged; this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed; this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow; this.approveBorrow = deps.approveBorrow;
@@ -539,26 +531,44 @@ export class ToolDispatcher {
signal, signal,
}); });
case "tool.record_start": case "tool.record_start":
return this.recording return handleRecordStart(this.sessions, req.params as RecordStartParams, {
? handleRecordStart(this.sessions, req.params as RecordStartParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, bypassOverlay: async (tabId, enabled) => {
}) try {
: recordingRuntimeUnavailable(); await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
case "tool.record_stop": case "tool.record_stop":
return this.recording return handleRecordStop(this.sessions, req.params as RecordStopParams, {
? handleRecordStop(this.sessions, req.params as RecordStopParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, bypassOverlay: async (tabId, enabled) => {
}) try {
: recordingRuntimeUnavailable(); await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
case "tool.record_await": case "tool.record_await":
return this.recording return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, { tabsApi: chromeTabsApi,
...this.recording, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal, signal,
}) });
: recordingRuntimeUnavailable();
default: default:
return { return {
code: "unknown_method", code: "unknown_method",
@@ -694,13 +704,6 @@ function isRpcError(v: unknown): v is RpcError {
); );
} }
function recordingRuntimeUnavailable(): RpcError {
return {
code: "protocol_error",
message: "recording runtime is unavailable",
};
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) { switch (req.method) {
case "tool.tab_create": case "tool.tab_create":
File diff suppressed because it is too large. Load diff
+63 -384
View File
@@ -15,37 +15,22 @@ import {
type RecordQueryResponse, type RecordQueryResponse,
type RecordStartAck, type RecordStartAck,
type RecordStartMessage, type RecordStartMessage,
type RecordStepAck,
type RecordStopMessage, type RecordStopMessage,
} from "@/lib/record-bridge"; } from "@/lib/record-bridge";
import { import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
type RecordFrameCoordinator,
type RecordingCaptureScope,
recordFrameCoordinator,
} from "@/lib/recording/frame-coordinator";
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
import {
appendRecordedPayload,
observeRecordedNavigation,
type RecordingStepBuffer,
} from "@/lib/recording/step-buffer";
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2"; import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types"; import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import { EXTENSION_VERSION } from "@/transport/handshake";
import type { import type {
RecordAwaitParams, RecordAwaitParams,
RecordAwaitResult, RecordAwaitResult,
RecordedTrace,
RecordStartParams, RecordStartParams,
RecordStartResult, RecordStartResult,
RecordStopParams, RecordStopParams,
RecordStopResult, RecordStopResult,
RpcError, RpcError,
StopReason, TraceV2,
} from "@/transport/types"; } from "@/transport/types";
import { TRACE_VERSION_V3 } from "@/transport/types";
import { handleNavigate } from "./navigation"; import { handleNavigate } from "./navigation";
import { import {
type CdpRunner, type CdpRunner,
@@ -58,46 +43,21 @@ import {
interface ActiveRecording { interface ActiveRecording {
requestId: string; requestId: string;
tabs: RecordingTabCoordinator; tabId: number;
agentWindowId: number; agentWindowId: number;
startUrl?: string; startUrl?: string;
purpose?: string; purpose?: string;
steps: RecordingDraftStep[]; steps: RecordingDraftStep[];
startedAt: string; startedAt: string;
startedAtMs: number; startedAtMs: number;
traceVersion: 2 | 3; finishPromise: Promise<TraceV2>;
supportsTabSwitchSteps: boolean; resolveFinish: (trace: TraceV2) => void;
finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void;
rejectFinish: (err: Error) => void; rejectFinish: (err: Error) => void;
settled: boolean; settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null; finishing: boolean;
observation: RecordingObservationRuntime | null; currentUrl?: string;
stoppedBy: StopReason; pendingNavigation: boolean;
/** Navigation callbacks tracked from event receipt through action enqueue. */ pendingNavigationDeadline?: number;
navigationCallbacks: Set<Promise<void>>;
/** Synchronous intake gate closed only after finish drains to stability. */
acceptingNavigation: boolean;
/**
* Serializes step appends with navigation observation so a click is always
* in `steps` before a same-turn `webNavigation` tries to annotate it.
*/
actionQueue: Promise<void>;
/** Last accepted sequence for each content-script document producer. */
lastStepSequenceByProducer: Map<string, number>;
}
function enqueueRecordingAction(
recording: ActiveRecording,
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
function isRecordingFinishing(recording: ActiveRecording): boolean {
return recording.settled || recording.finishAttempt !== null;
} }
const recordings = new Map<string, ActiveRecording>(); const recordings = new Map<string, ActiveRecording>();
@@ -118,9 +78,6 @@ function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
/** Recording producer version mirrored into trace.recorder.bsk. */
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */ /** Injectable http(s) landing page when `tool.record_start` omits `url`. */
export const RECORD_DEFAULT_START_URL = "https://example.com/"; export const RECORD_DEFAULT_START_URL = "https://example.com/";
@@ -180,11 +137,9 @@ async function sendRecordStartWithAck(
tabId: number, tabId: number,
msg: RecordStartMessage, msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"], sendToTab: RecordDeps["sendToTab"],
cancelled?: () => boolean,
): Promise<void> { ): Promise<void> {
let lastError: unknown; let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) { for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
if (cancelled?.()) throw new Error("record start cancelled");
try { try {
const response = await sendToTab(tabId, msg); const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return; if (isRecordStartAck(response)) return;
@@ -192,7 +147,6 @@ async function sendRecordStartWithAck(
} catch (err) { } catch (err) {
lastError = err; lastError = err;
} }
if (cancelled?.()) throw new Error("record start cancelled");
if (attempt + 1 < RECORD_START_RETRIES) { if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS); await sleep(RECORD_START_RETRY_DELAY_MS);
} }
@@ -200,30 +154,7 @@ async function sendRecordStartWithAck(
throw lastError ?? new Error("failed to start recording in content script"); throw lastError ?? new Error("failed to start recording in content script");
} }
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError { function buildTrace(recording: ActiveRecording): TraceV2 {
if (params.trace_version === undefined) return 2;
if (params.trace_version === TRACE_VERSION_V3) return 3;
return {
code: "invalid_params",
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
};
}
function buildTrace(recording: ActiveRecording): RecordedTrace {
if (recording.traceVersion === 3 && recording.observation) {
return recording.observation.buildTrace({
drafts: recording.steps,
startedAt: recording.startedAt,
purpose: recording.purpose,
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
});
}
if (recording.traceVersion === 3) {
throw new Error("trace v3 observation runtime is unavailable");
}
return buildTraceV2({ return buildTraceV2({
steps: recording.steps, steps: recording.steps,
startedAt: recording.startedAt, startedAt: recording.startedAt,
@@ -232,51 +163,6 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
}); });
} }
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.commit(targetTabId, targetUrl);
}
async function processRecordedStep(
recording: ActiveRecording,
draftIndex: number,
tabId: number,
producerId?: string,
): Promise<void> {
if (!recording.observation) return;
try {
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
} catch (err) {
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
}
}
export interface RecordDeps { export interface RecordDeps {
tabsApi: ChromeTabsApi; tabsApi: ChromeTabsApi;
sendToTab( sendToTab(
@@ -284,25 +170,16 @@ export interface RecordDeps {
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>; ): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>; bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
frameCoordinator?: Pick<
RecordFrameCoordinator,
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
>;
cdp?: CdpRunner; cdp?: CdpRunner;
signal?: AbortSignal; signal?: AbortSignal;
} }
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
};
let defaultDeps: RecordDeps | null = null; let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps { function getDefaultDeps(): RecordDeps {
if (!defaultDeps) { if (!defaultDeps) {
defaultDeps = { defaultDeps = {
tabsApi: chromeTabsApi, tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
frameCoordinator: recordFrameCoordinator,
}; };
} }
return defaultDeps; return defaultDeps;
@@ -360,58 +237,16 @@ export function releaseBrowserObservationListenersIfIdle(): void {
detachBrowserObservation = null; detachBrowserObservation = null;
} }
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void { export function attachRecordStepListener(): () => void {
const listener = ( const listener = (
message: unknown, message: unknown,
sender: chrome.runtime.MessageSender, _sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordStepAck) => void, _sendResponse: () => void,
) => { ) => {
if (!isRecordStepMessage(message)) return false; if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue; if (recording.requestId !== message.requestId) continue;
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator appendRecordedPayload(recording, message.step);
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
: undefined;
if (deps.frameCoordinator && !source) return false;
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = source
? `${source.tabId}:${source.documentId}:${source.producerId}`
: `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
sendResponse({ ok: true, sequence: message.sequence });
return false;
}
if (message.sequence > expectedSequence) {
sendResponse({
ok: false,
expectedSequence,
error: `out-of-order recorded step ${message.sequence}`,
});
return false;
}
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => {
if (sourceTabId !== recording.tabs.currentTabId) {
if (!sourceWasActive) return;
if (sourceTabId !== recording.tabs.activeTabId)
recording.tabs.noteActivation(sourceTabId);
await activateRecordingTab(recording, sourceTabId);
}
await recording.observation?.flushRedirects(sourceTabId);
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
const draftIndex = appendRecordedPayload(
stepBufferFor(recording, sourceTabId, message.step.page_url),
message.step,
targetHint,
);
if (draftIndex !== null) {
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
}
});
sendResponse({ ok: true, sequence: message.sequence });
return false; return false;
} }
return false; return false;
@@ -438,7 +273,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null { function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording; if (recording.tabId === tabId && !recording.settled) return recording;
} }
return null; return null;
} }
@@ -475,7 +310,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording, recording: ActiveRecording,
deps: RecordDeps, deps: RecordDeps,
): Promise<void> { ): Promise<void> {
clearRearmTimer(recording.tabs.currentTabId); clearRearmTimer(recording.tabId);
try { try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) { for (const tab of tabs) {
@@ -490,16 +325,13 @@ async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording, recording: ActiveRecording,
deps: RecordDeps, deps: RecordDeps,
): Promise<void> { ): Promise<void> {
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
throw new Error("failed to flush one or more recording documents");
}
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabs.currentTabId]; let tabIds = [recording.tabId];
try { try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [ tabIds = [
...new Set([ ...new Set([
recording.tabs.currentTabId, recording.tabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])), ...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]), ]),
]; ];
@@ -510,11 +342,11 @@ async function stopRecordingOnAllAgentTabs(
for (const tabId of tabIds) { for (const tabId of tabIds) {
try { try {
const response = await deps.sendToTab(tabId, stopMsg); const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) { if (tabId === recording.tabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps"); throw new Error("content script did not confirm recorded steps");
} }
} catch { } catch {
if (tabId === recording.tabs.currentTabId) { if (tabId === recording.tabId) {
throw new Error("failed to flush recorded steps"); throw new Error("failed to flush recorded steps");
} }
} }
@@ -532,39 +364,22 @@ async function rearmRecording(
recording: ActiveRecording, recording: ActiveRecording,
targetTabId: number, targetTabId: number,
deps: RecordDeps, deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> { ): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the // Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay // content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set. // already hides the control chrome while activeRecord is set.
const isFinishing = () => isRecordingFinishing(recording);
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
if (isFinishing()) return false;
const startMsg: RecordStartMessage = { const startMsg: RecordStartMessage = {
type: RECORD_START, type: RECORD_START,
requestId: recording.requestId, requestId: recording.requestId,
startedAtMs: recording.startedAtMs, startedAtMs: recording.startedAtMs,
}; };
try { try {
if (deps.frameCoordinator) { await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId); recording.tabId = targetTabId;
if (!frameStarted) throw new Error("recording document did not start");
}
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
if (isFinishing()) return false;
if (activation) {
if (!recording.tabs.isLatest(activation)) return true;
await enqueueRecordingAction(recording, async () => {
if (isFinishing() || !recording.tabs.isLatest(activation)) {
return;
}
await activateRecordingTab(recording, targetTabId);
});
}
return true; return true;
} catch { } catch {
if (isFinishing()) return false;
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS); await sleep(RECORD_REARM_RETRY_DELAY_MS);
} }
@@ -573,7 +388,7 @@ async function rearmRecording(
return false; return false;
} }
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void { function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
const existing = rearmTimers.get(tabId); const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing); if (existing) clearTimeout(existing);
rearmTimers.set( rearmTimers.set(
@@ -582,7 +397,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabAc
rearmTimers.delete(tabId); rearmTimers.delete(tabId);
void (async () => { void (async () => {
const current = await findRecordingForTab(tabId, deps); const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation); if (current) await rearmRecording(current, tabId, deps);
})(); })();
}, RECORD_REARM_DEBOUNCE_MS), }, RECORD_REARM_DEBOUNCE_MS),
); );
@@ -594,20 +409,14 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
const windowId = tab.windowId; const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return; if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) { for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue; if (recording.settled || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps); scheduleRearmForTab(tabId, deps);
return; return;
} }
}; };
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
for (const recording of recordings.values()) { scheduleRearmForTab(activeInfo.tabId, deps);
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
}; };
chrome.tabs.onCreated.addListener(onCreated); chrome.tabs.onCreated.addListener(onCreated);
@@ -619,47 +428,13 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
} }
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void { export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
const observeMainFrame = ( const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
tabId: number, void (async () => {
url?: string, const recording = await findRecordingForTab(tabId, deps);
causedByAction?: boolean, if (recording && url) {
transitionType?: string, observeRecordedNavigation(recording, url, causedByAction);
transitionQualifiers?: string[], }
) => { })();
if (!url) return;
const candidates = [...recordings.values()].filter(
(recording) =>
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
);
for (const recording of candidates) {
const queued = enqueueRecordingAction(recording, async () => {
if (tabId !== recording.tabs.currentTabId) {
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
const tracked = queued.catch(() => {});
recording.navigationCallbacks.add(tracked);
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
}
}; };
const onMainFrameComplete = (tabId: number, url?: string) => { const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => { void (async () => {
@@ -679,13 +454,18 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails, details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
) => { ) => {
if (details.frameId !== 0) return; if (details.frameId !== 0) return;
observeMainFrame( const causedByAction =
details.tabId, details.transitionType === "link" || details.transitionType === "form_submit"
details.url, ? true
undefined, : details.transitionType === "typed" ||
details.transitionType, details.transitionType === "auto_bookmark" ||
details.transitionQualifiers, details.transitionType === "generated" ||
); details.transitionType === "keyword" ||
details.transitionType === "keyword_generated" ||
details.transitionType === "reload"
? false
: undefined;
observeMainFrame(details.tabId, details.url, causedByAction);
}; };
chrome.webNavigation.onCompleted.addListener(completedListener); chrome.webNavigation.onCompleted.addListener(completedListener);
chrome.webNavigation.onCommitted?.addListener(committedListener); chrome.webNavigation.onCommitted?.addListener(committedListener);
@@ -703,26 +483,6 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
return () => chrome.tabs.onUpdated.removeListener(listener); return () => chrome.tabs.onUpdated.removeListener(listener);
} }
const MAX_FINISH_DRAIN_ROUNDS = 10;
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
await Promise.all([...recording.navigationCallbacks]);
const actionTail = recording.actionQueue;
await actionTail;
await recording.observation?.flush();
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
// No event or promise continuation can interleave with this synchronous
// check-and-close, so work accepted before the cutoff is fully drained.
recording.acceptingNavigation = false;
return true;
}
}
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
return false;
}
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void { export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = ( const listener = (
message: unknown, message: unknown,
@@ -741,8 +501,7 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false }); sendResponse({ active: false });
return; return;
} }
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined; await rearmRecording(recording, tabId, deps);
await rearmRecording(recording, tabId, deps, activation);
sendResponse({ sendResponse({
active: true, active: true,
requestId: recording.requestId, requestId: recording.requestId,
@@ -764,50 +523,23 @@ async function finishRecordingByRequest(
if (recording.requestId !== requestId || recording.settled) continue; if (recording.requestId !== requestId || recording.settled) continue;
const match = await findRecordingForTab(tabId, deps); const match = await findRecordingForTab(tabId, deps);
if (match !== recording) continue; if (match !== recording) continue;
await finishRecording(sessionId, deps, "user_finish"); await finishRecording(sessionId, deps);
return; return;
} }
} }
async function finishRecording( async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
sessionId: string,
deps: RecordDeps,
stoppedBy: StopReason,
): Promise<RecordedTrace | null> {
const recording = recordings.get(sessionId); const recording = recordings.get(sessionId);
if (!recording || recording.settled) return null; if (!recording || recording.settled || recording.finishing) return null;
if (recording.finishAttempt) return recording.finishAttempt; recording.finishing = true;
recording.stoppedBy = stoppedBy;
const attempt = finishRecordingAttempt(sessionId, recording, deps);
recording.finishAttempt = attempt;
try {
return await attempt;
} finally {
if (!recording.settled) {
recording.finishAttempt = null;
}
}
}
async function finishRecordingAttempt(
sessionId: string,
recording: ActiveRecording,
deps: RecordDeps,
): Promise<RecordedTrace | null> {
await clearRearmTimersForRecording(recording, deps); await clearRearmTimersForRecording(recording, deps);
try { try {
// Disposing content capture may commit a final dirty fill. Flush content
// first so all resulting RECORD_STEP messages enter actionQueue before it
// and the observation queues are drained.
await stopRecordingOnAllAgentTabs(recording, deps); await stopRecordingOnAllAgentTabs(recording, deps);
} catch { } catch {
recording.finishing = false;
return null; return null;
} }
if (!(await drainRecordingToStability(recording))) {
return null;
}
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
recording.settled = true; recording.settled = true;
recordings.delete(sessionId); recordings.delete(sessionId);
@@ -822,15 +554,6 @@ export async function handleRecordStart(
params: RecordStartParams, params: RecordStartParams,
deps: RecordDeps = getDefaultDeps(), deps: RecordDeps = getDefaultDeps(),
): Promise<RecordStartResult | RpcError> { ): Promise<RecordStartResult | RpcError> {
const traceVersionOrErr = negotiatedTraceVersion(params);
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
if (traceVersionOrErr === 3 && !deps.cdp) {
return {
code: "protocol_error",
message: "trace v3 recording requires an active CDP connection",
};
}
const ctxOrErr = lookupSession(manager, params, "record_start"); const ctxOrErr = lookupSession(manager, params, "record_start");
if (isRpcError(ctxOrErr)) return ctxOrErr; if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr; const ctx = ctxOrErr;
@@ -847,64 +570,37 @@ export async function handleRecordStart(
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay // on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制"). // instead of flashing ControlOverlay ("Agent 正在控制").
const requestId = makeRequestId(target.tabId); const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: RecordedTrace) => void; let resolveFinish!: (trace: TraceV2) => void;
let rejectFinish!: (err: Error) => void; let rejectFinish!: (err: Error) => void;
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => { const finishPromise = new Promise<TraceV2>((resolve, reject) => {
resolveFinish = resolve; resolveFinish = resolve;
rejectFinish = reject; rejectFinish = reject;
}); });
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL; const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
const startedAtMs = Date.now(); const startedAtMs = Date.now();
const maxPageTokens = params.max_page_tokens;
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, { recordings.set(params.session_id, {
requestId, requestId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl), tabId: target.tabId,
agentWindowId: ctx.agentWindowId, agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl, startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}), ...(params.purpose ? { purpose: params.purpose } : {}),
steps: [], steps: [],
startedAt: new Date(startedAtMs).toISOString(), startedAt: new Date(startedAtMs).toISOString(),
startedAtMs, startedAtMs,
traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise, finishPromise,
resolveFinish, resolveFinish,
rejectFinish, rejectFinish,
settled: false, settled: false,
finishAttempt: null, finishing: false,
observation: currentUrl: navigateUrl,
traceVersionOrErr === 3 && deps.cdp pendingNavigation: false,
? new RecordingObservationRuntime({ pendingNavigationDeadline: undefined,
cdp: deps.cdp,
tabsApi: deps.tabsApi,
maxTokens: maxPageTokens,
redactValues,
})
: null,
stoppedBy: "user_finish",
navigationCallbacks: new Set(),
acceptingNavigation: true,
actionQueue: Promise.resolve(),
lastStepSequenceByProducer: new Map(),
});
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
const recording = recordings.get(params.session_id);
if (!recording || recording.requestId !== requestId || recording.settled) return;
try {
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
} catch {
// The normal action-time capture remains available if a child Document
// is replaced while its readiness refresh is in flight.
}
}); });
// Observe navigations for the whole recording lifetime; attach before // Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture. // optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps); ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => { const abortPending = async (notifyContent: boolean) => {
recordings.get(params.session_id)?.observation?.cancel();
deps.frameCoordinator?.cancel(requestId);
recordings.delete(params.session_id); recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle(); releaseBrowserObservationListenersIfIdle();
if (notifyContent) { if (notifyContent) {
@@ -986,7 +682,7 @@ export async function handleRecordStart(
return cancelledError(); return cancelledError();
} }
active.startUrl = startUrl; active.startUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl; active.currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) { if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false); await abortPending(false);
@@ -994,7 +690,7 @@ export async function handleRecordStart(
code: "invalid_params", code: "invalid_params",
message: params.url message: params.url
? `cannot record on restricted URL (${startUrl}); use an http(s) page` ? `cannot record on restricted URL (${startUrl}); use an http(s) page`
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`, : `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
}; };
} }
@@ -1021,10 +717,6 @@ export async function handleRecordStart(
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs }; const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
try { try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
if (!frameStarted) throw new Error("top recording document did not start");
}
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
} catch { } catch {
await abortPending(true); await abortPending(true);
@@ -1035,17 +727,6 @@ export async function handleRecordStart(
}; };
} }
if (active.observation) {
const activeRecording = recordings.get(params.session_id);
if (activeRecording) {
try {
await activeRecording.observation?.captureInitial(target.tabId);
} catch {
// Proceed without initial observation; steps may be dropped by reducer.
}
}
}
{ {
const cancelled = await abortIfCancelled(true); const cancelled = await abortIfCancelled(true);
if (cancelled) return cancelled; if (cancelled) return cancelled;
@@ -1070,7 +751,7 @@ export async function handleRecordStop(
}; };
} }
const trace = await finishRecording(params.session_id, deps, "cli_stop"); const trace = await finishRecording(params.session_id, deps);
if (!trace) { if (!trace) {
return { return {
code: "protocol_error", code: "protocol_error",
@@ -1100,9 +781,9 @@ export async function handleRecordAwait(
return { code: "cancelled", message: "record_await aborted" }; return { code: "cancelled", message: "record_await aborted" };
} }
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => { const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
let settled = false; let settled = false;
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => { const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
if (settled) return; if (settled) return;
settled = true; settled = true;
if (timer) clearTimeout(timer); if (timer) clearTimeout(timer);
@@ -1144,8 +825,6 @@ export function clearRecordingForSession(sessionId: string): void {
} }
void clearRearmTimersForRecording(recording, getDefaultDeps()); void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) { if (!recording.settled) {
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
recording.observation?.cancel();
recording.settled = true; recording.settled = true;
recording.rejectFinish(new Error("recording cleared")); recording.rejectFinish(new Error("recording cleared"));
} }
@@ -36,7 +36,7 @@ function fakeSnapshotReply() {
}, },
layout: { layout: {
nodeIndex: [1, 2, 3], nodeIndex: [1, 2, 3],
// Legacy fixture: omitted style columns fall back to visible defaults. // styles columns follow requested computedStyles order: [position, pointer-events]
styles: [ styles: [
[i("static"), i("auto")], [i("static"), i("auto")],
[i("fixed"), i("auto")], [i("fixed"), i("auto")],
@@ -682,46 +682,6 @@ describe("captureViewModel", () => {
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
}); });
it("captures painted visibility as DOM fallback evidence", async () => {
const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [[], [], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3, 4],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("hidden"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 100, 30],
[10, 50, 100, 30],
[10, 90, 100, 30],
],
paintOrders: [0, 1, 2, 3],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("subtracts scroll offset to make rects viewport-relative", async () => { it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = { const cdp = {
send: vi.fn(async (_t: number, method: string) => { send: vi.fn(async (_t: number, method: string) => {
@@ -132,67 +132,4 @@ describe("captureFrameData", () => {
]), ]),
); );
}); });
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
}); });
@@ -1,183 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { CdpRunner } from "../../shared";
import type { CapturedNode } from "../capture";
import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
import { probeTooltipNames } from "../name-enrichment";
import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
import { stableIdentifierName } from "../semantic-graph/name-evidence";
import { frameBackendKey } from "../semantic-graph/types";
function document(
target: CdpTarget,
backendNodeId: number,
accessibleName?: string,
frameId = "child",
x = 10,
): FrameDocument<FrameOwnedAxNode> {
const node: CapturedNode = {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "button",
attrs: {
id: "toolInsertRecord",
"data-test-id": "toolInsertRecord",
...(accessibleName ? { "aria-label": accessibleName } : {}),
},
rect: { x, y: 20, w: 40, h: 30 },
localRect: { x: 2, y: 3, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
};
return {
frameId,
parentFrameId: "main",
contextScopeId: "child",
target,
axNodes: [],
domNodes: [node],
url: "https://example.com/frame",
};
}
function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
return resolveSemanticGraph(
buildSemanticGraph({
documents,
viewport: { width: 800, height: 600 },
rootFrameId: documents[0]?.frameId,
}),
{ identifierFallback: false },
);
}
describe("tooltip name enrichment", () => {
it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
const target = { tabId: 7, sessionId: "oopif-session" };
let hovered = false;
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Input.dispatchMouseEvent") {
hovered = (params as { x?: number }).x === 30;
}
return {};
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
if (method === "Runtime.callFunctionOn") {
return { result: { value: hovered ? ["插入行"] : [] } };
}
return {};
}) as CdpRunner["sendToTarget"],
};
const documents = [document(target, 501)];
const graph = semantics(documents);
const names = await probeTooltipNames(cdp, 7, documents, graph);
expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
expect(cdp.sendToTarget).toHaveBeenCalledWith(
target,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child" }),
);
const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
const targetSendsBeforeCache = vi.mocked(
cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
).mock.calls.length;
const cached = await probeTooltipNames(cdp, 7, documents, graph);
expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
});
it("does not probe controls that already have a higher-confidence name", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
};
const documents = [document({ tabId: 8 }, 601, "Add a new record")];
const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
expect(names.size).toBe(0);
expect(cdp.send).not.toHaveBeenCalled();
expect(cdp.sendToTarget).not.toHaveBeenCalled();
});
it("does not collect supplemental names from form controls", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
};
const inputDocument = document({ tabId: 8 }, 602);
inputDocument.domNodes[0] = {
...inputDocument.domNodes[0],
tag: "input",
attrs: { type: "email" },
formValue: "user@example.com",
formDefaultValue: "",
formState: "filled",
};
const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument]));
expect(names.size).toBe(0);
expect(cdp.send).not.toHaveBeenCalled();
expect(cdp.sendToTarget).not.toHaveBeenCalled();
});
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
let hoveredX = -10;
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Input.dispatchMouseEvent") {
hoveredX = (params as { x: number }).x;
}
return {};
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (target, method, params) => {
if (method === "Page.createIsolatedWorld") {
return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
}
if (method === "DOM.resolveNode") {
return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
}
if (method === "Runtime.callFunctionOn") {
const objectId = (params as { objectId: string }).objectId;
const value =
objectId === "left" && hoveredX === 30
? ["Left action"]
: objectId === "right" && hoveredX === 130
? ["Right action"]
: [];
return { result: { value } };
}
return {};
}) as CdpRunner["sendToTarget"],
};
const documents = [
document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
];
const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
});
});
describe("stable identifier evidence", () => {
it("accepts readable action identifiers and rejects framework-generated ids", () => {
expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
});
});
@@ -1,82 +0,0 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { projectRecordSafeObservation } from "../record-safe-observation";
describe("projectRecordSafeObservation", () => {
it("projects the opaque recording identity without exposing captured attributes", () => {
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
const result = projectRecordSafeObservation({
rootFrameId: "child",
frameDocuments: [
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: {
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
value: "user@example.com",
},
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
recordingDocumentId,
},
]);
const dumped = JSON.stringify(result);
expect(dumped).not.toContain("user@example.com");
expect(dumped).not.toContain("attrs");
expect(dumped).not.toContain("domNodes");
expect(dumped).not.toContain("axNodes");
});
it("does not project a page-controlled marker value as recording identity", () => {
const result = projectRecordSafeObservation({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
target: { tabId: 4 },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
expect(JSON.stringify(result)).not.toContain("user@example.com");
});
});
@@ -1,782 +0,0 @@
import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "../capture";
import type { FrameDocument } from "../frame-document";
import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
function dom(
backendNodeId: number,
parentBackendNodeId: number | null,
tag: string,
attrs: Record<string, string> = {},
textContent?: string,
): CapturedNode {
return {
backendNodeId,
parentBackendNodeId,
tag,
attrs,
rect: { x: 0, y: 0, w: 100, h: 30 },
paintOrder: backendNodeId,
position: "static",
pointerEvents: "auto",
...(textContent ? { textContent } : {}),
};
}
function document(
frameId: string,
axNodes: SemanticAxNode[],
domNodes: CapturedNode[],
frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
): FrameDocument<SemanticAxNode> {
return {
frameId,
contextScopeId: frameId,
target: { tabId: 1 },
axNodes,
domNodes,
...frame,
};
}
describe("semantic VOM graph", () => {
it("preserves backend-less AX structure without making it referenceable", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["toolbar"],
},
{
nodeId: "toolbar",
parentId: "root",
role: { type: "role", value: "toolbar" },
name: { type: "computedString", value: "Formatting" },
childIds: ["bold", "italic"],
},
{
nodeId: "bold",
parentId: "toolbar",
backendDOMNodeId: 2,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Bold" },
},
{
nodeId: "italic",
parentId: "toolbar",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Italic" },
},
],
[dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
),
],
});
const toolbar = scene.nodes.find((node) => node.role === "toolbar");
const buttons = scene.nodes.filter((node) => node.role === "button");
expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
});
it("resolves missing AX names through one DOM naming pipeline", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["labelled", "titled", "text"],
},
{
nodeId: "labelled",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "button" },
},
{
nodeId: "titled",
parentId: "root",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
},
{
nodeId: "text",
parentId: "root",
backendDOMNodeId: 4,
role: { type: "role", value: "button" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "button", { "aria-label": "Add row" }),
dom(3, 1, "button", { title: "Settings" }),
dom(4, 1, "button"),
dom(5, 4, "span", {}, "Save"),
],
),
],
});
expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
"Add row",
"Settings",
"Save",
]);
});
it("aggregates AX text at the semantic text frontier", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["covered", "inline-only", "repeated", "labelled"],
},
{
nodeId: "covered",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "paragraph" },
childIds: ["covered-static"],
},
{
nodeId: "covered-static",
parentId: "covered",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Covered text" },
childIds: ["covered-inline"],
},
{
nodeId: "covered-inline",
parentId: "covered-static",
role: { type: "role", value: "InlineTextBox" },
name: { type: "computedString", value: "Covered text" },
},
{
nodeId: "inline-only",
parentId: "root",
backendDOMNodeId: 3,
role: { type: "role", value: "paragraph" },
childIds: ["inline-leaf"],
},
{
nodeId: "inline-leaf",
parentId: "inline-only",
role: { type: "role", value: "InlineTextBox" },
name: { type: "computedString", value: "Inline fallback" },
},
{
nodeId: "repeated",
parentId: "root",
backendDOMNodeId: 4,
role: { type: "role", value: "paragraph" },
childIds: ["first-go", "second-go"],
},
{
nodeId: "first-go",
parentId: "repeated",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Go" },
},
{
nodeId: "second-go",
parentId: "repeated",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Go" },
},
{
nodeId: "labelled",
parentId: "root",
backendDOMNodeId: 5,
role: { type: "role", value: "button" },
childIds: ["visible-label"],
},
{
nodeId: "visible-label",
parentId: "labelled",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Visible text" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "p"),
dom(3, 1, "p"),
dom(4, 1, "p"),
dom(5, 1, "button", { "aria-label": "Explicit action" }),
],
),
],
});
expect(scene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Covered text");
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Inline fallback");
expect(scene.nodes.find((node) => node.backendNodeId === 4)?.name).toBe("Go Go");
expect(scene.nodes.find((node) => node.backendNodeId === 5)?.name).toBe("Explicit action");
expect(renderVom(scene).text).not.toContain("Covered text Covered text");
});
it("uses stable identifiers only as a final name fallback", () => {
const base = document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["fallback", "labelled", "container"],
},
{
nodeId: "fallback",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "button" },
},
{
nodeId: "labelled",
parentId: "root",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Localized name" },
},
{
nodeId: "container",
parentId: "root",
backendDOMNodeId: 4,
role: { type: "role", value: "generic" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
dom(3, 1, "button", { id: "toolIgnoredFallback" }),
dom(4, 1, "div", { id: "toolbarContainer" }),
],
);
const staticScene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [base],
});
const enrichedScene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [base],
supplementalNames: new Map([["main\u00002", "插入行"]]),
});
expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
"Localized name",
);
});
it("isolates equal backend node ids in sibling frames", () => {
const scene = buildSemanticVomScene({
viewport: { width: 1000, height: 800 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["owner-a", "owner-b"],
},
{
nodeId: "owner-a",
parentId: "root",
backendDOMNodeId: 10,
role: { type: "role", value: "Iframe" },
},
{
nodeId: "owner-b",
parentId: "root",
backendDOMNodeId: 20,
role: { type: "role", value: "Iframe" },
},
],
[dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
),
document(
"frame-a",
[
{
nodeId: "button-a",
backendDOMNodeId: 5,
role: { type: "role", value: "button" },
childIds: ["text-a"],
},
{
nodeId: "text-a",
parentId: "button-a",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Accept" },
childIds: ["inline-a"],
},
{
nodeId: "inline-a",
parentId: "text-a",
role: { type: "role", value: "InlineTextBox" },
name: { type: "computedString", value: "Accept" },
},
],
[dom(5, null, "button")],
{ parentFrameId: "main", ownerBackendNodeId: 10 },
),
document(
"frame-b",
[
{
nodeId: "button-b",
backendDOMNodeId: 5,
role: { type: "role", value: "button" },
childIds: ["text-b"],
},
{
nodeId: "text-b",
parentId: "button-b",
role: { type: "role", value: "StaticText" },
name: { type: "computedString", value: "Decline" },
childIds: ["inline-b"],
},
{
nodeId: "inline-b",
parentId: "text-b",
role: { type: "role", value: "InlineTextBox" },
name: { type: "computedString", value: "Decline" },
},
],
[dom(5, null, "button")],
{ parentFrameId: "main", ownerBackendNodeId: 20 },
),
],
});
const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
const owners = new Map(
scene.nodes
.filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
.map((node) => [node.backendNodeId, node.id]),
);
expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
expect(frameButtons.map((node) => node.name)).toEqual(["Accept", "Decline"]);
expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
["frame-a", 5],
["frame-b", 5],
]);
});
it("requires render evidence before promoting DOM fallback semantics", () => {
const hiddenOwner = {
...dom(10, 1, "iframe", { hidden: "" }),
rect: null,
localRect: null,
rendered: false,
};
const hiddenButton = {
...dom(2, 1, "button", { "aria-label": "Hidden action" }),
rendered: false,
};
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
},
],
[dom(1, null, "body"), hiddenButton, hiddenOwner],
),
document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], {
parentFrameId: "main",
ownerBackendNodeId: 10,
}),
],
});
expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false);
expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false);
expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false);
expect(renderVom(scene).refs).toEqual([]);
});
it("retains rendered DOM fallback controls when AX semantics are unavailable", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[],
[dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }],
),
],
});
expect(renderVom(scene).refs).toEqual([
expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }),
]);
});
it("lets explicit DOM structure override ignored AX metadata", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["toolbar"],
},
{
nodeId: "toolbar",
parentId: "root",
backendDOMNodeId: 2,
ignored: true,
role: { type: "role", value: "generic" },
childIds: ["wrapper"],
},
{
nodeId: "wrapper",
parentId: "toolbar",
backendDOMNodeId: 5,
role: { type: "role", value: "generic" },
childIds: ["bold", "italic"],
},
{
nodeId: "bold",
parentId: "wrapper",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Bold" },
},
{
nodeId: "italic",
parentId: "wrapper",
backendDOMNodeId: 4,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Italic" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "div", { role: "toolbar" }),
dom(5, 2, "div"),
dom(3, 5, "button"),
dom(4, 5, "button"),
],
),
],
});
const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
});
it("infers an unnamed group only from independent interaction branches", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["wrapper"],
},
{
nodeId: "wrapper",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "generic" },
childIds: ["first", "second"],
},
{
nodeId: "first",
parentId: "wrapper",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "First" },
},
{
nodeId: "second",
parentId: "wrapper",
backendDOMNodeId: 4,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Second" },
},
],
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
),
],
});
const group = scene.nodes.find((node) => node.backendNodeId === 2);
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
expect(group).toEqual(expect.objectContaining({ role: "group" }));
expect(group?.name).toBeUndefined();
expect(group && isVomReferenceNode(group)).toBe(false);
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
const rendered = renderVom(scene);
const lines = rendered.text.split("\n");
const groupLine = lines.findIndex((line) => line.trim() === "group");
expect(groupLine).toBeGreaterThanOrEqual(0);
for (const ref of rendered.refs) {
expect(ref.line).toBeGreaterThan(groupLine);
expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/);
}
});
it("never promotes a form value into its semantic name through supplemental evidence", () => {
const secret = "user@example.com";
const input = {
...dom(2, 1, "input", { type: "email" }),
formValue: secret,
formDefaultValue: "",
formState: "filled" as const,
};
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
supplementalNames: new Map([["main\u00002", secret]]),
identifierFallback: false,
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["email"],
},
{
nodeId: "email",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "textbox" },
value: { value: secret },
},
],
[dom(1, null, "body"), input],
),
],
});
const field = scene.nodes.find((node) => node.backendNodeId === 2);
expect(field?.name).toBeUndefined();
const rendered = renderVom(scene, { redactValues: true });
expect(rendered.text).toContain("•••");
expect(JSON.stringify(rendered)).not.toContain(secret);
});
it("keeps uncertain single-branch wrappers flat", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["wrapper"],
},
{
nodeId: "wrapper",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "generic" },
childIds: ["button"],
},
{
nodeId: "button",
parentId: "wrapper",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Only action" },
},
],
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
),
],
});
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
scene.nodes.find((node) => node.backendNodeId === 1)?.id,
);
});
it("does not infer across mixed content boundaries", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["wrapper"],
},
{
nodeId: "wrapper",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "generic" },
childIds: ["first", "second", "paragraph"],
},
{
nodeId: "first",
parentId: "wrapper",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "First" },
},
{
nodeId: "second",
parentId: "wrapper",
backendDOMNodeId: 4,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Second" },
},
{
nodeId: "paragraph",
parentId: "wrapper",
backendDOMNodeId: 5,
role: { type: "role", value: "paragraph" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "div"),
dom(3, 2, "button"),
dom(4, 2, "button"),
dom(5, 2, "p", {}, "Description"),
],
),
],
});
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
});
it("keeps only the innermost reliable inferred group", () => {
const scene = buildSemanticVomScene({
viewport: { width: 800, height: 600 },
rootFrameId: "main",
documents: [
document(
"main",
[
{
nodeId: "root",
backendDOMNodeId: 1,
role: { type: "role", value: "RootWebArea" },
childIds: ["outer"],
},
{
nodeId: "outer",
parentId: "root",
backendDOMNodeId: 2,
role: { type: "role", value: "generic" },
childIds: ["inner", "third"],
},
{
nodeId: "inner",
parentId: "outer",
backendDOMNodeId: 3,
role: { type: "role", value: "generic" },
childIds: ["first", "second"],
},
{
nodeId: "first",
parentId: "inner",
backendDOMNodeId: 4,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "First" },
},
{
nodeId: "second",
parentId: "inner",
backendDOMNodeId: 5,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Second" },
},
{
nodeId: "third",
parentId: "outer",
backendDOMNodeId: 6,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Third" },
},
],
[
dom(1, null, "body"),
dom(2, 1, "div"),
dom(3, 2, "div"),
dom(4, 3, "button"),
dom(5, 3, "button"),
dom(6, 2, "button"),
],
),
],
});
const groups = scene.nodes.filter((node) => node.role === "group");
expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
});
});
+36 -30
View File
@@ -1,17 +1,16 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics // CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches // → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a // raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through // shared string table); we request exactly three computed styles so the
// STYLE_COL so capture and visibility policy share one explicit contract. // `styles` columns are [position, pointer-events, cursor] in that order.
import type { Rect, Viewport } from "@browser-skill/vom"; import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
const STYLE_COL = Object.fromEntries( const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]), REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>; ) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,12 +38,6 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not. * `textContent`: the live parser always sets it, hand-built fixtures may not.
*/ */
cursor?: string; cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string; textContent?: string;
formState?: "empty" | "filled" | "default"; formState?: "empty" | "filled" | "default";
formValue?: string; formValue?: string;
@@ -467,6 +460,36 @@ function hoverStateExpression(): string {
})()`; })()`;
} }
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
throwIfAborted(signal);
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
reject(captureAbortError());
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
await cdp
.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: -10,
y: -10,
})
.catch(() =>
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
// best effort
}),
);
}
function capturedText(node: CapturedNode): string | undefined { function capturedText(node: CapturedNode): string | undefined {
const value = const value =
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
@@ -646,7 +669,7 @@ async function probeHoverSurfaces(
try { try {
await clearHover(cdp, tabId); await clearHover(cdp, tabId);
throwIfAborted(options.signal); throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal); await wait(HOVER_SETTLE_MS, options.signal);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", { const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(), expression: hoverStateExpression(),
returnByValue: true, returnByValue: true,
@@ -661,7 +684,7 @@ async function probeHoverSurfaces(
y: candidate.y, y: candidate.y,
}); });
throwIfAborted(options.signal); throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal); await wait(HOVER_SETTLE_MS, options.signal);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", { const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(), expression: hoverStateExpression(),
returnByValue: true, returnByValue: true,
@@ -794,8 +817,6 @@ function parseDocumentNodes(
const posCol = STYLE_COL.position; const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"]; const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor; const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue); const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue); const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []); const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -840,7 +861,6 @@ function parseDocumentNodes(
let position = "static"; let position = "static";
let pointerEvents = "auto"; let pointerEvents = "auto";
let cursor = "auto"; let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n); const li = layoutByNode.get(n);
if (li !== undefined) { if (li !== undefined) {
const b = dl?.bounds?.[li]; const b = dl?.bounds?.[li];
@@ -861,13 +881,6 @@ function parseDocumentNodes(
position = str(strings, styleRow[posCol]) || "static"; position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto"; pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto"; cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
} }
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
@@ -903,7 +916,6 @@ function parseDocumentNodes(
position, position,
pointerEvents, pointerEvents,
cursor, cursor,
rendered,
textContent, textContent,
...(formValue !== undefined ? { formValue } : {}), ...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea" ...(tag === "input" || tag === "textarea"
@@ -1008,13 +1020,7 @@ export async function captureViewModel(
options: CaptureViewModelOptions = {}, options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
throwIfAborted(options.signal); throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {}; const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal); throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics); const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
+22 -24
View File
@@ -33,19 +33,25 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
function transformFrameNodes( function transformFrameNodes(
nodes: CapturedNode[], nodes: CapturedNode[],
projection: GeometryProjection | null, projection: GeometryProjection,
): CapturedNode[] { ): CapturedNode[] {
return nodes.map((node) => ({ return nodes.map((node) => ({
...node, ...node,
rect: projection rect: (() => {
? (() => { const bounds = projectRectToViewport(node.rect, projection);
const bounds = projectRectToViewport(node.rect, projection); return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; })(),
})()
: null,
})); }));
} }
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
if (frame.ownerBackendNodeId === undefined) return undefined;
const parentNodes = frame.parentFrameId
? captured.frameNodes?.get(frame.parentFrameId)
: captured.nodes;
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
}
async function captureMissingFrameDocuments( async function captureMissingFrameDocuments(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
@@ -59,20 +65,14 @@ async function captureMissingFrameDocuments(
for (const frame of graph.frames) { for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue; if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
const owner = frameOwnerNode(captured, frame);
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
try { try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal, signal,
}); });
let projection: GeometryProjection | null = null; const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
try { if (!projection) continue;
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId; const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>(); const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes); if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
@@ -85,13 +85,11 @@ async function captureMissingFrameDocuments(
for (const [childFrameId, nodes] of childFrames) { for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection)); captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
} }
if (frame.ownerBackendNodeId !== undefined) { captured.iframeNodes.set(
captured.iframeNodes.set( frame.ownerBackendNodeId,
frame.ownerBackendNodeId, captured.frameNodes.get(frame.frameId) ?? [],
captured.frameNodes.get(frame.frameId) ?? [], );
); captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId); if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) { for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId); captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
@@ -1,37 +0,0 @@
import type { CdpRunner } from "../shared";
function abortError(): Error {
const error = new Error("hover perception aborted");
error.name = "AbortError";
return error;
}
export async function waitForHover(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) throw abortError();
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
reject(abortError());
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
await cdp
.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: -10,
y: -10,
})
.catch(() =>
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
// Best effort cleanup.
}),
);
}
@@ -1,296 +0,0 @@
import { isVomReferenceNode } from "@browser-skill/vom";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
import { clearHover, waitForHover } from "./hover-perception";
import { stableIdentifierName } from "./semantic-graph/name-evidence";
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
const MAX_TOOLTIP_CANDIDATES = 8;
const MAX_TOOLTIP_PROBE_MS = 2_400;
const TOOLTIP_SETTLE_MS = 250;
const CACHE_TTL_MS = 5 * 60_000;
const MAX_CACHE_ENTRIES = 500;
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
interface TooltipCacheEntry {
name: string;
expiresAt: number;
}
interface TooltipCandidate<T extends FrameOwnedAxNode> {
document: FrameDocument<T>;
backendNodeId: number;
x: number;
y: number;
stableIdentifier: boolean;
}
export interface TooltipNameProbeOptions {
signal?: AbortSignal;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
const tooltipCache = new Map<string, TooltipCacheEntry>();
function cacheKey<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
): string {
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
}
function cachedName<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
): string | undefined {
const key = cacheKey(document, backendNodeId);
const entry = tooltipCache.get(key);
if (!entry) return undefined;
if (entry.expiresAt <= Date.now()) {
tooltipCache.delete(key);
return undefined;
}
return entry.name;
}
function storeName<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
name: string,
): void {
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
const oldest = tooltipCache.keys().next().value;
if (oldest !== undefined) tooltipCache.delete(oldest);
}
tooltipCache.set(cacheKey(document, backendNodeId), {
name,
expiresAt: Date.now() + CACHE_TTL_MS,
});
}
function tooltipCollectorFunction(): string {
return `function() {
const ownRoot = this && typeof this.getRootNode === "function" ? this.getRootNode() : null;
const roots = ownRoot && ownRoot !== document ? [ownRoot, document] : [document];
const items = [];
const seen = new Set();
for (const root of roots) {
for (const el of Array.from(root.querySelectorAll("[role='tooltip']"))) {
const style = getComputedStyle(el);
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) continue;
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
const text = String(el.getAttribute("aria-label") || el.getAttribute("title") || el.textContent || "")
.replace(/\\s+/g, " ")
.trim();
if (!text || text.length > 160 || seen.has(text)) continue;
seen.add(text);
items.push(text);
}
}
return items.slice(0, 20);
}`;
}
function cleanTooltipDiff(before: string[], after: string[]): string | undefined {
const existing = new Set(before.map((value) => value.toLowerCase()));
const added = [...new Set(after.map((value) => value.replace(/\s+/g, " ").trim()))].filter(
(value) => value && !existing.has(value.toLowerCase()),
);
return added.length === 1 ? added[0] : undefined;
}
function candidates<T extends FrameOwnedAxNode>(
documents: FrameDocument<T>[],
graph: ResolvedSemanticGraph,
): TooltipCandidate<T>[] {
const documentByFrame = new Map(documents.map((document) => [document.frameId, document]));
const byFrame = new Map<string, TooltipCandidate<T>[]>();
for (const node of graph.nodes.values()) {
if (
node.vom.name ||
FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") ||
!node.referenceable ||
node.backendNodeId === undefined ||
!isVomReferenceNode({
...node.vom,
id: 0,
parentId: null,
backendNodeId: node.backendNodeId,
frameId: node.frameId,
referenceable: true,
})
) {
continue;
}
const document = documentByFrame.get(node.frameId);
const rect = node.dom?.rect;
if (!document || !rect || rect.w <= 0 || rect.h <= 0) continue;
if (
rect.x + rect.w <= 0 ||
rect.y + rect.h <= 0 ||
rect.x >= graph.viewport.width ||
rect.y >= graph.viewport.height
) {
continue;
}
if (
node.vom.disabled ||
node.vom.inert ||
node.vom.pointerEvents === "none" ||
node.vom.sensitive
) {
continue;
}
const frameCandidates = byFrame.get(node.frameId) ?? [];
frameCandidates.push({
document,
backendNodeId: node.backendNodeId,
x: (Math.max(0, rect.x) + Math.min(graph.viewport.width, rect.x + rect.w)) / 2,
y: (Math.max(0, rect.y) + Math.min(graph.viewport.height, rect.y + rect.h)) / 2,
stableIdentifier: stableIdentifierName(node.dom?.attrs ?? {}) !== undefined,
});
byFrame.set(node.frameId, frameCandidates);
}
const out: TooltipCandidate<T>[] = [];
for (const stableIdentifier of [true, false]) {
const queues = documents.map((document) =>
(byFrame.get(document.frameId) ?? []).filter(
(candidate) => candidate.stableIdentifier === stableIdentifier,
),
);
for (let index = 0; queues.some((queue) => index < queue.length); index += 1) {
for (const queue of queues) {
const candidate = queue[index];
if (candidate) out.push(candidate);
}
}
}
return out;
}
async function executionContextId<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
document: FrameDocument<T>,
): Promise<number | undefined> {
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
document.target,
"Page.createIsolatedWorld",
{
frameId: document.frameId,
worldName: "bsk-vom-tooltip",
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function resolvedObjectId<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
document: FrameDocument<T>,
contextId: number,
backendNodeId: number,
): Promise<string | undefined> {
const result = await sendToCdpTarget<{
object?: { objectId?: string };
}>(cdp, document.target, "DOM.resolveNode", {
backendNodeId,
executionContextId: contextId,
});
return result.object?.objectId;
}
async function tooltipTexts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
document: FrameDocument<T>,
objectId: string,
): Promise<string[]> {
const result = await sendToCdpTarget<{
result?: { value?: string[] };
}>(cdp, document.target, "Runtime.callFunctionOn", {
functionDeclaration: tooltipCollectorFunction(),
objectId,
returnByValue: true,
});
return result.result?.value ?? [];
}
async function releaseObject<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
document: FrameDocument<T>,
objectId: string,
): Promise<void> {
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch(
() => undefined,
);
}
export async function probeTooltipNames<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
documents: FrameDocument<T>[],
graph: ResolvedSemanticGraph,
options: TooltipNameProbeOptions = {},
): Promise<Map<string, string>> {
const names = new Map<string, string>();
const pending: TooltipCandidate<T>[] = [];
for (const candidate of candidates(documents, graph)) {
const cached = cachedName(candidate.document, candidate.backendNodeId);
if (cached) {
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
pending.push(candidate);
}
}
if (pending.length === 0) return names;
const started = Date.now();
const contexts = new Map<string, number>();
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of pending) {
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
let objectId: string | undefined;
try {
let contextId = contexts.get(candidate.document.frameId);
if (contextId === undefined) {
contextId = await executionContextId(cdp, candidate.document);
if (contextId === undefined) continue;
contexts.set(candidate.document.frameId, contextId);
}
objectId = await resolvedObjectId(
cdp,
candidate.document,
contextId,
candidate.backendNodeId,
);
if (!objectId) continue;
await clearHover(cdp, tabId);
const before = await tooltipTexts(cdp, candidate.document, objectId);
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
const after = await tooltipTexts(cdp, candidate.document, objectId);
const name = cleanTooltipDiff(before, after);
if (!name) continue;
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
storeName(candidate.document, candidate.backendNodeId, name);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") throw error;
} finally {
if (objectId) await releaseObject(cdp, candidate.document, objectId);
await clearHover(cdp, tabId);
}
}
} finally {
await clearHover(cdp, tabId);
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return names;
}
@@ -1,6 +1,5 @@
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
@@ -10,8 +9,6 @@ export interface CaptureVomFrame {
target: CdpTarget; target: CdpTarget;
parentFrameId?: string; parentFrameId?: string;
ownerBackendNodeId?: number; ownerBackendNodeId?: number;
/** BrowserSkill-generated identity for the recording agent in this Document. */
recordingDocumentId?: string;
} }
/** Allowlisted geometry used to match a recorded action to a rendered ref. */ /** Allowlisted geometry used to match a recorded action to a rendered ref. */
@@ -40,22 +37,14 @@ export interface CaptureVomObservationResult {
} }
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] { function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => { return documents.map((document) => ({
let recordingDocumentId: string | undefined; frameId: document.frameId,
for (const node of document.domNodes) { target: document.target,
recordingDocumentId = readRecordingDocumentIdentity(node.attrs); ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
if (recordingDocumentId) break; ...(document.ownerBackendNodeId !== undefined
} ? { ownerBackendNodeId: document.ownerBackendNodeId }
return { : {}),
frameId: document.frameId, }));
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
...(recordingDocumentId ? { recordingDocumentId } : {}),
};
});
} }
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] { function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
@@ -1,154 +0,0 @@
import type { Viewport } from "@browser-skill/vom";
import type { FrameDocument } from "../frame-document";
import {
frameAxKey,
frameBackendKey,
type SemanticAxNode,
type SemanticFrame,
type SemanticGraph,
type SemanticGraphNode,
type SemanticNodeId,
} from "./types";
export interface BuildSemanticGraphInput {
documents: FrameDocument<SemanticAxNode>[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds?: ReadonlySet<number>;
}
function domNodeId(frameId: string, backendNodeId: number): SemanticNodeId {
return JSON.stringify([frameId, "dom", backendNodeId]);
}
function axNodeId(frameId: string, nodeId: string): SemanticNodeId {
return JSON.stringify([frameId, "ax", nodeId]);
}
export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGraph {
const rootFrameId =
input.rootFrameId ??
input.documents.find((document) => !document.parentFrameId)?.frameId ??
input.documents[0]?.frameId ??
"root";
const frames = new Map<string, SemanticFrame>();
const nodes = new Map<SemanticNodeId, SemanticGraphNode>();
const nodeIdsByFrame = new Map<string, SemanticNodeId[]>();
const nodeByFrameBackend = new Map<string, SemanticNodeId>();
const nodeByFrameAx = new Map<string, SemanticNodeId>();
for (const [frameOrder, document] of input.documents.entries()) {
frames.set(document.frameId, {
frameId: document.frameId,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
contextScopeId: document.contextScopeId,
target: document.target,
...(document.url ? { url: document.url } : {}),
order: frameOrder,
});
const frameNodeIds: SemanticNodeId[] = [];
nodeIdsByFrame.set(document.frameId, frameNodeIds);
for (const [index, dom] of document.domNodes.entries()) {
const id = domNodeId(document.frameId, dom.backendNodeId);
const node: SemanticGraphNode = {
id,
frameId: document.frameId,
backendNodeId: dom.backendNodeId,
dom,
sourceOrder: index,
excluded:
document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
};
nodes.set(id, node);
frameNodeIds.push(id);
nodeByFrameBackend.set(frameBackendKey(document.frameId, dom.backendNodeId), id);
}
for (const [index, ax] of document.axNodes.entries()) {
const joinedId =
typeof ax.backendDOMNodeId === "number"
? nodeByFrameBackend.get(frameBackendKey(document.frameId, ax.backendDOMNodeId))
: undefined;
const id = joinedId ?? axNodeId(document.frameId, ax.nodeId);
const existing = nodes.get(id);
if (existing) {
existing.ax = ax;
existing.axNodeId = ax.nodeId;
} else {
nodes.set(id, {
id,
frameId: document.frameId,
...(typeof ax.backendDOMNodeId === "number"
? { backendNodeId: ax.backendDOMNodeId }
: {}),
axNodeId: ax.nodeId,
ax,
sourceOrder: document.domNodes.length + index,
excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
});
frameNodeIds.push(id);
}
nodeByFrameAx.set(frameAxKey(document.frameId, ax.nodeId), id);
}
}
for (const document of input.documents) {
for (const dom of document.domNodes) {
const id = nodeByFrameBackend.get(frameBackendKey(document.frameId, dom.backendNodeId));
const node = id ? nodes.get(id) : undefined;
if (!node || dom.parentBackendNodeId === null) continue;
node.domParentId = nodeByFrameBackend.get(
frameBackendKey(document.frameId, dom.parentBackendNodeId),
);
}
for (const ax of document.axNodes) {
const id = nodeByFrameAx.get(frameAxKey(document.frameId, ax.nodeId));
const node = id ? nodes.get(id) : undefined;
if (!node || !ax.parentId) continue;
node.axParentId = nodeByFrameAx.get(frameAxKey(document.frameId, ax.parentId));
}
}
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
const excludedQueue: SemanticNodeId[] = [];
for (const node of nodes.values()) {
if (node.axParentId) {
const children = axChildren.get(node.axParentId) ?? [];
children.push(node.id);
axChildren.set(node.axParentId, children);
}
if (node.excluded) excludedQueue.push(node.id);
}
for (let index = 0; index < excludedQueue.length; index += 1) {
for (const childId of axChildren.get(excludedQueue[index]) ?? []) {
const child = nodes.get(childId);
if (!child || child.excluded) continue;
child.excluded = true;
excludedQueue.push(childId);
}
}
for (const frame of frames.values()) {
if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) continue;
frame.ownerNodeId = nodeByFrameBackend.get(
frameBackendKey(frame.parentFrameId, frame.ownerBackendNodeId),
);
}
return {
viewport: input.viewport,
rootFrameId,
frames,
nodes,
nodeIdsByFrame,
nodeByFrameBackend,
};
}
@@ -1,59 +0,0 @@
import type { Viewport, VomScene } from "@browser-skill/vom";
import type { FrameDocument } from "../frame-document";
import { buildSemanticGraph as buildGraph } from "./build";
import { projectSemanticGraph as projectGraph } from "./project";
import { resolveSemanticGraph as resolveGraph } from "./resolve";
import { normalizeSemanticStructure as normalizeStructure } from "./structure";
import type {
ResolvedSemanticGraph,
SemanticAxNode,
SemanticGraph,
StructuredSemanticGraph,
} from "./types";
export type { SemanticAxNode } from "./types";
export interface SemanticVomInput {
documents: FrameDocument<SemanticAxNode>[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds?: ReadonlySet<number>;
}
export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
return buildGraph(input);
}
export function resolveSemanticGraph(
graph: SemanticGraph,
options: {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
} = {},
): ResolvedSemanticGraph {
return resolveGraph(graph, options);
}
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
return normalizeStructure(graph);
}
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
return projectGraph(graph);
}
export function buildSemanticVomScene(
input: SemanticVomInput & {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
},
): VomScene {
return projectSemanticGraph(
normalizeSemanticStructure(
resolveSemanticGraph(buildSemanticGraph(input), {
supplementalNames: input.supplementalNames,
identifierFallback: input.identifierFallback,
}),
),
);
}
@@ -1,34 +0,0 @@
const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
function identifierTokens(value: string): string[] {
return value
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
.split(/[^a-zA-Z0-9]+/)
.map((token) => token.toLowerCase())
.filter(Boolean);
}
function readableIdentifier(value: string | undefined): string | undefined {
const raw = value?.trim();
if (!raw || raw.length < 3 || raw.length > 80) return undefined;
if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
const tokens = identifierTokens(raw);
if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
return tokens
.map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
.join(" ");
}
export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
for (const name of IDENTIFIER_ATTRS) {
const label = readableIdentifier(attrs[name]);
if (label) return label;
}
return undefined;
}
@@ -1,85 +0,0 @@
import type { VomNode, VomScene } from "@browser-skill/vom";
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
function retained(node: StructuredSemanticNode): boolean {
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
}
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
const connected = new Set<string>([graph.rootFrameId]);
const childrenByParent = new Map<string, string[]>();
for (const frame of graph.frames.values()) {
if (!frame.parentFrameId) continue;
const children = childrenByParent.get(frame.parentFrameId) ?? [];
children.push(frame.frameId);
childrenByParent.set(frame.parentFrameId, children);
}
const queue = [graph.rootFrameId];
for (let index = 0; index < queue.length; index += 1) {
for (const frameId of childrenByParent.get(queue[index]) ?? []) {
if (connected.has(frameId)) continue;
const frame = graph.frames.get(frameId);
if (!frame?.ownerNodeId) continue;
const owner = graph.nodes.get(frame.ownerNodeId);
if (!owner || !retained(owner)) continue;
connected.add(frame.frameId);
queue.push(frame.frameId);
}
}
return connected;
}
function domAncestors(
graph: StructuredSemanticGraph,
node: StructuredSemanticNode,
numericId: ReadonlyMap<SemanticNodeId, number>,
): number[] {
const ancestors: number[] = [];
const seen = new Set<SemanticNodeId>();
let current = node.domParentId;
while (current && !seen.has(current)) {
seen.add(current);
const id = numericId.get(current);
if (id !== undefined) ancestors.push(id);
current = graph.nodes.get(current)?.domParentId;
}
return ancestors;
}
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
const connectedFrames = connectedFrameIds(graph);
const orderedNodes = [...graph.frames.values()]
.filter((frame) => connectedFrames.has(frame.frameId))
.sort((a, b) => a.order - b.order)
.flatMap((frame) =>
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
.map((id) => graph.nodes.get(id))
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
.sort((a, b) => a.sourceOrder - b.sourceOrder),
);
const numericId = new Map<SemanticNodeId, number>();
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
const nodes: VomNode[] = orderedNodes.map((node) => ({
...node.vom,
id: numericId.get(node.id) as number,
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
frameId: node.frameId,
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
referenceable: node.referenceable,
...(node.dom
? {
domParentId:
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
domAncestorIds: domAncestors(graph, node, numericId),
}
: {}),
}));
return {
viewport: graph.viewport,
nodes,
rootFrameId: graph.rootFrameId,
};
}
@@ -1,521 +0,0 @@
import type { VomNode } from "@browser-skill/vom";
import { stableIdentifierName } from "./name-evidence";
import type {
ResolvedSemanticGraph,
ResolvedSemanticNode,
SemanticAxNode,
SemanticGraph,
SemanticGraphNode,
SemanticNodeId,
} from "./types";
import { frameBackendKey } from "./types";
const FORM_TAGS = new Set(["input", "textarea", "select"]);
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]);
const INTERACTIVE_ROLES = new Set([
"button",
"checkbox",
"combobox",
"link",
"listbox",
"menuitem",
"menuitemcheckbox",
"menuitemradio",
"option",
"radio",
"searchbox",
"slider",
"spinbutton",
"switch",
"tab",
"textbox",
"treeitem",
]);
const SENSITIVE_INPUT_TYPES = new Set([
"password",
"credit-card",
"one-time-code",
"current-password",
"new-password",
]);
function clean(value: string | undefined): string | undefined {
const normalized = value?.replace(/\s+/g, " ").trim();
return normalized || undefined;
}
function axValue(field?: { value?: string | number | boolean }): string | undefined {
return field?.value === undefined ? undefined : clean(String(field.value));
}
function axProperty(node: SemanticAxNode | undefined, name: string): string | undefined {
return axValue(node?.properties?.find((property) => property.name === name)?.value);
}
function normalizeRole(value: string | undefined): string | undefined {
const role = clean(value)?.toLowerCase();
return role || undefined;
}
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
if (tag === "button") return "button";
if (tag === "textarea") return "textbox";
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
if (tag === "a" && attrs.href) return "link";
if (tag === "iframe") return "Iframe";
if (tag === "dialog") return "dialog";
if (tag === "img") return "img";
if (tag === "table") return "table";
if (tag === "tr") return "row";
if (tag === "th") return "columnheader";
if (tag === "td") return "cell";
if (tag === "ul" || tag === "ol") return "list";
if (tag === "li") return "listitem";
if (/^h[1-6]$/.test(tag)) return "heading";
if (tag === "nav") return "navigation";
if (tag === "main") return "main";
if (tag === "form") return "form";
if (tag === "input") {
const type = (attrs.type ?? "text").toLowerCase();
if (type === "hidden") return undefined;
if (type === "checkbox") return "checkbox";
if (type === "radio") return "radio";
if (type === "range") return "slider";
if (type === "number") return "spinbutton";
if (type === "search") return "searchbox";
if (["button", "submit", "reset", "image"].includes(type)) return "button";
return "textbox";
}
return undefined;
}
function domRole(tag: string, attrs: Record<string, string>): string | undefined {
return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
}
function resolvedRole(node: SemanticGraphNode): {
value?: string;
source: ResolvedSemanticNode["roleSource"];
} {
const rawAxRole = clean(axValue(node.ax?.role));
const axRole = normalizeRole(rawAxRole);
const attrs = node.dom?.attrs ?? {};
const explicitDomRole = normalizeRole(attrs.role);
const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
const usableAxRole = node.ax?.ignored !== true;
if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
return { value: rawAxRole, source: "ax" };
}
if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
if (rawAxRole) {
return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
}
return { source: "none" };
}
function sensitive(node: SemanticGraphNode): boolean {
const attrs = node.dom?.attrs ?? {};
const type = (attrs.type ?? "").toLowerCase();
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
return (
type === "password" ||
autocomplete.startsWith("cc-") ||
SENSITIVE_INPUT_TYPES.has(autocomplete) ||
SENSITIVE_INPUT_TYPES.has(axProperty(node.ax, "inputType") ?? "")
);
}
function inputState(
node: SemanticGraphNode,
role: string | undefined,
value: string | undefined,
isSensitive: boolean,
): VomNode["inputState"] {
if (node.dom?.formState) return node.dom.formState;
const tag = node.dom?.tag.toLowerCase() ?? "";
if (!FORM_TAGS.has(tag) && !["textbox", "searchbox"].includes(role ?? "")) return undefined;
if (isSensitive) return value ? "filled" : "empty";
if (node.dom?.formValue !== undefined) {
if (node.dom.formValue === "") return "empty";
return node.dom.formValue === (node.dom.formDefaultValue ?? "") ? "default" : "filled";
}
return value ? "filled" : "empty";
}
function externalHrefHost(
href: string | undefined,
pageUrl: string | undefined,
): string | undefined {
if (!href || !pageUrl) return undefined;
try {
const page = new URL(pageUrl);
const target = new URL(href, page);
return target.origin === page.origin ? undefined : target.hostname;
} catch {
return undefined;
}
}
interface ResolveIndexes {
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
domIdIndex: Map<string, SemanticNodeId>;
labelForIndex: Map<string, SemanticNodeId[]>;
domText: (nodeId: SemanticNodeId) => string | undefined;
axText: (nodeId: SemanticNodeId) => string | undefined;
hasNativeDescendant: (nodeId: SemanticNodeId) => boolean;
}
function buildIndexes(graph: SemanticGraph): ResolveIndexes {
const domChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
const domIdIndex = new Map<string, SemanticNodeId>();
const labelForIndex = new Map<string, SemanticNodeId[]>();
for (const node of graph.nodes.values()) {
if (node.domParentId) {
const children = domChildren.get(node.domParentId) ?? [];
children.push(node.id);
domChildren.set(node.domParentId, children);
}
if (node.axParentId) {
const children = axChildren.get(node.axParentId) ?? [];
children.push(node.id);
axChildren.set(node.axParentId, children);
}
const id = clean(node.dom?.attrs.id);
if (id) domIdIndex.set(`${node.frameId}\u0000${id}`, node.id);
const targetId = clean(node.dom?.attrs.for);
if (node.dom?.tag.toLowerCase() === "label" && targetId) {
const key = `${node.frameId}\u0000${targetId}`;
const labels = labelForIndex.get(key) ?? [];
labels.push(node.id);
labelForIndex.set(key, labels);
}
}
for (const [parentId, children] of axChildren) {
const order = graph.nodes.get(parentId)?.ax?.childIds ?? [];
const orderByAxId = new Map(order.map((nodeId, index) => [nodeId, index]));
children.sort((a, b) => {
const aIndex = orderByAxId.get(graph.nodes.get(a)?.axNodeId ?? "") ?? -1;
const bIndex = orderByAxId.get(graph.nodes.get(b)?.axNodeId ?? "") ?? -1;
return (
(aIndex < 0 ? Number.MAX_SAFE_INTEGER : aIndex) -
(bIndex < 0 ? Number.MAX_SAFE_INTEGER : bIndex)
);
});
}
const textMemo = new Map<SemanticNodeId, string | undefined>();
const resolvingText = new Set<SemanticNodeId>();
const domText = (nodeId: SemanticNodeId): string | undefined => {
if (textMemo.has(nodeId)) return textMemo.get(nodeId);
if (resolvingText.has(nodeId)) return undefined;
resolvingText.add(nodeId);
const node = graph.nodes.get(nodeId);
const parts: string[] = [];
const own = clean(node?.dom?.textContent);
if (own) parts.push(own);
for (const childId of domChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const childRole = child
? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
: undefined;
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
const childText = domText(childId);
if (childText) parts.push(childText);
if (parts.join(" ").length >= 240) break;
}
resolvingText.delete(nodeId);
const result = clean(parts.join(" ").slice(0, 240));
textMemo.set(nodeId, result);
return result;
};
const axTextMemo = new Map<SemanticNodeId, string | undefined>();
const resolvingAxText = new Set<SemanticNodeId>();
const axText = (nodeId: SemanticNodeId): string | undefined => {
if (axTextMemo.has(nodeId)) return axTextMemo.get(nodeId);
if (resolvingAxText.has(nodeId)) return undefined;
resolvingAxText.add(nodeId);
const complete = (value: string | undefined): string | undefined => {
resolvingAxText.delete(nodeId);
axTextMemo.set(nodeId, value);
return value;
};
const node = graph.nodes.get(nodeId);
const role = normalizeRole(axValue(node?.ax?.role)) ?? "";
if (TEXT_AX_ROLES.has(role)) {
const own = clean(axValue(node?.ax?.name) ?? axValue(node?.ax?.value));
if (own) return complete(own);
}
const parts: string[] = [];
for (const childId of axChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const childRole = normalizeRole(axValue(child?.ax?.role)) ?? "";
if (INTERACTIVE_ROLES.has(childRole)) continue;
const childText = axText(childId);
if (childText) parts.push(childText);
if (parts.join(" ").length >= 240) break;
}
return complete(clean(parts.join(" ").slice(0, 240)));
};
const nativeMemo = new Map<SemanticNodeId, boolean>();
const hasNativeDescendant = (nodeId: SemanticNodeId): boolean => {
const cached = nativeMemo.get(nodeId);
if (cached !== undefined) return cached;
nativeMemo.set(nodeId, false);
for (const childId of domChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const childRole =
(child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
(child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
if (
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
INTERACTIVE_ROLES.has(childRole ?? "") ||
hasNativeDescendant(childId)
) {
nativeMemo.set(nodeId, true);
return true;
}
}
return false;
};
return {
domChildren,
axChildren,
domIdIndex,
labelForIndex,
domText,
axText,
hasNativeDescendant,
};
}
function ariaLabelledText(node: SemanticGraphNode, indexes: ResolveIndexes): string | undefined {
const ids = (node.dom?.attrs["aria-labelledby"] ?? "").split(/\s+/).filter(Boolean);
const parts = ids.flatMap((id) => {
const target = indexes.domIdIndex.get(`${node.frameId}\u0000${id}`);
return target ? (indexes.domText(target) ?? []) : [];
});
return clean(parts.join(" "));
}
function nativeLabelText(
node: SemanticGraphNode,
graph: SemanticGraph,
indexes: ResolveIndexes,
): string | undefined {
const id = clean(node.dom?.attrs.id);
if (id) {
const labels = indexes.labelForIndex.get(`${node.frameId}\u0000${id}`) ?? [];
const explicit = clean(
labels
.map((labelId) => indexes.domText(labelId))
.filter(Boolean)
.join(" "),
);
if (explicit) return explicit;
}
let parentId = node.domParentId;
let guard = 0;
while (parentId && guard <= graph.nodes.size) {
const parent = graph.nodes.get(parentId);
if (!parent) break;
if (parent.dom?.tag.toLowerCase() === "label") return indexes.domText(parentId);
parentId = parent.domParentId;
guard += 1;
}
return undefined;
}
function precedingAxText(
node: SemanticGraphNode,
graph: SemanticGraph,
indexes: ResolveIndexes,
): string | undefined {
if (!node.axParentId) return undefined;
const siblings = indexes.axChildren.get(node.axParentId) ?? [];
const index = siblings.indexOf(node.id);
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
const sibling = graph.nodes.get(siblings[index - offset]);
const role = normalizeRole(axValue(sibling?.ax?.role)) ?? "";
if (INTERACTIVE_ROLES.has(role)) break;
if (!TEXT_AX_ROLES.has(role) && role !== "generic") continue;
const text = clean(axValue(sibling?.ax?.name) ?? axValue(sibling?.ax?.value))?.replace(
/[::]\s*$/,
"",
);
if (text && text.length <= 80) return text;
}
return undefined;
}
function precedingDomText(
node: SemanticGraphNode,
graph: SemanticGraph,
indexes: ResolveIndexes,
): string | undefined {
if (!node.domParentId) return undefined;
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
const index = siblings.indexOf(node.id);
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
const sibling = graph.nodes.get(siblings[index - offset]);
const siblingRole = sibling
? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
: undefined;
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
const text = indexes.domText(siblings[index - offset])?.replace(/[::]\s*$/, "");
if (text && text.length <= 80) return text;
}
return undefined;
}
function resolvedName(
node: SemanticGraphNode,
role: string | undefined,
graph: SemanticGraph,
indexes: ResolveIndexes,
supplementalNames: ReadonlyMap<string, string>,
identifierFallback: boolean,
): string | undefined {
const attrs = node.dom?.attrs ?? {};
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
const descendantText = indexes.domText(node.id);
const interactive =
INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
const candidates = [
node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
ariaLabelledText(node, indexes),
clean(attrs["aria-label"]),
nativeLabelText(node, graph, indexes),
clean(attrs.title),
clean(attrs.alt),
node.ax?.ignored === true ? undefined : indexes.axText(node.id),
descendantText,
formControl ? precedingAxText(node, graph, indexes) : undefined,
formControl ? precedingDomText(node, graph, indexes) : undefined,
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
formControl || !interactive || node.backendNodeId === undefined
? undefined
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
];
let name = candidates.find((candidate) => candidate !== undefined);
const hasPopup = axProperty(node.ax, "hasPopup");
if (name && hasPopup && hasPopup !== "false") {
name =
axProperty(node.ax, "expanded") === "true" ? `${name} [expanded]` : `${name} [has-submenu]`;
}
if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id);
return name;
}
function insideNative(node: SemanticGraphNode, graph: SemanticGraph): boolean {
let parentId = node.domParentId;
let guard = 0;
while (parentId && guard <= graph.nodes.size) {
const parent = graph.nodes.get(parentId);
if (!parent) break;
if (parent.dom && NATIVE_CONTROL_TAGS.has(parent.dom.tag.toLowerCase())) return true;
parentId = parent.domParentId;
guard += 1;
}
return false;
}
function nearbyText(
node: SemanticGraphNode,
graph: SemanticGraph,
indexes: ResolveIndexes,
): string | undefined {
if (!node.domParentId) return undefined;
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
const index = siblings.indexOf(node.id);
if (index < 0) return undefined;
const texts = siblings
.slice(Math.max(0, index - 3), index + 4)
.filter((id) => id !== node.id)
.flatMap((id) => indexes.domText(id) ?? [])
.slice(0, 3);
return clean(texts.join(" "));
}
export function resolveSemanticGraph(
graph: SemanticGraph,
options: {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
} = {},
): ResolvedSemanticGraph {
const indexes = buildIndexes(graph);
const supplementalNames = options.supplementalNames ?? new Map<string, string>();
const identifierFallback = options.identifierFallback ?? true;
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
for (const node of graph.nodes.values()) {
const roleResolution = resolvedRole(node);
const role = roleResolution.value;
const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
const isSensitive = sensitive(node);
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
const value = isSensitive ? undefined : sourceValue;
const attrs = { ...(node.dom?.attrs ?? {}) };
if (isSensitive) delete attrs.value;
const frame = graph.frames.get(node.frameId);
const tag = node.dom?.tag.toLowerCase() ?? "";
const vom: ResolvedSemanticNode["vom"] = {
tag,
rect: node.dom?.rect ?? null,
paintOrder: node.dom?.paintOrder ?? 0,
position: node.dom?.position || "static",
pointerEvents: node.dom?.pointerEvents || "auto",
...(node.dom?.cursor ? { cursor: node.dom.cursor } : {}),
...(role ? { role } : {}),
...(name ? { name } : {}),
...(value !== undefined ? { value } : {}),
...(clean(node.dom?.formPlaceholder ?? attrs.placeholder)
? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) }
: {}),
attrs,
...(clean(node.dom?.textContent) ? { text: clean(node.dom?.textContent) } : {}),
...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
inputState: inputState(node, role, sourceValue, isSensitive),
sensitive: isSensitive,
modal:
["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") ||
tag === "dialog" ||
(attrs["aria-modal"] ?? "").toLowerCase() === "true",
disabled:
Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
(attrs["aria-disabled"] ?? "").toLowerCase() === "true",
inert: Object.prototype.hasOwnProperty.call(attrs, "inert"),
hasNativeDescendant: indexes.hasNativeDescendant(node.id),
insideNative: insideNative(node, graph),
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
};
const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
}
if (expanded === "true" && !vom.attrs?.["aria-expanded"]) {
vom.attrs = { ...(vom.attrs ?? {}), "aria-expanded": "true" };
}
if (controls && !vom.attrs?.["aria-controls"]) {
vom.attrs = { ...(vom.attrs ?? {}), "aria-controls": controls };
}
nodes.set(node.id, {
...node,
vom,
referenceable: node.backendNodeId !== undefined,
roleSource: roleResolution.source,
});
}
return { ...graph, nodes };
}
@@ -1,456 +0,0 @@
import {
applyVomInteractionRecovery,
isVomReferenceNode,
isVomStructuralRole,
type VomNode,
} from "@browser-skill/vom";
import type {
ResolvedSemanticGraph,
ResolvedSemanticNode,
SemanticNodeId,
StructureDecision,
StructuredSemanticGraph,
StructuredSemanticNode,
} from "./types";
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
const CONTENT_BOUNDARY_ROLES = new Set([
"article",
"dialog",
"document",
"feed",
"form",
"grid",
"heading",
"iframe",
"list",
"main",
"navigation",
"paragraph",
"region",
"section",
"table",
"tree",
]);
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
const attrs = node.vom.attrs ?? {};
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
attrs[name]?.trim(),
);
}
function explicitStructural(node: ResolvedSemanticNode): boolean {
return (
isVomStructuralRole(node.vom.role) &&
node.roleSource !== "none" &&
node.roleSource !== "ax-ignored"
);
}
function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean {
return node.ax !== undefined && node.ax.ignored !== true;
}
function isRenderedDomFallback(node: ResolvedSemanticNode): boolean {
if (!node.dom) return false;
const attrs = node.dom.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
) {
return false;
}
return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null);
}
function initialParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
): SemanticNodeId | undefined {
let parentId = node.axParentId ?? node.domParentId;
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
const axParent = graph.nodes.get(node.axParentId);
const axRole = axParent?.vom.role?.toLowerCase() ?? "";
const meaningfulAxParent =
axParent !== undefined &&
!ROOT_ROLES.has(axRole) &&
(explicitStructural(axParent) ||
!TRANSPARENT_ROLES.has(axRole) ||
Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
}
if (parentId) return parentId;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
return [...graph.frames.values()]
.sort((a, b) => a.order - b.order)
.flatMap((frame) =>
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
.map((id) => graph.nodes.get(id))
.filter((node): node is ResolvedSemanticNode => node !== undefined)
.sort((a, b) => a.sourceOrder - b.sourceOrder),
);
}
function recoveredNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
): Map<SemanticNodeId, VomNode> {
const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
const provisional = nodes.map((node) => ({
...node.vom,
id: numericId.get(node.id) as number,
parentId: (() => {
const parent = initialParentId(graph, node);
return parent ? (numericId.get(parent) ?? null) : null;
})(),
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
frameId: node.frameId,
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
referenceable: node.referenceable,
}));
return new Map(
applyVomInteractionRecovery(provisional).flatMap((node) => {
const id = semanticId.get(node.id);
return id ? [[id, node] as const] : [];
}),
);
}
function redundantSourceNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
): Set<SemanticNodeId> {
const byKey = new Map<string, ResolvedSemanticNode[]>();
for (const node of nodes) {
const output = recovered.get(node.id) as VomNode;
const name = output.name?.trim().toLowerCase() ?? "";
if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
const matches = byKey.get(key) ?? [];
matches.push(node);
byKey.set(key, matches);
}
const redundant = new Set<SemanticNodeId>();
for (const matches of byKey.values()) {
const axOnly = matches.filter((node) => node.ax && !node.dom);
const domOnly = matches.filter((node) => node.dom && !node.ax);
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
redundant.add(domOnly[0].id);
}
}
return redundant;
}
function baseDecision(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
output: VomNode,
frameOwnerIds: ReadonlySet<SemanticNodeId>,
redundant: ReadonlySet<SemanticNodeId>,
): StructureDecision {
if (node.excluded) return { kind: "excluded", reason: "excluded" };
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) {
return { kind: "excluded", reason: "non-rendered-dom-fallback" };
}
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
return { kind: "transparent", reason: "child-document-root" };
}
if (node.ax?.ignored === true && !node.dom) {
return { kind: "transparent", reason: "ax-ignored" };
}
const role = output.role?.toLowerCase() ?? "";
if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
if (
output.name ||
output.text ||
output.value ||
output.placeholder ||
hasMeaningfulRelations(node) ||
(output.rect && ["fixed", "sticky"].includes(output.position))
) {
return { kind: "keep", reason: "semantic" };
}
return { kind: "transparent", reason: "wrapper" };
}
function buildChildren(
nodes: ResolvedSemanticNode[],
relation: "axParentId" | "domParentId",
): Map<SemanticNodeId, SemanticNodeId[]> {
const children = new Map<SemanticNodeId, SemanticNodeId[]>();
for (const node of nodes) {
const parentId = node[relation];
if (!parentId) continue;
const siblings = children.get(parentId) ?? [];
siblings.push(node.id);
children.set(parentId, siblings);
}
return children;
}
function explicitDomAncestors(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const inside = new Set<SemanticNodeId>();
const roots = nodes.filter((node) => !node.domParentId);
const queue = roots.map((node) => node.id);
for (let index = 0; index < queue.length; index += 1) {
const parentId = queue[index];
const parent = graph.nodes.get(parentId);
const parentIsBoundary =
parent !== undefined &&
explicitStructural(parent) &&
!ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
for (const childId of domChildren.get(parentId) ?? []) {
if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
queue.push(childId);
}
}
return inside;
}
function reliableGroupCandidates(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
const remainingChildren = new Map(
nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
);
const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
const queue = nodes
.filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
.map((node) => node.id);
for (let index = 0; index < queue.length; index += 1) {
const nodeId = queue[index];
const node = graph.nodes.get(nodeId);
const output = recovered.get(nodeId);
if (!node || !output) continue;
const role = output.role?.toLowerCase() ?? "";
const excluded = decisions.get(nodeId)?.kind === "excluded";
const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
const summary = excluded
? { blocked: true, interactions: 0 }
: isVomReferenceNode(output)
? { blocked: false, interactions: 1 }
: blocked
? { blocked: true, interactions: 0 }
: (domChildren.get(nodeId) ?? []).reduce(
(combined, childId) => {
const child = summaries.get(childId);
return {
blocked: combined.blocked || !child || child.blocked,
interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
};
},
{ blocked: false, interactions: 0 },
);
summaries.set(nodeId, summary);
const parentId = node.domParentId;
if (!parentId) continue;
const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
remainingChildren.set(parentId, remaining);
if (remaining === 0) queue.push(parentId);
}
const candidates = new Set<SemanticNodeId>();
for (const node of nodes) {
const output = recovered.get(node.id) as VomNode;
const role = output.role?.toLowerCase() ?? "";
if (
!node.dom ||
node.dom.tag.toLowerCase() !== "div" ||
!TRANSPARENT_ROLES.has(role) ||
insideExplicitStructure.has(node.id) ||
decisions.get(node.id)?.kind === "excluded" ||
node.dom.attrs.hidden !== undefined ||
(node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
node.dom.attrs.inert !== undefined
) {
continue;
}
const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
const blocked = branches.some((summary) => !summary || summary.blocked);
const contributingBranches = branches.filter(
(summary) => (summary?.interactions ?? 0) > 0,
).length;
const interactions = branches.reduce(
(count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
0,
);
if (!blocked && contributingBranches >= 2 && interactions >= 2) {
candidates.add(node.id);
}
}
return candidates;
}
function leafCandidates(
nodes: ResolvedSemanticNode[],
candidates: ReadonlySet<SemanticNodeId>,
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const parentById = new Map(
nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
);
const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
const containsCandidate = new Set<SemanticNodeId>();
const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
const selected = new Set<SemanticNodeId>();
for (let index = 0; index < queue.length; index += 1) {
const nodeId = queue[index];
if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
const parentId = parentById.get(nodeId);
if (!parentId) continue;
if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
const remaining = (childCount.get(parentId) ?? 1) - 1;
childCount.set(parentId, remaining);
if (remaining === 0) queue.push(parentId);
}
return selected;
}
function nearestKept(
graph: ResolvedSemanticGraph,
startId: SemanticNodeId | undefined,
relation: "axParentId" | "domParentId",
kept: ReadonlySet<SemanticNodeId>,
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
let currentId = startId;
const path: SemanticNodeId[] = [];
const seen = new Set<SemanticNodeId>();
let result: SemanticNodeId | undefined;
while (currentId && !seen.has(currentId)) {
if (kept.has(currentId)) {
result = currentId;
break;
}
if (memo.has(currentId)) {
result = memo.get(currentId);
break;
}
seen.add(currentId);
path.push(currentId);
currentId = graph.nodes.get(currentId)?.[relation];
}
for (const id of path) memo.set(id, result);
return result;
}
function finalParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
kept: ReadonlySet<SemanticNodeId>,
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
const axNode = axParent ? graph.nodes.get(axParent) : undefined;
const domNode = domParent ? graph.nodes.get(domParent) : undefined;
const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
const domIsStructural =
domNode !== undefined &&
(explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
if (axIsExplicit) return axParent;
if (domIsStructural) return domParent;
if (axParent && !axIsRoot) return axParent;
if (domParent) return domParent;
if (axParent) return axParent;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
const {
id: _id,
parentId: _parentId,
backendNodeId: _backendNodeId,
frameId: _frameId,
contextScopeId: _contextScopeId,
referenceable: _referenceable,
...vom
} = node;
if (!inferredGroup) return vom;
const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
return { ...groupVom, role: "group" };
}
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
const nodes = orderedNodes(graph);
const recovered = recoveredNodes(graph, nodes);
const redundant = redundantSourceNodes(graph, nodes, recovered);
const frameOwnerIds = new Set(
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
);
const decisions = new Map<SemanticNodeId, StructureDecision>();
for (const node of nodes) {
decisions.set(
node.id,
baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
);
}
const domChildren = buildChildren(nodes, "domParentId");
const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
decisions.set(nodeId, {
kind: "inferred-group",
reason: "independent-interaction-branches",
});
}
const kept = new Set(
nodes
.filter((node) => {
const kind = decisions.get(node.id)?.kind;
return kind === "keep" || kind === "inferred-group";
})
.map((node) => node.id),
);
const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
for (const node of nodes) {
const structure = decisions.get(node.id) as StructureDecision;
const inferredGroup = structure.kind === "inferred-group";
structuredNodes.set(node.id, {
...node,
vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
structure,
...(kept.has(node.id)
? {
semanticParentId: finalParentId(
graph,
node,
kept,
decisions,
axParentMemo,
domParentMemo,
),
}
: {}),
});
}
return { ...graph, nodes: structuredNodes };
}
@@ -1,88 +0,0 @@
import type { Viewport, VomNode } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { CapturedNode } from "../capture";
export interface SemanticAxNode {
nodeId: string;
frameId?: string;
parentId?: string;
backendDOMNodeId?: number;
ignored?: boolean;
role?: { type: string; value?: string };
name?: { type: string; value?: string };
description?: { value?: string };
value?: { value?: string | number | boolean };
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
childIds?: string[];
}
export type SemanticNodeId = string;
export interface SemanticFrame {
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
ownerNodeId?: SemanticNodeId;
contextScopeId: string;
target: CdpTarget;
url?: string;
order: number;
}
export interface SemanticGraphNode {
id: SemanticNodeId;
frameId: string;
backendNodeId?: number;
axNodeId?: string;
ax?: SemanticAxNode;
dom?: CapturedNode;
axParentId?: SemanticNodeId;
domParentId?: SemanticNodeId;
sourceOrder: number;
excluded: boolean;
}
export interface SemanticGraph {
viewport: Viewport;
rootFrameId: string;
frames: Map<string, SemanticFrame>;
nodes: Map<SemanticNodeId, SemanticGraphNode>;
nodeIdsByFrame: Map<string, SemanticNodeId[]>;
nodeByFrameBackend: Map<string, SemanticNodeId>;
}
export interface ResolvedSemanticNode extends SemanticGraphNode {
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
referenceable: boolean;
roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
}
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
}
export type StructureDecision =
| { kind: "keep"; reason: "semantic" | "frame-owner" }
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
| {
kind: "excluded";
reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback";
};
export interface StructuredSemanticNode extends ResolvedSemanticNode {
structure: StructureDecision;
semanticParentId?: SemanticNodeId;
}
export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
nodes: Map<SemanticNodeId, StructuredSemanticNode>;
}
export function frameBackendKey(frameId: string, backendNodeId: number): string {
return `${frameId}\u0000${backendNodeId}`;
}
export function frameAxKey(frameId: string, axNodeId: string): string {
return `${frameId}\u0000${axNodeId}`;
}
@@ -65,11 +65,6 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
} }
describe("performHandshake", () => { describe("performHandshake", () => {
it("advertises the protocol compatibility boundary", () => {
expect(PROTOCOL_VERSION).toBe("1.1");
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.0");
});
it("sends system.handshake with identity and both compat fields", async () => { it("sends system.handshake with identity and both compat fields", async () => {
let sentFrame: ProtocolFrame | null = null; let sentFrame: ProtocolFrame | null = null;
const transport = fakeTransport((req) => { const transport = fakeTransport((req) => {
@@ -79,7 +74,7 @@ describe("performHandshake", () => {
result: { result: {
server: "browser-skill-daemon", server: "browser-skill-daemon",
version: "0.1.0", version: "0.1.0",
protocol_version: "1.1", protocol_version: "1.0",
min_compatible_peer: "0.0.0", min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.0", min_compatible_protocol: "1.0",
}, },
@@ -134,8 +129,8 @@ describe("performHandshake", () => {
const response = { const response = {
server: "browser-skill-daemon", server: "browser-skill-daemon",
version: "0.1.0", version: "0.1.0",
protocol_version: "1.1", protocol_version: "1.0",
min_compatible_protocol: "1.1", min_compatible_protocol: "1.0",
} satisfies HandshakeResult; } satisfies HandshakeResult;
const transport = fakeTransport((req) => ({ const transport = fakeTransport((req) => ({
id: (req as { id: string }).id, id: (req as { id: string }).id,
@@ -150,7 +145,7 @@ describe("performHandshake", () => {
}); });
expect(outcome.result.min_compatible_peer).toBeUndefined(); expect(outcome.result.min_compatible_peer).toBeUndefined();
expect(outcome.result.min_compatible_protocol).toBe("1.1"); expect(outcome.result.min_compatible_protocol).toBe("1.0");
}); });
it("rejects when the daemon responds with an error", async () => { it("rejects when the daemon responds with an error", async () => {
@@ -183,9 +178,9 @@ describe("performHandshake", () => {
result: { result: {
server: "browser-skill-daemon", server: "browser-skill-daemon",
version: "0.1.0", version: "0.1.0",
protocol_version: "1.1", protocol_version: "1.0",
min_compatible_peer: "0.0.0", min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.1", min_compatible_protocol: "1.0",
}, },
}); });
emit({ emit({
@@ -193,14 +188,14 @@ describe("performHandshake", () => {
result: { result: {
server: "browser-skill-daemon", server: "browser-skill-daemon",
version: "0.1.0", version: "0.1.0",
protocol_version: "1.1", protocol_version: "1.0",
min_compatible_peer: "0.0.0", min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.1", min_compatible_protocol: "1.0",
}, },
}); });
await expect(pending).resolves.toMatchObject({ await expect(pending).resolves.toMatchObject({
result: { server: "browser-skill-daemon", protocol_version: "1.1" }, result: { server: "browser-skill-daemon", protocol_version: "1.0" },
}); });
}); });
+1 -1
View File
@@ -7,7 +7,7 @@ import type {
ResponseFrame, ResponseFrame,
} from "./types"; } from "./types";
export const PROTOCOL_VERSION = "1.1"; export const PROTOCOL_VERSION = "1.0";
/** /**
* Extension semver, injected at build time from `package.json` via * Extension semver, injected at build time from `package.json` via
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`). * Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
-7
View File
@@ -780,7 +780,6 @@ export interface SelectedOptionV3 {
export type StepV3 = export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause }) | ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "switch_tab" } & StepCommonV3)
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 }) | ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 }) | ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & { | ({ op: "fill" } & StepCommonV3 & {
@@ -820,12 +819,6 @@ export interface RecordStartParams {
tab_id?: number; tab_id?: number;
url?: string; url?: string;
purpose?: string; purpose?: string;
max_page_tokens?: number;
redact_values?: boolean;
/** Omitted means v2; `3` requests a state-linked v3 trace. */
trace_version?: number;
/** Client can decode the v3 `switch_tab` step variant. */
supports_tab_switch_steps?: boolean;
} }
export interface RecordStartResult { export interface RecordStartResult {
-1
View File
@@ -44,7 +44,6 @@
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css", "!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css", "!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css", "!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
"!evals/browser/results/**",
"!**/target/**", "!**/target/**",
"!**/.wxt/**", "!**/.wxt/**",
"!**/.output/**", "!**/.output/**",
+275 -98
View File
@@ -1,142 +1,319 @@
--- ---
name: browser-skill name: browser-skill
description: | description: |
Use when the user asks to automate their logged-in Chromium browser: visit Use when the user asks to perform browser automation tasks against their
and read pages, fill forms, scrape data, click through flows, regression-test logged-in browser: visit and read pages, fill forms, scrape data, click
a PR's UI, validate a deployed page, or operate a tab they identify. Requires through a flow, regression-test a PR's UI, validate a deployed page.
the bsk CLI and browser extension. Requires the bsk CLI installed and the browser-skill extension loaded.
--- ---
# browser-skill # browser-skill
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
are explicitly borrowed.
Do not use this skill for tasks with no browser, for extension installation, or when the user only ## When to use
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
## Required lifecycle - Open pages, read titles/text, scrape structured data from sites the user can already access
- Fill forms, click through multi-step flows, smoke-test a UI change
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
Every browser task owns a bounded session: ## When NOT to use
```text - Tasks with **no browser** involved (files, APIs, databases only)
1. bsk session start # retain the printed 4-letter session id - Installing or configuring the extension (point the user to setup docs instead)
2. bsk ... --session <id> # pass it to every session-scoped command - **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
3. bsk session stop <id> # always run on success and error paths - Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
- Replacing the user's manual browsing when they only wanted an explanation
## Prerequisites
1. `bsk` on `PATH` (Rust CLI from browser-skill)
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
## Mandatory workflow
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
```
1. bsk session start → capture the 4-letter session id printed on stdout
2. … every tool command … → always pass --session <id>
3. bsk session stop <id> → REQUIRED when done (even on error paths)
``` ```
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
When multiple browsers are connected, use `bsk browsers` and start with
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
Run `bsk doctor` when startup or transport problems persist after one retry.
## Work toward one observable goal ## Stop when the goal is met
- Derive a concrete success condition from the user's request or a supplied trace. Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
ambiguous result.
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
- If a human-only step appears or two attempts make no progress, request help instead of
brute-forcing.
With a trace, follow its semantic target information and values in order, but treat its refs as 1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the 2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
task; it does not expand the user's goal or authorize additional actions. 3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
## Observe, act, observe **With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
Use this default loop: **Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
```text ## Core interaction loop
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
```
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk click|hover|fill|select|press ... --session <id> bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk observe --session <id> # after navigation or a meaningful DOM change bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
``` ```
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also **Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
make them stale. Observe again before the next interaction.
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
Escalate page reading only as needed: When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
1. `bsk observe` for normal semantic understanding, text, controls, and refs. ## Observation priority
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
needed, obtain a fresh observation before acting on screenshot or HTML findings.
## Respect the Agent Window boundary 1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
borrowed across unrelated work.
## Ask the human when needed ## Sandbox rules
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the | Rule | Detail |
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete |------|--------|
controls can be highlighted. Use completion criteria only when the page has a clear stable success | Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
signal. | User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` ## Global flags
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
reasoning about the page or using refs.
## Command inventory | Flag | Purpose |
|------|---------|
| `--json` | Machine-readable JSON on stdout (errors too) |
| `--quiet` | Suppress informational stderr |
| `-v` / `-vv` | More verbose logging |
This list of names is complete. Never invent a command outside it; read Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
`bsk <command...> --help` for flags instead of guessing them.
```text Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms ## CLI command reference (one line each)
observe snapshot get-html screenshot console network
click hover fill select press evaluate Details and flags: **`bsk <cmd> --help`**
tab list|create|close|select|borrow|return window resize emulate
request-help record start|stop ### Diagnostics
| Command | Summary |
|---------|---------|
| `bsk status` | Connection health, connected browsers, active sessions |
| `bsk doctor` | Deep diagnostics and repair hints |
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
### Session
| Command | Summary |
|---------|---------|
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
| `bsk session stop --all` | Stop every active session |
| `bsk session list` | List active sessions |
### Window (require `--session <id>`)
| Command | Summary |
|---------|---------|
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
### Device emulation — `bsk emulate` (requires `--session <id>`)
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
```bash
bsk emulate --session <id> --device iphone-14
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
bsk emulate --session <id> --off
``` ```
Required flags that are easy to get wrong: - Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
```text ### Tabs (require `--session <id>`)
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
bsk screenshot --out <path> bsk emulate --device <preset-id> | Command | Summary |
|---------|---------|
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
| `bsk tab close <tab-id>` | Close an agent tab |
| `bsk tab select <tab-id>` | Focus an agent tab |
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
### Observation (require `--session` unless noted)
| Command | Summary |
|---------|---------|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
### Console & network debugging (read-only; require `--session`)
| Command | Summary |
|---------|---------|
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
### Navigation
| Command | Summary |
|---------|---------|
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
| `bsk navigate-back` | History back one step |
| `bsk navigate-forward` | History forward one step |
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
### Interaction
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
### Scripting & timing
| Command | Summary |
|---------|---------|
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
### Ask the human for help — `bsk request-help`
When a step needs a human (captcha, login, OTP) or you want the user to
confirm an important action, pause and ask:
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
- `--prompt` (required): what the user should do.
- `--title` (optional): custom title for the overlay panel. When omitted,
the extension shows its default localized title.
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
whenever the prompt refers to a concrete element (a button to click, a
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
user is guided straight to the right spot instead of hunting for it. For
interaction scenarios, always include the relevant target(s); reserve a
prompt with no `--target` for cases where there is genuinely no specific
element to point at (e.g. "wait for the page to finish loading").
- `--timeout` (default `5m`): how long to wait.
- `--completion-criteria` (optional): JSON success detector. Use it only
when there is a concrete post-help success signal, e.g.
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
The target tab is brought to the foreground; the page stays interactive
while the agent control mask is hidden. The call blocks until the user
explicitly acts, the timeout expires, cancellation arrives, or explicit
completion criteria match. Page reloads, SPA route changes, and captcha
refreshes do not return control by themselves. The result `outcome` is one of:
- `continued` — the user finished and clicked Done / return control (treat as confirm).
- `cancelled` — the user clicked Cancel (treat as reject/abort).
- `timed_out` — nobody acted within the timeout.
- `completed` — the explicit `--completion-criteria` matched while the user had control.
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.
`request-help` does not refresh the page model after the user returns
control. After a `continued` or `completed` result, issue a separate
observation tool call (usually `bsk snapshot --session <id>`) before using
new refs or reasoning about the post-help page state.
#### Disabling request-help (unattended mode)
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
returns immediately with `outcome="disabled"` (no overlay, no waiting,
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
retry — complete the task autonomously or stop gracefully.
### Recording — `bsk record`
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
```bash
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
``` ```
`select` matches an option's `value` attribute, not its visible label. Device preset ids are - The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
lowercase and hyphenated, such as `iphone-14`. - `--purpose` is optional context metadata; it does **not** change what gets captured.
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
- `console` and `network` provide bounded, read-only debugging evidence. ## Error handling
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
them. Use `--off` to restore the real environment.
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
and never record banking, SSO, password-manager, or other sensitive pages.
## Recover without wandering ### Exit codes (`echo $?` after `bsk …`)
- Stale ref: observe again and retry the intended action once. | Code | Meaning | What to do |
- Unknown tab or session: list current tabs/sessions; never guess identifiers. |------|---------|------------|
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful. | `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
- Unsupported command: continue with available capabilities; suggest updating only when the missing | `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
command is necessary. | `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
- Unrecoverable failure: report the blocker and stop the session in a finally-style path. | `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
The CLI's current help and error hints are authoritative for flags, parameters, and recovery Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
details.
### When to run diagnostics
| Situation | Command |
|-----------|---------|
| Before first task in a session | `bsk status` — extension connected? |
| Any failure you cannot fix in one retry | `bsk doctor` |
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
## Red lines
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
---
**More detail for any command:** `bsk <cmd> --help`
+11 -18
View File
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
} }
} }
/// `bsk doctor` check: connected browsers should speak a protocol the /// `bsk doctor` check: every connected browser should use the same
/// daemon accepts. A different protocol string is still a live /// protocol version as the daemon. Minor protocol drift is accepted by
/// connection — report Ok so agents keep working. The detail is an /// the daemon but flagged here so the user can see who needs updating
/// upgrade reminder, not a blocker. /// (M10.4).
/// ///
/// Review M2 (round-1 minor): when no browsers are connected, the /// Review M2 (round-1 minor): when no browsers are connected, the
/// check has nothing to compare against, so it now reports /// check has nothing to compare against, so it now reports
@@ -396,12 +396,13 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
}) })
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(", "); .join(", ");
CheckResult::ok( CheckResult::fail(
name, name,
format!( format!(
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}", "{} browser(s) have protocol minor drift from the daemon: {stale}",
status.version_skew_browsers.len() status.version_skew_browsers.len()
), ),
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
) )
} }
@@ -580,7 +581,7 @@ mod m2_tests {
} }
#[test] #[test]
fn browsers_check_reports_ok_when_skew_present() { fn browsers_check_reports_fail_when_skew_present() {
let status = fake_status( let status = fake_status(
vec![BrowserStatusEntry { vec![BrowserStatusEntry {
instance_id: "alpha".into(), instance_id: "alpha".into(),
@@ -604,17 +605,9 @@ mod m2_tests {
}], }],
); );
let check = check_browsers_protocol_compatible(Some(&status)); let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok); assert_eq!(check.status, CheckStatus::Fail);
assert!(check.detail.contains("alpha")); assert!(check.detail.contains("alpha"));
assert!( assert!(check.hint.is_some());
check
.detail
.contains("still usable — continue, and upgrade soon")
);
assert!(
check.hint.is_none(),
"a protocol-version note must not hint to stop"
);
} }
#[test] #[test]
@@ -642,7 +635,7 @@ mod m2_tests {
}], }],
); );
let check = check_browsers_protocol_compatible(Some(&status)); let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok); assert_eq!(check.status, CheckStatus::Fail);
assert!( assert!(
check check
.detail .detail
-1
View File
@@ -22,7 +22,6 @@ pub mod navigate;
pub mod network; pub mod network;
pub mod observe; pub mod observe;
pub mod record; pub mod record;
pub mod record_recovery;
pub mod record_state; pub mod record_state;
pub mod render_error; pub mod render_error;
pub mod screenshot; pub mod screenshot;
+68 -266
View File
@@ -1,30 +1,23 @@
//! `bsk record start|stop` — capture user actions in the Agent Window. //! `bsk record start|stop` — capture user actions in the Agent Window.
use std::path::{Path, PathBuf}; use std::fs;
use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
use anyhow::Context; use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, RecordedTrace, TRACE_VERSION_V3, RecordStopResult, RecordedTrace,
}; };
use bsk_protocol::{ErrorCode, Method};
use clap::{Args, Subcommand}; use clap::{Args, Subcommand};
use crate::cli::TOOL_IPC_TIMEOUT; use crate::cli::TOOL_IPC_TIMEOUT;
mod export;
use crate::cli::business_rpc; use crate::cli::business_rpc;
use crate::cli::ensure_daemon::ensure_daemon; use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format}; use crate::cli::error::{CliError, Format};
use crate::cli::record_recovery;
use crate::cli::record_state; use crate::cli::record_state;
use crate::cli::session::{SessionStartOptions, start_session, stop_session}; use crate::cli::session::{SessionStartOptions, start_session, stop_session};
use export::{
ExportMeta, export_with_recovery, states_dir_for_output, trace_json_path,
validate_record_output,
};
/// Max wait for the user to click 结束 in the browser (24 hours). /// Max wait for the user to click 结束 in the browser (24 hours).
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000; const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
@@ -57,8 +50,7 @@ pub struct RecordStartArgs {
pub tab_id: Option<i64>, pub tab_id: Option<i64>,
/// Navigate to this http(s) URL before recording. When omitted, defaults /// Navigate to this http(s) URL before recording. When omitted, defaults
/// to `https://example.com/`. If that page does not load, pass `--url` /// to `https://example.com/`.
/// with a site you can open in this browser.
#[arg(long)] #[arg(long)]
pub url: Option<String>, pub url: Option<String>,
@@ -66,25 +58,15 @@ pub struct RecordStartArgs {
#[arg(long)] #[arg(long)]
pub purpose: Option<String>, pub purpose: Option<String>,
/// Max VOM tokens per page observation file (default 3000). /// Output path for the trace JSON (default `./trace.json`).
#[arg(long = "max-page-tokens")] #[arg(long, default_value = "trace.json")]
pub max_page_tokens: Option<u32>,
/// Redact all form values in page observations (`[filled]` only).
#[arg(long = "redact-values")]
pub redact_values: bool,
/// Output directory for the trace bundle (default `./trace`).
/// Writes `<dir>/trace.json` and `<dir>/states/*.txt`.
#[arg(long, default_value = "trace")]
pub output: PathBuf, pub output: PathBuf,
} }
#[derive(Debug, Clone, Args)] #[derive(Debug, Clone, Args)]
pub struct RecordStopArgs { pub struct RecordStopArgs {
/// Output directory for the trace bundle (default `./trace`). /// Output path for the trace JSON (default `./trace.json`).
/// Writes `<dir>/trace.json` and `<dir>/states/*.txt`. #[arg(long, default_value = "trace.json")]
#[arg(long, default_value = "trace")]
pub output: PathBuf, pub output: PathBuf,
} }
@@ -96,7 +78,11 @@ pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> {
} }
fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> { fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> {
prepare_record_start(&args.output)?; if record_state::read().is_ok() {
return Err(CliError::Local(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
)));
}
let info = ensure_daemon().context("ensure daemon is running")?; let info = ensure_daemon().context("ensure daemon is running")?;
let session = start_session( let session = start_session(
@@ -110,12 +96,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
let start_params = RecordStartParams { let start_params = RecordStartParams {
session_id: session.session_id.clone(), session_id: session.session_id.clone(),
tab_id: args.tab_id, tab_id: args.tab_id,
url: args.url.clone(), url: args.url,
purpose: args.purpose.clone(), purpose: args.purpose.clone(),
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
}; };
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>( let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(), info.sock_path.clone(),
@@ -129,7 +111,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
Ok(result) => result, Ok(result) => result,
Err(err) => { Err(err) => {
let _ = stop_session(info.sock_path, &session.session_id); let _ = stop_session(info.sock_path, &session.session_id);
return Err(annotate_default_start_page_error(err, args.url.as_deref())); return Err(err);
} }
}; };
@@ -160,8 +142,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
// Keep write/render inside a Result so `?` cannot skip session teardown. // Keep write/render inside a Result so `?` cannot skip session teardown.
let run_result: Result<(), CliError> = match await_result { let run_result: Result<(), CliError> = match await_result {
Ok(result) => (|| { Ok(result) => (|| {
let exported = export_with_recovery(&args.output, &result.trace)?; write_trace_file(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, &exported, format) render_finish(&result.trace, &args.output, format)
})(), })(),
Err(err) => Err(err), Err(err) => Err(err),
}; };
@@ -175,12 +157,11 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
} }
fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> { fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
validate_record_output(&args.output)?; let state = record_state::read().map_err(CliError::Local)?;
let info = ensure_daemon().context("ensure daemon is running")?;
if let Ok(state) = record_state::read() { let session_id = state.session_id.clone();
let info = ensure_daemon().context("ensure daemon is running")?;
let session_id = state.session_id.clone();
let run_result: Result<(), CliError> = (|| {
let params = RecordStopParams { let params = RecordStopParams {
session_id: session_id.clone(), session_id: session_id.clone(),
}; };
@@ -191,85 +172,16 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
Some(params), Some(params),
TOOL_IPC_TIMEOUT, TOOL_IPC_TIMEOUT,
)?; )?;
write_trace_file(&args.output, &result.trace)?;
render_stop(&result, &args.output, format)
})();
let run_result: Result<(), CliError> = (|| { let session_stop_result = stop_session(info.sock_path, &session_id);
let exported = export_with_recovery(&args.output, &result.trace)?; record_state::clear();
render_stop(&result, &args.output, &exported, format)
})();
let session_stop_result = stop_session(info.sock_path, &session_id); run_result?;
record_state::clear(); session_stop_result?;
Ok(())
run_result?;
session_stop_result?;
return Ok(());
}
let Some(trace) = record_recovery::load().map_err(CliError::Local)? else {
return Err(CliError::Local(anyhow::anyhow!(
"no recording in progress; run `bsk record start` first"
)));
};
let exported = export_with_recovery(&args.output, &trace)?;
render_finish(&trace, &args.output, &exported, format)
}
/// When `record start` omitted `--url` and the default example.com page
/// never loaded, rewrite the RPC error so the CLI hint points at `--url`.
fn annotate_default_start_page_error(err: CliError, user_url: Option<&str>) -> CliError {
if user_url.is_some() {
return err;
}
let CliError::Rpc {
code,
message,
data,
source,
} = err
else {
return err;
};
if !is_default_start_page_failure(code, &message) {
return CliError::Rpc {
code,
message,
data,
source,
};
}
CliError::Rpc {
code,
message,
data: Some(serde_json::json!({
"reason": crate::cli::render_error::reason::RECORD_START_PAGE_UNREACHABLE
})),
source,
}
}
fn is_default_start_page_failure(code: ErrorCode, message: &str) -> bool {
match code {
ErrorCode::CdpFailed | ErrorCode::Timeout => true,
ErrorCode::InvalidParams => {
message.contains("restricted URL") || message.contains("about:blank")
}
_ => false,
}
}
fn prepare_record_start(output: &Path) -> Result<(), CliError> {
if record_state::read().is_ok() {
return Err(CliError::Local(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
)));
}
if record_recovery::exists() {
return Err(CliError::Local(anyhow::anyhow!(
"a previous recording was not exported; run `bsk record stop --output <dir>` to recover it first"
)));
}
validate_record_output(output)
} }
fn record_await_ipc_timeout(timeout_ms: u32) -> Duration { fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
@@ -278,66 +190,43 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15)) .unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
} }
fn render_finish( fn write_trace_file(output: &PathBuf, trace: &RecordedTrace) -> Result<(), CliError> {
trace: &RecordedTrace, let json = serde_json::to_string_pretty(trace)
output_dir: &PathBuf, .context("serialize trace JSON")
exported: &ExportMeta, .map_err(CliError::Local)?;
format: Format, if let Some(parent) = output.parent() {
) -> Result<(), CliError> { if !parent.as_os_str().is_empty() {
let trace_path = trace_json_path(output_dir); fs::create_dir_all(parent)
.with_context(|| format!("create output directory {}", parent.display()))
.map_err(CliError::Local)?;
}
}
fs::write(output, format!("{json}\n"))
.with_context(|| format!("write trace to {}", output.display()))
.map_err(CliError::Local)?;
Ok(())
}
fn render_finish(trace: &RecordedTrace, output: &PathBuf, format: Format) -> Result<(), CliError> {
match format { match format {
Format::Json => { Format::Json => {
let payload = match trace {
RecordedTrace::V3(t) => serde_json::json!({
"output": output_dir,
"trace_json": trace_path,
"trace_version": exported.trace_version,
"states_dir": exported.states_dir,
"trace": t,
"window_closed": true,
}),
RecordedTrace::V2(t) => serde_json::json!({
"output": output_dir,
"trace_json": trace_path,
"trace_version": exported.trace_version,
"states_dir": null,
"trace": t,
"window_closed": true,
}),
};
println!( println!(
"{}", "{}",
serde_json::to_string(&payload).map_err(|e| CliError::Local(anyhow::anyhow!(e)))? serde_json::to_string(&serde_json::json!({
"output": output,
"trace": trace,
"window_closed": true,
}))
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
); );
} }
Format::Human => match trace { Format::Human => {
RecordedTrace::V3(t) => { let step_count = match trace {
let states_dir = exported RecordedTrace::V2(trace) => trace.steps.len(),
.states_dir RecordedTrace::V3(trace) => trace.steps.len(),
.as_ref() };
.map(|p| p.display().to_string()) println!("saved {step_count} steps to {}", output.display());
.unwrap_or_else(|| states_dir_for_output(output_dir).display().to_string()); }
println!(
"saved {} steps to {} and {} states to {}",
t.steps.len(),
trace_path.display(),
t.states.len(),
states_dir
);
}
RecordedTrace::V2(t) => {
if exported.v2_fallback {
eprintln!(
"note: extension returned trace v2 (no page observations); update the BrowserSkill extension for v3 bundles"
);
}
println!(
"saved {} steps to {} (trace v2)",
t.steps.len(),
trace_path.display()
);
}
},
} }
Ok(()) Ok(())
} }
@@ -345,89 +234,33 @@ fn render_finish(
fn render_stop( fn render_stop(
result: &RecordStopResult, result: &RecordStopResult,
output: &PathBuf, output: &PathBuf,
exported: &ExportMeta,
format: Format, format: Format,
) -> Result<(), CliError> { ) -> Result<(), CliError> {
render_finish(&result.trace, output, exported, format) render_finish(&result.trace, output, format)
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = record_recovery::test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
f();
record_recovery::clear();
record_state::clear();
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
#[test] #[test]
fn prepare_record_start_rejects_existing_trace_json_file() { fn default_output_is_trace_json() {
with_temp_home(|| {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("trace.json");
std::fs::write(&output, "{}\n").unwrap();
let err = prepare_record_start(&output).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"{msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"{msg}"
);
});
}
#[test]
fn prepare_record_start_rejects_unexported_recovery() {
with_temp_home(|| {
crate::daemon::paths::ensure_bsk_home().unwrap();
std::fs::write(
crate::daemon::paths::record_recovery_path().unwrap(),
"{}\n",
)
.unwrap();
let err = prepare_record_start(Path::new("trace")).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("not exported") || msg.contains("recover"),
"{msg}"
);
});
}
#[test]
fn default_output_is_trace_dir() {
let args = RecordStopArgs { let args = RecordStopArgs {
output: PathBuf::from("trace"), output: PathBuf::from("trace.json"),
}; };
assert_eq!(args.output, PathBuf::from("trace")); assert_eq!(args.output, PathBuf::from("trace.json"));
} }
#[test] #[test]
fn start_args_default_output_is_trace_dir() { fn start_args_default_output_is_trace_json() {
let args = RecordStartArgs { let args = RecordStartArgs {
browser: None, browser: None,
tab_id: None, tab_id: None,
url: None, url: None,
purpose: None, purpose: None,
max_page_tokens: None, output: PathBuf::from("trace.json"),
redact_values: false,
output: PathBuf::from("trace"),
}; };
assert_eq!(args.output, PathBuf::from("trace")); assert_eq!(args.output, PathBuf::from("trace.json"));
} }
#[test] #[test]
@@ -435,35 +268,4 @@ mod tests {
let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS); let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS);
assert!(got >= Duration::from_secs(86_400)); assert!(got >= Duration::from_secs(86_400));
} }
#[test]
fn annotate_default_start_page_error_adds_url_reason() {
let err = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::CdpFailed,
message: "Page.navigate rejected: net::ERR_ABORTED".into(),
data: None,
});
let annotated = annotate_default_start_page_error(err, None);
assert_eq!(
crate::cli::render_error::reason_for_data(annotated.data()),
Some(crate::cli::render_error::reason::RECORD_START_PAGE_UNREACHABLE)
);
let info =
crate::cli::render_error::info_for_error(annotated.code().unwrap(), annotated.data());
assert!(info.hint.unwrap().contains("--url"));
}
#[test]
fn annotate_default_start_page_error_skips_when_user_passed_url() {
let err = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::CdpFailed,
message: "Page.navigate rejected: net::ERR_ABORTED".into(),
data: None,
});
let annotated = annotate_default_start_page_error(err, Some("https://www.example.org/"));
assert_eq!(
crate::cli::render_error::reason_for_data(annotated.data()),
None
);
}
} }
-976
View File
@@ -1,976 +0,0 @@
use std::collections::HashSet;
use std::fs::{self, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use anyhow::Context;
use bsk_protocol::tools::{RecordedTrace, TRACE_VERSION_V2, TRACE_VERSION_V3, TraceV2, TraceV3};
use serde::Serialize;
use crate::cli::error::CliError;
#[derive(Debug)]
pub(super) struct ExportMeta {
pub(super) states_dir: Option<PathBuf>,
pub(super) trace_version: u32,
/// True when the CLI requested v3 but the extension returned legacy v2.
pub(super) v2_fallback: bool,
}
pub(super) fn export_recorded_trace(
output_dir: &Path,
trace: &RecordedTrace,
) -> Result<ExportMeta, CliError> {
match trace {
RecordedTrace::V3(trace) => {
let states_dir = write_trace_bundle(output_dir, trace)?;
Ok(ExportMeta {
states_dir: Some(states_dir),
trace_version: TRACE_VERSION_V3,
v2_fallback: false,
})
}
RecordedTrace::V2(trace) => {
write_trace_v2(output_dir, trace)?;
Ok(ExportMeta {
states_dir: None,
trace_version: TRACE_VERSION_V2,
v2_fallback: true,
})
}
}
}
/// Save the completed Trace first, then write the bundle. Recovery is
/// removed only after the export commits so a bad `--output` cannot drop
/// a recording the extension already returned.
pub(super) fn export_with_recovery(
output_dir: &Path,
trace: &RecordedTrace,
) -> Result<ExportMeta, CliError> {
let save_result = crate::cli::record_recovery::save(trace);
match export_recorded_trace(output_dir, trace) {
Ok(meta) => {
crate::cli::record_recovery::clear();
Ok(meta)
}
Err(export_err) => {
let export_err = annotate_recovery_hint(export_err, save_result.is_ok());
if let Err(save_err) = save_result {
return Err(match export_err {
CliError::Local(inner) => CliError::Local(
inner.context(format!("also failed to save recovery data: {save_err}")),
),
other => other,
});
}
Err(export_err)
}
}
}
fn annotate_recovery_hint(err: CliError, recovery_saved: bool) -> CliError {
if !recovery_saved {
return err;
}
match err {
CliError::Local(inner) => CliError::Local(inner.context(
"export failed; the recorded trace was saved and can be recovered with `bsk record stop --output <dir>`",
)),
other => other,
}
}
fn looks_like_json_output(path: &Path) -> bool {
path.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
}
fn legacy_json_output_error(path: &Path) -> CliError {
CliError::Local(anyhow::anyhow!(
"--output {} is a JSON file path; Trace v3 writes a bundle directory \
(`<dir>/trace.json` and `<dir>/states/`), not a single file. \
Use `--output trace` or another directory.",
path.display()
))
}
/// Reject legacy `--output trace.json` (and any existing non-directory)
/// before a recording starts, so the user is not blocked only after Finish.
pub(super) fn validate_record_output(path: &Path) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
"--output {} must not be a symlink",
path.display()
))),
Ok(metadata) if metadata.is_dir() => Ok(()),
Ok(_) if looks_like_json_output(path) => Err(legacy_json_output_error(path)),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"--output {} is not a directory; Trace v3 writes a bundle directory \
(`<dir>/trace.json` and `<dir>/states/`). Use `--output trace`.",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => {
if looks_like_json_output(path) {
Err(legacy_json_output_error(path))
} else {
Ok(())
}
}
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect --output {}", path.display())),
)),
}
}
fn acquire_export_lock(output_dir: &Path) -> Result<std::fs::File, CliError> {
validate_bundle_directory(output_dir, "output bundle directory")?;
fs::create_dir_all(output_dir)
.with_context(|| format!("create output dir {}", output_dir.display()))
.map_err(CliError::Local)?;
let lock_path = output_dir.join(".bsk-record-export.lock");
let lock_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)
.with_context(|| format!("open export lock {}", lock_path.display()))
.map_err(CliError::Local)?;
fs2::FileExt::lock_exclusive(&lock_file)
.with_context(|| format!("lock export bundle {}", output_dir.display()))
.map_err(CliError::Local)?;
Ok(lock_file)
}
pub(super) fn write_trace_v2(output_dir: &Path, trace: &TraceV2) -> Result<(), CliError> {
let trace_path = trace_json_path(output_dir);
let states_dir = states_dir_for_output(output_dir);
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
let _lock = acquire_export_lock(output_dir)?;
validate_replaceable_file(&trace_path, "trace JSON")?;
let transaction_id = uuid::Uuid::new_v4();
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
fs::create_dir_all(&staging_dir)
.with_context(|| format!("create staging dir {}", staging_dir.display()))
.map_err(CliError::Local)?;
let staged_trace = staging_dir.join("trace.json");
let stage_result: Result<(), CliError> = (|| {
fs::write(&staged_trace, format!("{json}\n"))
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
.map_err(CliError::Local)?;
Ok(())
})();
if let Err(err) = stage_result {
let _ = fs::remove_dir_all(&staging_dir);
return Err(err);
}
commit_export_transaction(
&staging_dir,
&states_dir,
&HashSet::new(),
transaction_id,
vec![(
staged_trace,
trace_path,
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
)],
)?;
Ok(())
}
pub(super) fn states_dir_for_output(output_dir: &Path) -> PathBuf {
output_dir.join("states")
}
pub(super) fn trace_json_path(output_dir: &Path) -> PathBuf {
output_dir.join("trace.json")
}
fn is_canonical_state_id(id: &str) -> bool {
let Some(number) = id.strip_prefix('s') else {
return false;
};
let mut digits = number.bytes();
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
}
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
"{description} {} must not be a symlink",
path.display()
))),
Ok(metadata) if metadata.is_dir() => Ok(()),
Ok(_) if looks_like_json_output(path) => Err(legacy_json_output_error(path)),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a directory; Trace v3 writes `<dir>/trace.json` and `<dir>/states/`. Use `--output trace`.",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a regular file",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
let old_file = match fs::symlink_metadata(target) {
Ok(_) => {
fs::rename(target, backup)?;
Some(backup.to_path_buf())
}
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
Err(err) => return Err(err),
};
if let Err(err) = fs::rename(staged, target) {
if let Some(old_file) = old_file.as_deref() {
if let Err(restore_err) = fs::rename(old_file, target) {
return Err(io::Error::new(
restore_err.kind(),
format!(
"{err}; additionally failed to restore {}: {restore_err}",
target.display()
),
));
}
}
return Err(err);
}
Ok(old_file)
}
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
let mut rollback_error = None;
for (target, backup) in installed.iter().rev() {
match fs::remove_file(target) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
rollback_error.get_or_insert(err);
continue;
}
}
if let Some(backup) = backup {
if let Err(err) = fs::rename(backup, target) {
rollback_error.get_or_insert(err);
}
}
}
match rollback_error {
Some(err) => Err(err),
None => Ok(()),
}
}
fn is_generated_state_name(name: &str) -> bool {
name.strip_suffix(".txt").is_some_and(is_canonical_state_id)
}
fn restore_staged_stale_states(staged: &[(PathBuf, PathBuf)]) -> io::Result<()> {
let mut restore_error = None;
for (original, backup) in staged.iter().rev() {
if let Err(err) = fs::rename(backup, original) {
restore_error.get_or_insert(err);
}
}
match restore_error {
Some(err) => Err(err),
None => Ok(()),
}
}
/// Move stale generated states aside so replacement and cleanup form one
/// rollback boundary. User-owned files, including arbitrary non-`sN.txt`, stay.
fn stage_stale_states(
states_dir: &Path,
keep: &HashSet<String>,
transaction_id: uuid::Uuid,
) -> Result<Vec<(PathBuf, PathBuf)>, CliError> {
let entries = match fs::read_dir(states_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(err) => {
return Err(CliError::Local(
anyhow::Error::new(err)
.context(format!("read states dir {}", states_dir.display())),
));
}
};
let mut staged = Vec::new();
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(err) => {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("read stale state entry: {err}"),
Err(restore_err) => {
format!("read stale state entry: {err}; restore also failed: {restore_err}")
}
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !is_generated_state_name(name) || keep.contains(name) {
continue;
}
let original = entry.path();
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(err) => {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("inspect stale state {}: {err}", original.display()),
Err(restore_err) => format!(
"inspect stale state {}: {err}; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
if file_type.is_dir() {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("stale state {} is a directory", original.display()),
Err(restore_err) => format!(
"stale state {} is a directory; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
let backup = states_dir.join(format!(".{name}.{transaction_id}.stale"));
if let Err(err) = fs::rename(&original, &backup) {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("stage stale state {}: {err}", original.display()),
Err(restore_err) => format!(
"stage stale state {}: {err}; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
staged.push((original, backup));
}
Ok(staged)
}
fn commit_export_transaction(
staging_dir: &Path,
states_dir: &Path,
keep_states: &HashSet<String>,
transaction_id: uuid::Uuid,
replacements: Vec<(PathBuf, PathBuf, PathBuf)>,
) -> Result<(), CliError> {
let mut installed = Vec::with_capacity(replacements.len());
for (staged, target, backup) in replacements {
match commit_staged_file(&staged, &target, &backup) {
Ok(old_file) => installed.push((target, old_file)),
Err(err) => {
let rollback_result = rollback_installed_files(&installed);
let _ = fs::remove_dir_all(staging_dir);
let message = match rollback_result {
Ok(()) => format!("commit replacement {}: {err}", target.display()),
Err(rollback_err) => format!(
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
target.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
}
}
let stale_states = match stage_stale_states(states_dir, keep_states, transaction_id) {
Ok(stale) => stale,
Err(err) => {
let rollback_result = rollback_installed_files(&installed);
let _ = fs::remove_dir_all(staging_dir);
let message = match rollback_result {
Ok(()) => err.to_string(),
Err(rollback_err) => {
format!("{err}; replacement rollback also failed: {rollback_err}")
}
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
for (_, backup) in &installed {
if let Some(backup) = backup {
let _ = fs::remove_file(backup);
}
}
for (_, backup) in stale_states {
let _ = fs::remove_file(backup);
}
let _ = fs::remove_dir_all(staging_dir);
Ok(())
}
#[derive(Serialize)]
struct BundleState<'a> {
id: &'a str,
url: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<&'a str>,
page: String,
#[serde(skip_serializing_if = "std::ops::Not::not")]
truncated: bool,
}
#[derive(Serialize)]
struct BundleTrace<'a> {
version: u32,
#[serde(skip_serializing_if = "Option::is_none")]
purpose: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
started_at: Option<&'a str>,
recorded_at: &'a str,
stopped_by: &'a bsk_protocol::tools::StopReason,
entry: &'a bsk_protocol::tools::TraceEntry,
recorder: &'a bsk_protocol::tools::RecorderInfo,
states: Vec<BundleState<'a>>,
steps: &'a [bsk_protocol::tools::StepV3],
}
pub(super) fn write_trace_bundle(output_dir: &Path, trace: &TraceV3) -> Result<PathBuf, CliError> {
let states_dir = states_dir_for_output(output_dir);
let trace_path = trace_json_path(output_dir);
if trace.version != TRACE_VERSION_V3 {
return Err(CliError::Local(anyhow::anyhow!(
"trace version {} is not supported (expected {})",
trace.version,
TRACE_VERSION_V3
)));
}
let mut state_ids = HashSet::with_capacity(trace.states.len());
for state in &trace.states {
if !is_canonical_state_id(&state.id) {
return Err(CliError::Local(anyhow::anyhow!(
"state id {:?} is not canonical (expected s1, s2, ...)",
state.id
)));
}
if !state_ids.insert(state.id.as_str()) {
return Err(CliError::Local(anyhow::anyhow!(
"duplicate state id {:?}",
state.id
)));
}
}
let mut disk_states = Vec::with_capacity(trace.states.len());
let mut state_writes = Vec::new();
let mut keep = HashSet::new();
for state in &trace.states {
let filename = format!("{}.txt", state.id);
let state_path = states_dir.join(&filename);
if state_path.parent() != Some(states_dir.as_path()) {
return Err(CliError::Local(anyhow::anyhow!(
"state path {} escapes states directory {}",
state_path.display(),
states_dir.display()
)));
}
keep.insert(filename.clone());
state_writes.push((filename.clone(), state.body.as_str()));
disk_states.push(BundleState {
id: &state.id,
url: &state.url,
title: state.title.as_deref(),
page: filename,
truncated: state.truncated,
});
}
let disk_trace = BundleTrace {
version: trace.version,
purpose: trace.purpose.as_deref(),
started_at: trace.started_at.as_deref(),
recorded_at: &trace.recorded_at,
stopped_by: &trace.stopped_by,
entry: &trace.entry,
recorder: &trace.recorder,
states: disk_states,
steps: &trace.steps,
};
let json = serde_json::to_string_pretty(&disk_trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
validate_bundle_directory(output_dir, "output bundle directory")?;
fs::create_dir_all(output_dir)
.with_context(|| format!("create output dir {}", output_dir.display()))
.map_err(CliError::Local)?;
validate_bundle_directory(&states_dir, "states directory")?;
let _lock = acquire_export_lock(output_dir)?;
fs::create_dir_all(&states_dir)
.with_context(|| format!("create states dir {}", states_dir.display()))
.map_err(CliError::Local)?;
for (filename, _) in &state_writes {
validate_replaceable_file(&states_dir.join(filename), "state observation")?;
}
validate_replaceable_file(&trace_path, "trace JSON")?;
let transaction_id = uuid::Uuid::new_v4();
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
let staging_states = staging_dir.join("states");
fs::create_dir_all(&staging_states)
.with_context(|| format!("create staging dir {}", staging_states.display()))
.map_err(CliError::Local)?;
let stage_result: Result<(), CliError> = (|| {
for (filename, body) in &state_writes {
let staged_path = staging_states.join(filename);
fs::write(&staged_path, body)
.with_context(|| {
format!("write staged state observation {}", staged_path.display())
})
.map_err(CliError::Local)?;
}
let staged_trace = staging_dir.join("trace.json");
fs::write(&staged_trace, format!("{json}\n"))
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
.map_err(CliError::Local)?;
Ok(())
})();
if let Err(err) = stage_result {
let _ = fs::remove_dir_all(&staging_dir);
return Err(err);
}
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
Vec::with_capacity(state_writes.len() + 1);
for (filename, _) in &state_writes {
replacements.push((
staging_states.join(filename),
states_dir.join(filename),
states_dir.join(format!(".{filename}.{transaction_id}.bak")),
));
}
replacements.push((
staging_dir.join("trace.json"),
trace_path.clone(),
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
));
commit_export_transaction(
&staging_dir,
&states_dir,
&keep,
transaction_id,
replacements,
)?;
Ok(states_dir)
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
use bsk_protocol::tools::{
NavigationCause, PageRefV2 as PageRef, RecorderInfo, StepCommonV3 as StepCommon,
StepResultV3 as StepResult, StepV3 as Step, StopReason, TRACE_VERSION_V2, TRACE_VERSION_V3,
TraceEntry, TraceStateV3, TraceV2, TraceV3, VOM_FORMAT_VERSION,
};
fn sample_trace(state_id: &str, body: &str) -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: state_id.into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: body.into(),
truncated: false,
}],
steps: vec![Step::Navigate {
common: StepCommon {
id: 1,
state: state_id.into(),
result: StepResult {
state: state_id.into(),
},
},
to: "https://example.com/".into(),
cause: NavigationCause::UserTyped,
}],
purpose: None,
started_at: None,
}
}
#[test]
fn write_trace_bundle_writes_dir_layout() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("s1", "@vom 1\nL1 page");
let states_dir = write_trace_bundle(&output, &trace).unwrap();
let trace_path = output.join("trace.json");
assert!(trace_path.exists());
assert_eq!(states_dir, output.join("states"));
assert!(states_dir.join("s1.txt").exists());
let disk: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
assert_eq!(disk["states"][0]["page"], "s1.txt");
assert!(disk["states"][0].get("body").is_none());
assert_eq!(
fs::read_to_string(states_dir.join("s1.txt")).unwrap(),
"@vom 1\nL1 page"
);
// Re-exporting drops states from the previous run but keeps the
// directory in place for the concurrent writer.
fs::write(states_dir.join("s9.txt"), "stale").unwrap();
fs::write(states_dir.join("notes.md"), "keep me").unwrap();
fs::write(states_dir.join("reference.txt"), "user file").unwrap();
write_trace_bundle(&output, &trace).unwrap();
assert!(!states_dir.join("s9.txt").exists());
assert!(states_dir.join("s1.txt").exists());
assert!(states_dir.join("notes.md").exists());
assert_eq!(
fs::read_to_string(states_dir.join("reference.txt")).unwrap(),
"user file"
);
}
#[test]
fn write_trace_bundle_rejects_traversal_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("state id"));
assert!(!output.join("escaped.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[test]
fn write_trace_bundle_rejects_duplicate_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
trace.states.push(TraceStateV3 {
body: "@vom 1\nL1 duplicate".into(),
..trace.states[0].clone()
});
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("duplicate state id"));
assert!(!output.exists());
}
#[test]
fn unsupported_version_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
let mut replacement = sample_trace("s2", "replacement");
replacement.version += 1;
let err = write_trace_bundle(&output, &replacement).unwrap_err();
assert!(err.to_string().contains("not supported"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(!output.join("states/s2.txt").exists());
}
#[test]
fn state_replacement_failure_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::write(output.join("states/notes.md"), "keep me").unwrap();
fs::create_dir(output.join("states/s2.txt")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
assert!(err.to_string().contains("state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert_eq!(
fs::read_to_string(output.join("states/notes.md")).unwrap(),
"keep me"
);
}
#[test]
fn stale_state_cleanup_failure_rolls_back_replacement() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::create_dir(output.join("states/s9.txt")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
assert!(err.to_string().contains("stale state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(!output.join("states/s2.txt").exists());
assert!(output.join("states/s9.txt").is_dir());
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_states_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let external_states = dir.path().join("external-states");
fs::create_dir_all(&output).unwrap();
fs::create_dir_all(&external_states).unwrap();
fs::write(external_states.join("s9.txt"), "external").unwrap();
symlink(&external_states, output.join("states")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_states.join("s9.txt")).unwrap(),
"external"
);
assert!(!external_states.join("s1.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_output_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let external_output = dir.path().join("external-output");
let output = dir.path().join("flow");
fs::create_dir_all(&external_output).unwrap();
fs::write(external_output.join("notes.md"), "external").unwrap();
symlink(&external_output, &output).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_output.join("notes.md")).unwrap(),
"external"
);
assert!(!external_output.join("trace.json").exists());
assert!(!external_output.join("states").exists());
}
#[test]
fn write_trace_v2_writes_single_json_and_prunes_stale_states() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
fs::create_dir_all(output.join("states")).unwrap();
fs::write(output.join("states/s1.txt"), "stale").unwrap();
let trace = TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![PageRef {
id: "p1".into(),
url: "https://example.com/".into(),
title: None,
}],
steps: vec![],
};
write_trace_v2(&output, &trace).unwrap();
assert!(output.join("trace.json").exists());
assert!(!output.join("states/s1.txt").exists());
}
#[test]
fn write_trace_v2_stale_cleanup_failure_rolls_back_trace() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::create_dir(output.join("states/s9.txt")).unwrap();
let trace = TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![],
steps: vec![],
};
let err = write_trace_v2(&output, &trace).unwrap_err();
assert!(err.to_string().contains("stale state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(output.join("states/s9.txt").is_dir());
}
#[test]
fn export_recorded_trace_accepts_v2() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = RecordedTrace::V2(TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![],
steps: vec![],
});
let exported = export_recorded_trace(&output, &trace).unwrap();
assert_eq!(exported.trace_version, TRACE_VERSION_V2);
assert!(exported.states_dir.is_none());
assert!(output.join("trace.json").exists());
}
#[test]
fn existing_trace_json_file_is_rejected_with_bundle_usage() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("trace.json");
fs::write(&output, "{}\n").unwrap();
let err = validate_record_output(&output).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"should name the legacy JSON file usage: {msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"should hint at the new bundle directory usage: {msg}"
);
assert_eq!(fs::read_to_string(&output).unwrap(), "{}\n");
}
#[test]
fn missing_trace_json_path_is_rejected_as_legacy_output() {
let err = validate_record_output(Path::new("trace.json")).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"{msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"{msg}"
);
}
#[test]
fn export_failure_keeps_recovery_for_retry() {
let _lock = crate::cli::record_recovery::test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
let dir = tempfile::tempdir().unwrap();
let bad_output = dir.path().join("trace.json");
fs::write(&bad_output, "{}\n").unwrap();
let trace = RecordedTrace::V3(sample_trace("s1", "@vom 1\nL1 page"));
let err = export_with_recovery(&bad_output, &trace).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("recover") || msg.contains("record stop"),
"export failure should tell the user how to recover: {msg}"
);
assert!(
crate::cli::record_recovery::exists(),
"recovery data must survive a failed bundle export"
);
let recovered = crate::cli::record_recovery::load()
.unwrap()
.expect("recovery file");
assert_eq!(recovered, trace);
assert_eq!(fs::read_to_string(&bad_output).unwrap(), "{}\n");
let good_output = dir.path().join("bundle");
export_with_recovery(&good_output, &recovered).unwrap();
assert!(good_output.join("trace.json").exists());
assert!(good_output.join("states/s1.txt").exists());
assert!(
!crate::cli::record_recovery::exists(),
"successful export must delete recovery data"
);
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
}
-112
View File
@@ -1,112 +0,0 @@
//! Persist a completed Trace so a failed bundle export can be retried.
use std::fs;
use anyhow::{Context, Result};
use bsk_protocol::tools::RecordedTrace;
use crate::daemon::paths;
pub fn save(trace: &RecordedTrace) -> Result<()> {
let path = paths::record_recovery_path()?;
paths::ensure_bsk_home()?;
let json = serde_json::to_string_pretty(trace).context("serialize record recovery")?;
let tmp = path.with_extension("tmp");
fs::write(&tmp, format!("{json}\n")).with_context(|| format!("write {}", tmp.display()))?;
fs::rename(&tmp, &path).with_context(|| format!("commit {}", path.display()))?;
Ok(())
}
pub fn load() -> Result<Option<RecordedTrace>> {
let path = paths::record_recovery_path()?;
if !path.exists() {
return Ok(None);
}
let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
let trace = serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display()))?;
Ok(Some(trace))
}
pub fn exists() -> bool {
paths::record_recovery_path()
.map(|path| path.exists())
.unwrap_or(false)
}
pub fn clear() {
if let Ok(path) = paths::record_recovery_path() {
let _ = fs::remove_file(path);
}
}
#[cfg(test)]
pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::daemon::paths::test_env_lock()
}
#[cfg(test)]
mod tests {
use super::*;
use bsk_protocol::tools::{
RecorderInfo, StopReason, TRACE_VERSION_V3, TraceEntry, TraceStateV3, TraceV3,
VOM_FORMAT_VERSION,
};
fn sample_trace() -> RecordedTrace {
RecordedTrace::V3(TraceV3 {
version: TRACE_VERSION_V3,
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
purpose: None,
started_at: None,
})
}
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
f();
clear();
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
#[test]
fn save_load_round_trips() {
with_temp_home(|| {
let trace = sample_trace();
save(&trace).unwrap();
assert!(exists());
assert_eq!(load().unwrap(), Some(trace));
});
}
#[test]
fn clear_removes_recovery_file() {
with_temp_home(|| {
save(&sample_trace()).unwrap();
clear();
assert!(!exists());
assert_eq!(load().unwrap(), None);
});
}
}
+7 -1
View File
@@ -59,9 +59,15 @@ fn started_at_unix_ms() -> String {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use std::sync::Mutex;
fn env_guard() -> &'static Mutex<()> {
static GUARD: Mutex<()> = Mutex::new(());
&GUARD
}
fn with_temp_home<F: FnOnce()>(f: F) { fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = crate::daemon::paths::test_env_lock(); let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
unsafe { unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path()); std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
-27
View File
@@ -50,7 +50,6 @@ pub mod reason {
pub const BORROW_CONFLICT: &str = "borrow_conflict"; pub const BORROW_CONFLICT: &str = "borrow_conflict";
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed"; pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY; pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
} }
/// Per-code rendering metadata. The `summary` is what the CLI prints /// Per-code rendering metadata. The `summary` is what the CLI prints
@@ -270,16 +269,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
), ),
exit_code: base.exit_code, exit_code: base.exit_code,
}, },
(
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
reason::RECORD_START_PAGE_UNREACHABLE,
) => RenderInfo {
summary: "default start page https://example.com/ could not be opened",
hint: Some(
"that site may be unreachable from your network. Pass --url with a page you can open, e.g. `bsk record start --url https://<site>/`",
),
exit_code: base.exit_code,
},
_ => base, _ => base,
} }
} }
@@ -389,22 +378,6 @@ mod tests {
assert_eq!(info.exit_code, 4); assert_eq!(info.exit_code, 4);
} }
#[test]
fn record_start_page_unreachable_overrides_cdp_failed_copy() {
let data = serde_json::json!({ "reason": reason::RECORD_START_PAGE_UNREACHABLE });
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
assert_eq!(
info.summary,
"default start page https://example.com/ could not be opened"
);
assert!(
info.hint.unwrap().contains("--url"),
"expected --url hint, got {:?}",
info.hint
);
assert_eq!(info.exit_code, 3);
}
#[test] #[test]
fn screenshot_capture_failed_overrides_cdp_failed_copy() { fn screenshot_capture_failed_overrides_cdp_failed_copy() {
let data = serde_json::json!({ "reason": reason::SCREENSHOT_CAPTURE_FAILED }); let data = serde_json::json!({ "reason": reason::SCREENSHOT_CAPTURE_FAILED });
+2 -2
View File
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
let bold = "\x1b[1m"; let bold = "\x1b[1m";
let reset = "\x1b[0m"; let reset = "\x1b[0m";
eprintln!( eprintln!(
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}", "{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
s.version_skew_browsers.len() s.version_skew_browsers.len()
); );
for skew in &s.version_skew_browsers { for skew in &s.version_skew_browsers {
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
skew.label.clone() skew.label.clone()
}; };
eprintln!( eprintln!(
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade", " {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
skew.instance_id, skew.instance_id,
label, label,
display_protocol(&skew.client_protocol_version), display_protocol(&skew.client_protocol_version),
+9 -21
View File
@@ -100,14 +100,6 @@ pub fn record_session_path() -> Result<PathBuf> {
Ok(bsk_home()?.join("record-session.json")) Ok(bsk_home()?.join("record-session.json"))
} }
/// Path to a completed Trace saved before bundle export (`record-recovery.json`).
///
/// Kept until export succeeds so a failed `--output` write cannot drop the
/// recording the extension already returned.
pub fn record_recovery_path() -> Result<PathBuf> {
Ok(bsk_home()?.join("record-recovery.json"))
}
/// Windows named-pipe name. Include the resolved `BSK_HOME` path in the /// Windows named-pipe name. Include the resolved `BSK_HOME` path in the
/// token so test homes and custom installs do not share a predictable /// token so test homes and custom installs do not share a predictable
/// per-username pipe. /// per-username pipe.
@@ -127,13 +119,6 @@ pub fn pipe_name() -> String {
render_pipe_name(&user, &home) render_pipe_name(&user, &home)
} }
/// Process-wide lock for tests that mutate `BSK_HOME`.
#[cfg(test)]
pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> {
static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(());
GUARD.lock().unwrap_or_else(|e| e.into_inner())
}
/// Render the Windows named-pipe name for a user/home pair. Kept /// Render the Windows named-pipe name for a user/home pair. Kept
/// platform-agnostic so unit tests on any host can pin the output /// platform-agnostic so unit tests on any host can pin the output
/// character set. /// character set.
@@ -151,12 +136,19 @@ fn render_pipe_name(user: &str, home: &str) -> String {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use std::sync::Mutex;
use tempfile::TempDir; use tempfile::TempDir;
/// Serialise tests that mutate the global `BSK_HOME` env var.
fn env_guard() -> &'static Mutex<()> {
static GUARD: Mutex<()> = Mutex::new(());
&GUARD
}
fn with_temp_home<F: FnOnce(&Path)>(f: F) { fn with_temp_home<F: FnOnce(&Path)>(f: F) {
let _lock = test_env_lock(); let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
let tmp = TempDir::new().unwrap(); let tmp = TempDir::new().unwrap();
// SAFETY: serialised by test_env_lock above. // SAFETY: serialised by env_guard above.
unsafe { unsafe {
std::env::set_var(BSK_HOME_ENV, tmp.path().join("bsk")); std::env::set_var(BSK_HOME_ENV, tmp.path().join("bsk"));
} }
@@ -195,10 +187,6 @@ mod tests {
record_session_path().unwrap(), record_session_path().unwrap(),
home.join("record-session.json") home.join("record-session.json")
); );
assert_eq!(
record_recovery_path().unwrap(),
home.join("record-recovery.json")
);
}); });
} }
+29 -97
View File
@@ -15,6 +15,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use std::sync::Mutex; use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
@@ -26,7 +27,7 @@ use crate::daemon::{
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT}, browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
info as daemon_info, ipc, lockfile, paths, info as daemon_info, ipc, lockfile, paths,
sessions::{StopSessionError, forget_session, stop_session}, sessions::{StopSessionError, forget_session, stop_session},
state::{DaemonState, PROTOCOL_VERSION}, state::DaemonState,
ws, ws,
}; };
@@ -290,14 +291,15 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
}); });
let started_at = Instant::now(); let started_at = Instant::now();
let activity = Arc::new(Mutex::new(IpcActivityState::new(started_at))); let activity: Arc<Mutex<Instant>> = Arc::new(Mutex::new(started_at));
let active_ipc_connections = Arc::new(AtomicUsize::new(0));
let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>(); let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let status = ipc::DaemonStatus { let status = ipc::DaemonStatus {
started_at, started_at,
ws_port, ws_port,
sock_path: sock_path.clone(), sock_path: sock_path.clone(),
daemon_version: env!("CARGO_PKG_VERSION"), daemon_version: env!("CARGO_PKG_VERSION"),
protocol_version: PROTOCOL_VERSION, protocol_version: "1.0",
}; };
let handler = ipc::full_handler(status, Arc::clone(&state)); let handler = ipc::full_handler(status, Arc::clone(&state));
@@ -312,14 +314,18 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let handler = handler.clone(); let handler = handler.clone();
let ipc_open = { let ipc_open = {
let activity = activity.clone(); let activity = activity.clone();
let active = active_ipc_connections.clone();
move || { move || {
record_ipc_open(&activity); active.fetch_add(1, Ordering::SeqCst);
record_activity(&activity);
} }
}; };
let ipc_close = { let ipc_close = {
let activity = activity.clone(); let activity = activity.clone();
let active = active_ipc_connections.clone();
move || { move || {
record_ipc_close(&activity); active.fetch_sub(1, Ordering::SeqCst);
record_activity(&activity);
} }
}; };
tokio::spawn(ipc::serve( tokio::spawn(ipc::serve(
@@ -338,6 +344,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let idle_task = { let idle_task = {
let activity = activity.clone(); let activity = activity.clone();
let state = Arc::clone(&state); let state = Arc::clone(&state);
let active_ipc_connections = active_ipc_connections.clone();
let daemon_idle = cfg.daemon_idle; let daemon_idle = cfg.daemon_idle;
tokio::spawn(async move { tokio::spawn(async move {
let tick = (daemon_idle / 4).max(Duration::from_millis(250)); let tick = (daemon_idle / 4).max(Duration::from_millis(250));
@@ -345,19 +352,22 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop { loop {
ticker.tick().await; ticker.tick().await;
let ipc_is_idle = let last = match activity.lock() {
lock_activity(&activity).should_exit_for_idle(Instant::now(), daemon_idle); Ok(a) => *a,
if !ipc_is_idle { Err(p) => *p.into_inner(),
};
if last.elapsed() < daemon_idle {
continue; continue;
} }
// IPC liveness (open-connection count + idle window) is already // Bridge from M2/M3 (which only knew about connection
// enforced above by `should_exit_for_idle`, which reads // counts) to M4/M5 (browser + session registries):
// `IpcActivityState` under one lock and returns true only when // hold the daemon alive while any IPC client is
// there are zero connections AND the idle interval has elapsed. // connected, any browser is paired, or any session
// Here we additionally hold the daemon alive while any browser // is live (design §3.2).
// is paired or any session is live (design §3.2, M4/M5 if active_ipc_connections.load(Ordering::SeqCst) > 0
// registries). || !state.browsers.is_empty()
if !state.browsers.is_empty() || !state.sessions.is_empty() { || !state.sessions.is_empty()
{
continue; continue;
} }
info!( info!(
@@ -672,55 +682,10 @@ fn restart_start_args(cfg: &DaemonConfig) -> StartArgs {
} }
} }
#[derive(Debug)] fn record_activity(activity: &Arc<Mutex<Instant>>) {
struct IpcActivityState { if let Ok(mut a) = activity.lock() {
last_activity: Instant, *a = Instant::now();
active_connections: usize,
}
impl IpcActivityState {
fn new(now: Instant) -> Self {
Self {
last_activity: now,
active_connections: 0,
}
} }
fn touch(&mut self, now: Instant) {
self.last_activity = now;
}
fn opened(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_add(1);
self.last_activity = now;
}
fn closed(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_sub(1);
self.last_activity = now;
}
fn should_exit_for_idle(&self, now: Instant, idle: Duration) -> bool {
self.active_connections == 0 && now.saturating_duration_since(self.last_activity) >= idle
}
}
fn lock_activity(
activity: &Arc<Mutex<IpcActivityState>>,
) -> std::sync::MutexGuard<'_, IpcActivityState> {
activity.lock().unwrap_or_else(|poison| poison.into_inner())
}
fn record_activity(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).touch(Instant::now());
}
fn record_ipc_open(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).opened(Instant::now());
}
fn record_ipc_close(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).closed(Instant::now());
} }
/// Initialise tracing for the daemon: write a daily-rotated file in /// Initialise tracing for the daemon: write a daily-rotated file in
@@ -1103,37 +1068,4 @@ mod tests {
assert_eq!(args.session_idle, Some(Duration::from_secs(11))); assert_eq!(args.session_idle, Some(Duration::from_secs(11)));
assert_eq!(args.daemon_idle, Some(Duration::from_secs(22))); assert_eq!(args.daemon_idle, Some(Duration::from_secs(22)));
} }
#[test]
fn ipc_close_restarts_the_idle_window_atomically() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
activity.opened(started);
let closed_at = started + Duration::from_secs(30);
assert!(
!activity.should_exit_for_idle(closed_at, idle),
"an open IPC connection must hold the daemon alive"
);
activity.closed(closed_at);
assert_eq!(activity.active_connections, 0);
assert_eq!(activity.last_activity, closed_at);
assert!(!activity.should_exit_for_idle(closed_at, idle));
assert!(!activity.should_exit_for_idle(closed_at + idle - Duration::from_millis(1), idle));
assert!(activity.should_exit_for_idle(closed_at + idle, idle));
}
#[test]
fn ipc_activity_keeps_the_daemon_alive_until_a_full_idle_interval_passes() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
let request_at = started + Duration::from_secs(20);
activity.touch(request_at);
assert!(!activity.should_exit_for_idle(request_at + Duration::from_secs(9), idle));
assert!(activity.should_exit_for_idle(request_at + idle, idle));
}
} }
+1 -1
View File
@@ -16,7 +16,7 @@ use super::start::DaemonConfig;
use super::ws::WsHandle; use super::ws::WsHandle;
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION"); pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
pub const PROTOCOL_VERSION: &str = "1.1"; pub const PROTOCOL_VERSION: &str = "1.0";
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`). /// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0"; pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer` /// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
+10 -10
View File
@@ -108,12 +108,12 @@ async fn send_handshake_with_floors(
async fn handshake_ok_when_protocol_matches() { async fn handshake_ok_when_protocol_matches() {
let (handle, _sock) = spawn_daemon().await; let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await; let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake(&mut ws, "1.1", env!("CARGO_PKG_VERSION")).await; let resp = send_handshake(&mut ws, "1.0", env!("CARGO_PKG_VERSION")).await;
let result: HandshakeResult = match resp.body { let result: HandshakeResult = match resp.body {
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(), ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"), ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
}; };
assert_eq!(result.protocol_version, "1.1"); assert_eq!(result.protocol_version, "1.0");
assert_eq!( assert_eq!(
result result
.min_compatible_peer .min_compatible_peer
@@ -135,7 +135,7 @@ async fn handshake_ok_when_app_versions_differ_but_protocol_matches() {
let (handle, _sock) = spawn_daemon().await; let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await; let mut ws = open_ws(handle.ws_addr()).await;
let resp = let resp =
send_handshake_with_floors(&mut ws, "1.1", "9.9.9", Some("0.0.0"), Some("1.1")).await; send_handshake_with_floors(&mut ws, "1.0", "9.9.9", Some("0.0.0"), Some("1.0")).await;
match resp.body { match resp.body {
ResponseBody::Ok(_) => {} ResponseBody::Ok(_) => {}
other => panic!("expected ok when protocol matches, got {other:?}"), other => panic!("expected ok when protocol matches, got {other:?}"),
@@ -149,10 +149,10 @@ async fn handshake_skew_when_protocol_minor_differs() {
let mut ws = open_ws(handle.ws_addr()).await; let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors( let resp = send_handshake_with_floors(
&mut ws, &mut ws,
"1.2", "1.1",
env!("CARGO_PKG_VERSION"), env!("CARGO_PKG_VERSION"),
Some("0.0.0"), Some("0.0.0"),
Some("1.1"), Some("1.0"),
) )
.await; .await;
match resp.body { match resp.body {
@@ -210,7 +210,7 @@ async fn handshake_legacy_ext_without_protocol_floor_still_ok() {
let mut ws = open_ws(handle.ws_addr()).await; let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors( let resp = send_handshake_with_floors(
&mut ws, &mut ws,
"1.1", "1.0",
env!("CARGO_PKG_VERSION"), env!("CARGO_PKG_VERSION"),
Some("0.1.0"), Some("0.1.0"),
None, None,
@@ -235,7 +235,7 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
browser_name: "chrome".into(), browser_name: "chrome".into(),
browser_version: "131.0".into(), browser_version: "131.0".into(),
extension_version: "9.9.9".into(), extension_version: "9.9.9".into(),
extension_protocol_version: "1.2".into(), extension_protocol_version: "1.1".into(),
label: "Older".into(), label: "Older".into(),
sink: bsk::daemon::browsers::BrowserSink { tx }, sink: bsk::daemon::browsers::BrowserSink { tx },
pending: Mutex::new(bsk::daemon::browsers::Pending::default()), pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
@@ -263,8 +263,8 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
.iter() .iter()
.find(|s| s.instance_id == "skew-only-test") .find(|s| s.instance_id == "skew-only-test")
.expect("status must list our skew client"); .expect("status must list our skew client");
assert_eq!(skew.client_protocol_version, "1.2"); assert_eq!(skew.client_protocol_version, "1.1");
assert_eq!(skew.server_protocol_version, "1.1"); assert_eq!(skew.server_protocol_version, "1.0");
assert_eq!(skew.client_version, "9.9.9"); assert_eq!(skew.client_version, "9.9.9");
let entry = status let entry = status
.browsers .browsers
@@ -281,7 +281,7 @@ async fn handshake_rejects_when_local_below_peer_min_compatible_protocol() {
let mut ws = open_ws(handle.ws_addr()).await; let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors( let resp = send_handshake_with_floors(
&mut ws, &mut ws,
"1.1", "1.0",
env!("CARGO_PKG_VERSION"), env!("CARGO_PKG_VERSION"),
Some("0.0.0"), Some("0.0.0"),
Some("99.0.0"), Some("99.0.0"),
@@ -1,327 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use bsk::daemon::info::DaemonInfo;
use bsk::daemon::paths::BSK_HOME_ENV;
use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
RecordStopResult, RecordedTrace, RecorderInfo, SessionStartParams, SessionStartResult,
StopReason, TraceEntry, TraceStateV3, TraceV3, VOM_FORMAT_VERSION,
};
use bsk_protocol::{BrowserPeerInfo, Frame, Method, RequestFrame, ResponseBody, ResponseFrame};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::process::Command;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use tokio_tungstenite::tungstenite::http::Request;
use tokio_tungstenite::tungstenite::protocol::Message;
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
type Ws =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
fn bsk_bin() -> PathBuf {
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
}
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
let origin = format!("chrome-extension://{TEST_EXT_ID}");
let request = Request::builder()
.method("GET")
.uri(format!("ws://{addr}/"))
.header("Host", addr.to_string())
.header("Upgrade", "websocket")
.header("Connection", "Upgrade")
.header("Sec-WebSocket-Version", "13")
.header("Sec-WebSocket-Key", generate_key())
.header("Origin", origin)
.body(())
.unwrap();
tokio_tungstenite::connect_async(request).await.unwrap().0
}
async fn handshake(ws: &mut Ws) {
let request = RequestFrame {
id: "handshake".into(),
method: Method::SystemHandshake,
params: Some(
serde_json::to_value(HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.10".parse().unwrap(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.10".parse().unwrap()),
min_compatible_protocol: Some("1.1".into()),
label: "Record export recovery test".into(),
})
.unwrap(),
),
};
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
.await
.unwrap();
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
panic!("expected text handshake response");
};
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
let ResponseBody::Ok(value) = response.body else {
panic!("handshake failed");
};
let _: HandshakeResult = serde_json::from_value(value).unwrap();
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: 3,
purpose: Some("export recovery".into()),
started_at: Some("2026-08-12T10:00:00Z".into()),
recorded_at: "2026-08-12T10:01:00Z".into(),
stopped_by: StopReason::CliStop,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
}
}
fn run_extension(
mut ws: Ws,
record_stop_calls: Arc<AtomicUsize>,
session_stop_calls: Arc<AtomicUsize>,
) {
tokio::spawn(async move {
while let Some(Ok(Message::Text(text))) = ws.next().await {
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
continue;
};
let body = match request.method {
Method::ToolSessionStart => {
let _: SessionStartParams =
serde_json::from_value(request.params.clone().unwrap()).unwrap();
ResponseBody::Ok(
serde_json::to_value(SessionStartResult {
agent_window_id: Some(100),
})
.unwrap(),
)
}
Method::ToolRecordStop => {
record_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(
serde_json::to_value(RecordStopResult {
trace: RecordedTrace::V3(sample_trace()),
})
.unwrap(),
)
}
Method::ToolSessionStop => {
session_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(json!({}))
}
_ => ResponseBody::Err(bsk_protocol::RpcError {
code: bsk_protocol::ErrorCode::ProtocolError,
message: format!("unexpected method {:?}", request.method),
data: None,
}),
};
let response = ResponseFrame {
id: request.id,
body,
};
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
.await
.unwrap();
}
});
}
async fn start_session(sock: &Path) -> String {
#[derive(serde::Deserialize)]
struct StartResult {
session_id: String,
}
let mut client = IpcClient::connect(sock).await.unwrap();
client
.call::<(), StartResult>(
"session-start",
Method::SessionStart,
None,
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap()
.session_id
}
fn write_record_session(home: &Path, session_id: &str) {
std::fs::create_dir_all(home).unwrap();
std::fs::write(
home.join("record-session.json"),
format!("{{\"session_id\":\"{session_id}\",\"started_at\":\"1\"}}\n"),
)
.unwrap();
}
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
Command::new(bsk_bin())
.arg("record")
.arg("stop")
.arg("--output")
.arg(output)
.env(BSK_HOME_ENV, home)
.output()
.await
.unwrap()
}
async fn setup_daemon(temp: &Path) -> (bsk::daemon::DaemonHandle, PathBuf, PathBuf) {
let home = temp.join("bsk-home");
let sock = temp.join("daemon.sock");
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
.await
.unwrap();
bsk::daemon::info::write_to_path(
&DaemonInfo::now(
std::process::id(),
sock.clone(),
daemon.ws_addr().port(),
env!("CARGO_PKG_VERSION"),
),
&home.join("daemon.json"),
)
.unwrap();
(daemon, home, sock)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn existing_trace_json_file_is_rejected_before_record_stop() {
let temp = tempfile::Builder::new()
.prefix("bsk-record-json-")
.tempdir()
.unwrap();
let (daemon, home, sock) = setup_daemon(temp.path()).await;
let output = temp.path().join("trace.json");
std::fs::write(&output, "{}\n").unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
write_record_session(&home, &session_id);
let first = run_record_stop(&home, &output).await;
let stderr = String::from_utf8_lossy(&first.stderr);
assert!(
!first.status.success(),
"legacy JSON --output should fail: {stderr}"
);
assert!(
stderr.contains("JSON file") || stderr.contains("bundle directory"),
"{stderr}"
);
let state: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(home.join("record-session.json")).unwrap())
.unwrap();
assert_eq!(state["session_id"], session_id);
assert_eq!(
record_stop_calls.load(Ordering::SeqCst),
0,
"existing trace.json must be rejected before the extension returns the Trace"
);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 0);
assert_eq!(std::fs::read_to_string(&output).unwrap(), "{}\n");
daemon.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_bundle_export_can_be_recovered() {
let temp = tempfile::Builder::new()
.prefix("bsk-record-recov-")
.tempdir()
.unwrap();
let (daemon, home, sock) = setup_daemon(temp.path()).await;
let output = temp.path().join("trace");
std::fs::create_dir_all(&output).unwrap();
std::fs::write(output.join("states"), "not a directory\n").unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
write_record_session(&home, &session_id);
let first = run_record_stop(&home, &output).await;
let stderr = String::from_utf8_lossy(&first.stderr);
assert!(
!first.status.success(),
"blocked states path should fail export: {stderr}"
);
assert!(
home.join("record-recovery.json").exists(),
"completed Trace must be saved before the failed export"
);
assert!(
!home.join("record-session.json").exists(),
"session state is cleared after the extension returns the Trace"
);
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
std::fs::remove_file(output.join("states")).unwrap();
let second = run_record_stop(&home, &output).await;
assert!(
second.status.success(),
"recovery export failed: {}",
String::from_utf8_lossy(&second.stderr)
);
assert_eq!(
record_stop_calls.load(Ordering::SeqCst),
1,
"recovery must not call the extension again"
);
assert!(output.join("trace.json").exists());
assert!(output.join("states/s1.txt").exists());
assert!(
!home.join("record-recovery.json").exists(),
"successful export must delete recovery data"
);
daemon.shutdown().await;
}
-271
View File
@@ -1,271 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use bsk::cli::record_state;
use bsk::daemon::info::DaemonInfo;
use bsk::daemon::paths::BSK_HOME_ENV;
use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
RecordStopResult, RecordedTrace, RecorderInfo, SessionStartParams, SessionStartResult,
StopReason, TraceEntry, TraceStateV3, TraceV3, VOM_FORMAT_VERSION,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::process::Command;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use tokio_tungstenite::tungstenite::http::Request;
use tokio_tungstenite::tungstenite::protocol::Message;
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
type Ws =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
fn bsk_bin() -> PathBuf {
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
}
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
let origin = format!("chrome-extension://{TEST_EXT_ID}");
let request = Request::builder()
.method("GET")
.uri(format!("ws://{addr}/"))
.header("Host", addr.to_string())
.header("Upgrade", "websocket")
.header("Connection", "Upgrade")
.header("Sec-WebSocket-Version", "13")
.header("Sec-WebSocket-Key", generate_key())
.header("Origin", origin)
.body(())
.unwrap();
tokio_tungstenite::connect_async(request).await.unwrap().0
}
async fn handshake(ws: &mut Ws) {
let request = RequestFrame {
id: "handshake".into(),
method: Method::SystemHandshake,
params: Some(
serde_json::to_value(HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.10".parse().unwrap(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.10".parse().unwrap()),
min_compatible_protocol: Some("1.1".into()),
label: "Record stop retry test".into(),
})
.unwrap(),
),
};
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
.await
.unwrap();
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
panic!("expected text handshake response");
};
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
let ResponseBody::Ok(value) = response.body else {
panic!("handshake failed");
};
let _: HandshakeResult = serde_json::from_value(value).unwrap();
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: 3,
purpose: Some("retry record stop".into()),
started_at: Some("2026-08-12T10:00:00Z".into()),
recorded_at: "2026-08-12T10:01:00Z".into(),
stopped_by: StopReason::CliStop,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
}
}
fn run_extension(
mut ws: Ws,
record_stop_calls: Arc<AtomicUsize>,
session_stop_calls: Arc<AtomicUsize>,
) {
tokio::spawn(async move {
while let Some(Ok(Message::Text(text))) = ws.next().await {
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
continue;
};
let body = match request.method {
Method::ToolSessionStart => {
let _: SessionStartParams =
serde_json::from_value(request.params.clone().unwrap()).unwrap();
ResponseBody::Ok(
serde_json::to_value(SessionStartResult {
agent_window_id: Some(100),
})
.unwrap(),
)
}
Method::ToolRecordStop => {
if record_stop_calls.fetch_add(1, Ordering::SeqCst) == 0 {
ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: "recording remains active; retry record stop".into(),
data: Some(json!({ "recording": true, "retryable": true })),
})
} else {
ResponseBody::Ok(
serde_json::to_value(RecordStopResult {
trace: RecordedTrace::V3(sample_trace()),
})
.unwrap(),
)
}
}
Method::ToolSessionStop => {
session_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(json!({}))
}
_ => ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: format!("unexpected method {:?}", request.method),
data: None,
}),
};
let response = ResponseFrame {
id: request.id,
body,
};
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
.await
.unwrap();
}
});
}
async fn start_session(sock: &Path) -> String {
#[derive(serde::Deserialize)]
struct StartResult {
session_id: String,
}
let mut client = IpcClient::connect(sock).await.unwrap();
client
.call::<(), StartResult>(
"session-start",
Method::SessionStart,
None,
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap()
.session_id
}
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
Command::new(bsk_bin())
.arg("record")
.arg("stop")
.arg("--output")
.arg(output)
.env(BSK_HOME_ENV, home)
.output()
.await
.unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_record_stop_keeps_state_and_session_for_retry() {
// Keep the Unix-domain socket below macOS's SUN_LEN even from a deep worktree.
let temp = tempfile::Builder::new()
.prefix("bsk-record-stop-")
.tempdir()
.unwrap();
let home = temp.path().join("bsk-home");
let sock = temp.path().join("daemon.sock");
let output = temp.path().join("trace");
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
.await
.unwrap();
bsk::daemon::info::write_to_path(
&DaemonInfo::now(
std::process::id(),
sock.clone(),
daemon.ws_addr().port(),
env!("CARGO_PKG_VERSION"),
),
&home.join("daemon.json"),
)
.unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
unsafe {
std::env::set_var(BSK_HOME_ENV, &home);
}
record_state::write(&session_id).unwrap();
let first = run_record_stop(&home, &output).await;
assert!(
!first.status.success(),
"first stop unexpectedly succeeded: {}",
String::from_utf8_lossy(&first.stdout)
);
assert_eq!(record_state::read().unwrap().session_id, session_id);
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
assert_eq!(
session_stop_calls.load(Ordering::SeqCst),
0,
"failed record stop must leave the session active"
);
let second = run_record_stop(&home, &output).await;
assert!(
second.status.success(),
"retry failed: {}",
String::from_utf8_lossy(&second.stderr)
);
assert!(record_state::read().is_err());
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 2);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
assert!(output.join("trace.json").exists());
assert!(output.join("states/s1.txt").exists());
unsafe {
std::env::remove_var(BSK_HOME_ENV);
}
daemon.shutdown().await;
}
+1 -1
View File
@@ -61,7 +61,7 @@ fn bsk_status_json_returns_structured_payload() {
assert!(parsed["pid"].as_u64().unwrap() > 0); assert!(parsed["pid"].as_u64().unwrap() > 0);
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty()); assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
assert_eq!(parsed["protocol_version"], "1.1"); assert_eq!(parsed["protocol_version"], "1.0");
assert!(parsed["sock_path"].as_str().is_some()); assert!(parsed["sock_path"].as_str().is_some());
assert_eq!(parsed["browsers"], serde_json::json!([])); assert_eq!(parsed["browsers"], serde_json::json!([]));
assert_eq!(parsed["sessions"], serde_json::json!([])); assert_eq!(parsed["sessions"], serde_json::json!([]));
+2 -2
View File
@@ -53,7 +53,7 @@ pub async fn send_handshake(
let params = HandshakeParams { let params = HandshakeParams {
client: "browser-skill-extension".into(), client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(), version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.1".into(), protocol_version: "1.0".into(),
instance_id: instance_id.into(), instance_id: instance_id.into(),
browser: BrowserPeerInfo { browser: BrowserPeerInfo {
name: "chrome".into(), name: "chrome".into(),
@@ -91,7 +91,7 @@ async fn ws_handshake_registers_browser_in_state() {
let mut ws = connect_ext(handle.ws_addr(), &origin).await; let mut ws = connect_ext(handle.ws_addr(), &origin).await;
let result = send_handshake(&mut ws, TEST_EXT_ID).await; let result = send_handshake(&mut ws, TEST_EXT_ID).await;
assert_eq!(result.server, "browser-skill-daemon"); assert_eq!(result.server, "browser-skill-daemon");
assert_eq!(result.protocol_version, "1.1"); assert_eq!(result.protocol_version, "1.0");
let state = handle.state(); let state = handle.state();
let browsers = state.browsers.snapshot(); let browsers = state.browsers.snapshot();
@@ -480,36 +480,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
@@ -6,36 +6,15 @@
"session_id" "session_id"
], ],
"properties": { "properties": {
"max_page_tokens": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"purpose": { "purpose": {
"type": [ "type": [
"string", "string",
"null" "null"
] ]
}, },
"redact_values": {
"type": [
"boolean",
"null"
]
},
"session_id": { "session_id": {
"type": "string" "type": "string"
}, },
"supports_tab_switch_steps": {
"description": "Client can decode the v3 `switch_tab` step variant.",
"type": [
"boolean",
"null"
]
},
"tab_id": { "tab_id": {
"type": [ "type": [
"integer", "integer",
@@ -43,15 +22,6 @@
], ],
"format": "int64" "format": "int64"
}, },
"trace_version": {
"description": "Desired trace export format. Omitted means v2; `3` requests v3.",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"url": { "url": {
"type": [ "type": [
"string", "string",
@@ -480,36 +480,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
-30
View File
@@ -469,36 +469,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
@@ -431,36 +431,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
@@ -41,36 +41,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
-30
View File
@@ -173,36 +173,6 @@
} }
} }
}, },
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{ {
"description": "Fields shared by every step variant (flattened in JSON).", "description": "Fields shared by every step variant (flattened in JSON).",
"type": "object", "type": "object",
-292
View File
@@ -32,16 +32,6 @@ pub struct RecordStartParams {
pub url: Option<String>, pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>, pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_page_tokens: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub redact_values: Option<bool>,
/// Desired trace export format. Omitted means v2; `3` requests v3.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trace_version: Option<u32>,
/// Client can decode the v3 `switch_tab` step variant.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tab_switch_steps: Option<bool>,
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -277,249 +267,6 @@ mod tests {
use super::*; use super::*;
use serde_json::json; use serde_json::json;
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommonV3 {
StepCommonV3 {
id,
state: state.into(),
result: StepResultV3 {
state: result_state.into(),
},
}
}
fn sample_target() -> TargetDescriptorV3 {
TargetDescriptorV3 {
element_ref: Some("e21".into()),
role: Some("button".into()),
name: Some("发布".into()),
ctx: Some("金桔柠檬 6 号".into()),
unmatched: false,
}
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
purpose: Some("把草稿商品发布上架".into()),
started_at: Some("2026-08-10T02:10:41.080Z".into()),
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![
TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example Domain".into()),
body: "@vom 1\nRootWebArea \"Example Domain\"".into(),
truncated: false,
},
TraceStateV3 {
id: "s2".into(),
url: "https://shop.example.com/products?status=draft".into(),
title: Some("商品管理".into()),
body: "@vom 1\nRootWebArea \"商品管理\"".into(),
truncated: false,
},
],
steps: vec![
StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://shop.example.com/products?status=draft".into(),
cause: NavigationCause::UserTyped,
},
StepV3::Fill {
common: sample_common(2, "s2", "s2"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "金桔柠檬".into(),
commit: FillCommit::Enter,
redacted: false,
},
StepV3::Click {
common: sample_common(3, "s2", "s2"),
target: sample_target(),
},
],
}
}
#[test]
fn step_click_round_trips() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
assert_eq!(v["result"]["state"], "s2");
assert_eq!(v["target"]["ref"], "e21");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_with_commit_round_trips() {
let step = StepV3::Fill {
common: sample_common(2, "s2", "s3"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "browser skill".into(),
commit: FillCommit::Enter,
redacted: false,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["commit"], "enter");
assert!(v.get("redacted").is_none());
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = StepV3::Fill {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e3".into()),
role: Some("textbox".into()),
name: Some("密码".into()),
ctx: None,
unmatched: false,
},
value: "***".into(),
commit: FillCommit::Blur,
redacted: true,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_navigate_with_cause_round_trips() {
let step = StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://example.com".into(),
cause: NavigationCause::UserTyped,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["cause"], "user_typed");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
assert!(v.get("pages").is_none());
assert_eq!(v["recorder"]["vom"], 1);
assert_eq!(v["states"].as_array().unwrap().len(), 2);
let round: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
}
#[test]
fn default_fields_are_omitted() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e1".into()),
role: Some("button".into()),
name: Some("OK".into()),
ctx: None,
unmatched: false,
},
};
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("unmatched").is_none());
assert!(v["target"].get("ctx").is_none());
assert!(v["target"].get("unmatched").is_none());
}
#[test]
fn record_start_trace_options_are_explicit_and_optional() {
let legacy = RecordStartParams {
session_id: "session".into(),
tab_id: None,
url: None,
purpose: None,
max_page_tokens: None,
redact_values: None,
trace_version: None,
supports_tab_switch_steps: None,
};
let legacy_value = serde_json::to_value(legacy).unwrap();
assert!(legacy_value.get("trace_version").is_none());
assert!(legacy_value.get("max_page_tokens").is_none());
assert!(legacy_value.get("supports_tab_switch_steps").is_none());
let v3 = RecordStartParams {
session_id: "session".into(),
tab_id: None,
url: None,
purpose: None,
max_page_tokens: Some(4_000),
redact_values: Some(true),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let v3_value = serde_json::to_value(v3).unwrap();
assert_eq!(v3_value["trace_version"], TRACE_VERSION_V3);
assert_eq!(v3_value["max_page_tokens"], 4_000);
assert_eq!(v3_value["redact_values"], true);
assert_eq!(v3_value["supports_tab_switch_steps"], true);
}
#[test]
fn unmatched_target_serializes_flag() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: TargetDescriptorV3 {
element_ref: None,
role: Some("button".into()),
name: Some("发布".into()),
ctx: None,
unmatched: true,
},
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["target"]["unmatched"], true);
assert!(v["target"].get("ref").is_none());
}
#[test] #[test]
fn recorded_trace_classifies_v2_and_v3() { fn recorded_trace_classifies_v2_and_v3() {
let v2 = json!({ let v2 = json!({
@@ -705,43 +452,4 @@ mod tests {
assert!(schema["definitions"].get("StepV2").is_some()); assert!(schema["definitions"].get("StepV2").is_some());
assert!(schema["definitions"].get("StepV3").is_some()); assert!(schema["definitions"].get("StepV3").is_some());
} }
#[test]
fn extension_trace_deserializes() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/editor" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [
{ "id": "s1", "url": "https://example.com/editor", "body": "@vom 1" },
{ "id": "s2", "url": "https://example.com/p/99", "body": "@vom 1" }
],
"steps": [
{
"op": "fill",
"id": 1,
"state": "s1",
"result": { "state": "s1" },
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
"value": "hello",
"commit": "blur"
},
{
"op": "click",
"id": 2,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e2", "role": "button", "name": "发布" }
}
]
});
let trace: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(trace.version, 3);
assert_eq!(trace.states.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
} }
@@ -126,10 +126,6 @@ pub enum StepV3 {
to: String, to: String,
cause: NavigationCause, cause: NavigationCause,
}, },
SwitchTab {
#[serde(flatten)]
common: StepCommonV3,
},
Click { Click {
#[serde(flatten)] #[serde(flatten)]
common: StepCommonV3, common: StepCommonV3,
@@ -344,19 +340,6 @@ mod tests {
assert_eq!(round, step); assert_eq!(round, step);
} }
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test] #[test]
fn trace_v3_round_trips() { fn trace_v3_round_trips() {
let trace = sample_trace(); let trace = sample_trace();
Loaded 100 of 196 files, more files were not shown because too many files have changed in this diff. Show more