Compare commits

..
Author SHA1 Message Date
Ljy-0827 eeba301080 feat(vom): fixture test 2026-08-26 12:11:19 +08:00
Ljy-0827 5ca158a729 fix(vom): canvas recognition 2026-08-26 11:51:18 +08:00
Ljy-0827 ed91f41291 feat(vom): vom can distinguish important canvas area 2026-08-25 16:12:19 +08:00
Ljy-0827 b9fd6e46af fix(vom): fix repeat names 2026-08-24 14:52:16 +08:00
Ljy-0827 36135c1a2d fix(vom): bug fix 2026-08-24 14:31:35 +08:00
Ljy-0827 28d8a5f605 fix(vom): safety policy and rebase bug fix 2026-08-24 14:06:42 +08:00
Ljy-0827 54ce2f8dc3 feat(vom): fix name lost 2026-08-24 14:06:42 +08:00
Ljy-0827 b7e62d8cf1 feat(vom): function refactor 2026-08-24 14:06:42 +08:00
134 changed files with 2667 additions and 9153 deletions
-177
View File
@@ -325,180 +325,3 @@ jobs:
echo "Chrome Web Store submission created successfully:"
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
publish-edge-addons:
needs: [resolve, build-extension]
runs-on: ubuntu-latest
environment: edge-addons
permissions:
contents: read
steps:
- name: Download build artifact
uses: actions/download-artifact@v8
with:
name: extension-zip
path: dist
- name: Validate Microsoft Edge Add-ons configuration
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
run: |
set -euo pipefail
if [ -z "$API_KEY" ]; then
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
exit 1
fi
if [ -z "$CLIENT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
exit 1
fi
if [ -z "$PRODUCT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
exit 1
fi
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
exit 1
fi
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
if [ -z "$ARCHIVE" ]; then
echo "::error::No extension zip found in the downloaded artifact"
exit 1
fi
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
- name: Upload and publish Microsoft Edge Add-ons extension
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
operation_id_from_headers() {
local headers="$1"
local location
location="$(awk '
tolower($1) == "location:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
print
exit
}
' "$headers")"
location="${location%%\?*}"
printf '%s' "${location##*/}"
}
poll_operation() {
local operation_name="$1"
local status_url="$2"
local status="InProgress"
local http_status
for attempt in $(seq 1 30); do
sleep 5
http_status="$(curl --silent --show-error \
--output "$STATUS_RESPONSE" \
--write-out '%{http_code}' \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
"$status_url")"
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
exit 1
fi
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
echo "$operation_name status check $attempt/30: $status"
case "$status" in
Succeeded)
return 0
;;
InProgress)
;;
Failed)
echo "::error::Microsoft Edge Add-ons $operation_name failed"
jq . "$STATUS_RESPONSE"
exit 1
;;
*)
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
jq . "$STATUS_RESPONSE"
exit 1
;;
esac
done
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
jq . "$STATUS_RESPONSE"
exit 1
}
HTTP_STATUS="$(curl --silent --show-error \
--output "$UPLOAD_RESPONSE" \
--dump-header "$UPLOAD_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/zip" \
--data-binary "@$ARCHIVE" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
exit 1
fi
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
if [ -z "$UPLOAD_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
cat "$UPLOAD_HEADERS"
exit 1
fi
poll_operation \
"package upload" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
HTTP_STATUS="$(curl --silent --show-error \
--output "$PUBLISH_RESPONSE" \
--dump-header "$PUBLISH_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/json" \
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
"$API_ROOT/products/$PRODUCT_ID/submissions")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
exit 1
fi
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
if [ -z "$PUBLISH_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
cat "$PUBLISH_HEADERS"
exit 1
fi
poll_operation \
"submission" \
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
echo "Microsoft Edge Add-ons submission created successfully:"
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
-8
View File
@@ -7,18 +7,10 @@ dist/
.output/
.wxt/
.web-extension-id
.pnpm-store/
# Git worktrees
.worktrees/
# Logs
*.log
# Local browser evaluation artifacts
evals/browser/results/
evals/browser/agents.local.json
# macOS
.DS_Store
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.1.11"
version = "0.1.10"
dependencies = [
"anyhow",
"bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.1.11"
version = "0.1.10"
dependencies = [
"schemars",
"semver",
+1 -1
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.1.11"
version = "0.1.10"
edition = "2024"
rust-version = "1.85"
license = "MIT"
-1
View File
@@ -208,7 +208,6 @@ The repository is a Cargo + pnpm workspace:
- `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
## License
-1
View File
@@ -178,7 +178,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
## 许可证
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.1.7",
"version": "0.1.6",
"private": true,
"type": "module",
"scripts": {
@@ -247,41 +247,6 @@ describe("App", () => {
expect(copyButton.getAttribute("disabled")).not.toBeNull();
expect(screen.getByText("连接后可用")).toBeTruthy();
});
it("treats protocol drift as still connected and prompts an upgrade", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
state: "version_skew",
instanceId: "03c3e47f",
handshake: {
server: "bh",
version: mockDaemonVersion,
protocol_version: "1.0",
},
},
statusState: "version_skew",
setLabel,
setConnectionEnabled,
});
render(<App />);
expect(screen.getByText("已连接")).toBeTruthy();
expect(screen.getByText("可升级")).toBeTruthy();
expect(screen.queryByText("协议不一致")).toBeNull();
expect(screen.queryByText("Action needed")).toBeNull();
expect(screen.queryByText("兼容")).toBeNull();
const warning = screen.getByText(/协议版本不同,请及时升级/);
expect(warning.textContent).toContain("CLI 协议");
expect(warning.textContent).toContain("扩展协议");
openRecordView();
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
expect(copyButton.getAttribute("disabled")).toBeNull();
fireEvent.click(copyButton);
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
});
});
describe("control hints toggle", () => {
+3 -4
View File
@@ -79,7 +79,6 @@ export function App() {
}, [copiedTick]);
const isSkewed = statusState === "version_skew";
const connectionLive = statusState === "connected" || isSkewed;
const daemonVersion = snapshot.handshake?.version ?? "—";
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—";
@@ -91,7 +90,7 @@ export function App() {
setCopiedInstanceId(true);
};
const recordReady = connectionLive && Boolean(snapshot.instanceId);
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
const recordPurpose = purposeDraft.trim();
const recordStartUrl = startUrlDraft.trim();
const recordCommand = snapshot.instanceId
@@ -207,12 +206,12 @@ export function App() {
</div>
{isSkewed && (
<p
className="mt-2 text-xs leading-snug text-muted-foreground"
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
data-slot="popup-version-skew-warning"
>
{t("popup.versionSkewWarning", {
extensionProtocol: PROTOCOL_VERSION,
cliProtocol: daemonProtocol,
daemonProtocol,
})}
</p>
)}
@@ -13,6 +13,8 @@ describe("RefStore", () => {
backendNodeId: 42,
tabId: 7,
generation: 0,
kind: "dom",
capabilities: ["interact", "screenshot"],
});
expect(s.size()).toBe(1);
expect(s.isEmpty()).toBe(false);
@@ -48,4 +50,31 @@ describe("RefStore", () => {
cdpSessionId: "child-session",
});
});
it("preserves explicit capabilities for visual surface refs", () => {
const s = new RefStore();
s.set("e1", 42, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
expect(s.resolveEntry("e1")).toMatchObject({
kind: "surface",
capabilities: ["screenshot"],
});
});
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
const s = new RefStore();
s.set("e1", 42, { tabId: 7, kind: "surface" });
s.set("e2", 43, {
tabId: 7,
kind: "surface",
capabilities: ["interact", "screenshot"],
});
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
});
});
@@ -1,3 +1,5 @@
import type { Rect } from "@browser-skill/vom";
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
@@ -12,12 +14,17 @@
* inside the `SessionContext`.
*/
export type BackendNodeId = number;
export type RefCapability = "interact" | "screenshot";
export type RefTargetKind = "dom" | "surface";
export interface RefEntry {
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
generation: number;
}
@@ -28,8 +35,19 @@ export type RefInput =
tabId: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
};
function capabilitiesFor(
kind: RefTargetKind,
capabilities: RefCapability[] | undefined,
): RefCapability[] {
if (kind === "surface") return ["screenshot"];
return capabilities ?? ["interact", "screenshot"];
}
export class RefStore {
private readonly map = new Map<string, RefEntry>();
private generation = 0;
@@ -70,13 +88,20 @@ export class RefStore {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
} = {},
): void {
const kind = opts.kind ?? "dom";
this.map.set(normaliseRef(ref), {
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, opts.capabilities),
generation: this.generation,
});
}
@@ -94,14 +119,20 @@ export class RefStore {
return {
backendNodeId: input,
tabId: null,
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: this.generation,
};
}
const kind = input.kind ?? "dom";
return {
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, input.capabilities),
generation: this.generation,
};
}
@@ -123,6 +123,29 @@ describe("handleClick", () => {
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
const fake = makeFakeCdp({});
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({
code: "permission_denied",
data: { reason: "ref_capability_denied", required_capability: "interact" },
});
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
});
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { width: number; height: number; scale: number };
};
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
});
it("crops a visual surface screenshot to its observation-time visible region", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { x: number; y: number; width: number; height: number };
};
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
});
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
);
});
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Canvas app" },
backendDOMNodeId: 100,
};
const strings = ["body", "canvas", "static", "auto"];
const i = (value: string) => strings.indexOf(value);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("canvas")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 30, 800, 500],
],
paintOrders: [0, 1],
},
},
],
};
}
throw new Error(`unexpected CDP method ${method}`);
});
const deps = {
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
conditionalSurfaceProbe: false,
};
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
backendNodeId: 200,
kind: "surface",
capabilities: ["screenshot"],
});
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
expect(snapshot.text).not.toContain("visual surface");
expect(snapshot.ref_count).toBe(0);
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
});
@@ -53,6 +57,8 @@ describe("resolveSnapshotRef", () => {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
const expected = {
@@ -60,6 +66,8 @@ describe("resolveSnapshotRef", () => {
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom" as const,
capabilities: ["interact", "screenshot"] as const,
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
});
it("rejects an operation outside the ref's declared capabilities", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
code: "permission_denied",
message: "ref @e3 does not support interact",
data: {
reason: "ref_capability_denied",
required_capability: "interact",
kind: "surface",
},
});
});
});
+1 -1
View File
@@ -159,7 +159,7 @@ async function resolveBackendNode(
};
}
if (hasRef) {
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
if (isRpcError(resolved)) return resolved;
return {
backendNodeId: resolved.backendNodeId,
+61 -5
View File
@@ -8,6 +8,7 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
renderVom,
type VomNode,
type VomOptions,
@@ -59,6 +60,11 @@ import {
type CaptureVomObservationResult,
projectRecordSafeObservation,
} from "./vom/record-safe-observation";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "./vom/rendered-surfaces";
import {
buildSemanticGraph,
buildSemanticVomScene,
@@ -90,6 +96,9 @@ export const normaliseRef = sharedNormaliseRef;
// screenshot — `tool.screenshot`
// ---------------------------------------------------------------------------
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
/**
* Strip the `data:image/...;base64,` prefix from a Chrome
* `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -182,6 +191,8 @@ async function captureElementScreenshot(
target: CdpTarget,
backendNodeId: number,
frameId?: string,
visualSurface = false,
observedVisibleRect?: Rect,
signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
if (signal?.aborted) return cancelled("screenshot");
@@ -189,11 +200,36 @@ async function captureElementScreenshot(
cdp,
tabId,
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
{ scrollIntoView: true },
{ scrollIntoView: !visualSurface },
);
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds;
const liveRect = geometry.topBounds;
const rect =
visualSurface && observedVisibleRect
? (() => {
const x = Math.max(liveRect.x, observedVisibleRect.x);
const y = Math.max(liveRect.y, observedVisibleRect.y);
const right = Math.min(
liveRect.x + liveRect.width,
observedVisibleRect.x + observedVisibleRect.w,
);
const bottom = Math.min(
liveRect.y + liveRect.height,
observedVisibleRect.y + observedVisibleRect.h,
);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
})()
: liveRect;
if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
const scale = visualSurface
? Math.min(
1,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
)
: 1;
try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -203,7 +239,7 @@ async function captureElementScreenshot(
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1,
scale,
},
});
if (signal?.aborted) return cancelled("screenshot");
@@ -310,7 +346,7 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
const node = resolveSnapshotRef(ctx, ref, target.tabId);
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -330,6 +366,8 @@ export async function handleScreenshot(
nodeTarget,
node.backendNodeId,
node.frameId,
node.kind === "surface",
node.visibleRect,
signal,
),
deps.sendToTab,
@@ -1023,6 +1061,7 @@ async function captureForVom(
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
renderedSurfaceDiscovery?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
@@ -1064,7 +1103,20 @@ export async function captureVomObservation(
),
);
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const rendered = renderVom(decoratedScene, {
const observedScene = options.renderedSurfaceDiscovery
? projectRenderedSurfaces(
decoratedScene,
normalizedDocuments,
clusterRenderedSurfaces(
discoverRenderedSurfaces(
normalizedDocuments,
captured.viewport,
captured.excludedBackendNodeIds,
),
),
)
: decoratedScene;
const rendered = renderVom(observedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
@@ -1115,6 +1167,7 @@ async function handleVomObservation(
maxTokens: params.max_tokens,
activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
renderedSurfaceDiscovery: toolName === "observe",
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
signal,
});
@@ -1132,6 +1185,9 @@ async function handleVomObservation(
tabId: target.tabId,
...(ref.frameId ? { frameId: ref.frameId } : {}),
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
...(ref.kind ? { kind: ref.kind } : {}),
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
},
] as const;
}),
+17 -1
View File
@@ -2,8 +2,9 @@
// compound CDP node identity via the session RefStore, and build stable
// `ref_not_found` errors for hard-failure tool paths.
import type { Rect } from "@browser-skill/vom";
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef } from "@/session-manager/ref-store";
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
@@ -12,6 +13,9 @@ export interface SnapshotRefLookup {
refKey: string;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
}
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -37,6 +41,9 @@ export function lookupSnapshotRef(
refKey,
...(entry.frameId ? { frameId: entry.frameId } : {}),
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
kind: entry.kind,
capabilities: entry.capabilities,
};
}
@@ -49,6 +56,7 @@ export function resolveSnapshotRef(
ctx: SessionContext,
ref: string,
tabId: number,
requiredCapability?: RefCapability,
): SnapshotRefLookup | RpcError {
const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) {
@@ -58,5 +66,13 @@ export function resolveSnapshotRef(
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
);
}
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
return rpcError(
"permission_denied",
"ref_capability_denied",
`ref ${ref} does not support ${requiredCapability}`,
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
);
}
return looked;
}
@@ -722,6 +722,44 @@ describe("captureViewModel", () => {
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 200, 100],
[10, 10, 200, 100],
],
paintOrders: [0, 1, 2],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const canvas = nodes.find((node) => node.backendNodeId === 13);
expect(canvas?.rendered).toBe(true);
expect(canvas?.visuallySuppressed).toBe(true);
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
@@ -0,0 +1,256 @@
import type { VomScene } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "../rendered-surfaces";
import type { SemanticAxNode } from "../semantic-graph";
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "canvas",
attrs: {},
rect: { x: 10, y: 20, w: 800, h: 500 },
localRect: { x: 10, y: 20, w: 800, h: 500 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
rendered: true,
...overrides,
};
}
function document(
domNodes: CapturedNode[],
axNodes: SemanticAxNode[] = [],
): CapturedFrameDocument<SemanticAxNode> {
return {
frameId: domNodes[0]?.frameId ?? "main",
contextScopeId: domNodes[0]?.frameId ?? "main",
target: { tabId: 4 },
domNodes,
axNodes,
};
}
describe("rendered surface discovery", () => {
it("finds prominent rendered canvases and excludes hidden descendants", () => {
const visible = domNode(2, { parentBackendNodeId: 1 });
const hidden = domNode(3, { parentBackendNodeId: 4 });
const surfaces = discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
visible,
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
hidden,
]),
],
{ width: 1280, height: 720 },
);
expect(surfaces).toHaveLength(1);
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
});
it("keeps tiny canvases as individually addressable exact discoveries", () => {
const surfaces = discoverRenderedSurfaces(
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
{ width: 1280, height: 720 },
);
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("keeps a surface when the canvas only has partial native accessibility content", () => {
const axNodes = [
{
nodeId: "canvas",
backendDOMNodeId: 2,
role: { type: "role", value: "presentation" },
childIds: ["button"],
frameId: "main",
},
{
nodeId: "button",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
frameId: "main",
},
] as SemanticAxNode[];
expect(
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
const canvas = domNode(2, {
attrs: {
id: "must-not-be-a-label",
class: "must-not-be-a-label",
"data-testid": "must-not-be-a-label",
},
});
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
expect.not.objectContaining({
label: expect.anything(),
}),
]);
});
it("treats blank explicit Canvas names as unnamed", () => {
expect(
discoverRenderedSurfaces(
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
{ width: 1280, height: 720 },
),
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
});
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces(
[document([owner, canvas])],
{ width: 1280, height: 720 },
new Set([1]),
),
).toEqual([]);
});
it("excludes canvases suppressed by a fully transparent ancestor", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 0, y: 0, w: 900, h: 600 },
visuallySuppressed: true,
});
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
).toEqual([]);
});
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 10, y: 20, w: 100, h: 70 },
overflowX: "hidden",
overflowY: "hidden",
});
const partial = domNode(2, {
parentBackendNodeId: 1,
rect: { x: 55, y: 30, w: 140, h: 60 },
});
const fullyClipped = domNode(3, {
parentBackendNodeId: 1,
rect: { x: 140, y: 30, w: 80, h: 50 },
});
expect(
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
width: 1280,
height: 720,
}),
).toEqual([
expect.objectContaining({
backendNodeId: 2,
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
}),
]);
});
});
describe("rendered surface clustering and projection", () => {
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
const body = domNode(1, { tag: "body", rect: null, localRect: null });
const surfaces = discoverRenderedSurfaces(
[
document([
body,
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
]),
],
{ width: 1280, height: 720 },
);
const groups = clusterRenderedSurfaces(surfaces);
const scene: VomScene = {
viewport: { width: 1280, height: 720 },
nodes: [
{
id: 10,
parentId: null,
backendNodeId: 1,
frameId: "main",
tag: "body",
role: "RootWebArea",
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
};
expect(groups).toHaveLength(1);
expect(groups[0]?.members).toHaveLength(2);
expect(groups[0]?.label).toBeUndefined();
expect(
projectRenderedSurfaces(
scene,
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
groups,
),
).toMatchObject({
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
});
});
it("does not merge a contained canvas with a larger independent canvas", () => {
const groups = clusterRenderedSurfaces([
...discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
domNode(2, { parentBackendNodeId: 1 }),
domNode(3, {
parentBackendNodeId: 1,
rect: { x: 30, y: 40, w: 200, h: 100 },
}),
]),
],
{ width: 1280, height: 720 },
),
]);
expect(groups).toHaveLength(2);
});
it("does not merge coincident canvases owned by different containers", () => {
const groups = clusterRenderedSurfaces(
discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "section", rect: null, localRect: null }),
domNode(2, { tag: "section", rect: null, localRect: null }),
domNode(3, { parentBackendNodeId: 1 }),
domNode(4, { parentBackendNodeId: 2 }),
]),
],
{ width: 1280, height: 720 },
),
);
expect(groups).toHaveLength(2);
});
});
+30 -1
View File
@@ -11,7 +11,15 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"overflow-x",
"overflow-y",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -39,6 +47,10 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
overflowX?: string;
overflowY?: string;
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
visuallySuppressed?: boolean;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -796,6 +808,8 @@ function parseDocumentNodes(
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const overflowXCol = STYLE_COL["overflow-x"];
const overflowYCol = STYLE_COL["overflow-y"];
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -821,6 +835,7 @@ function parseDocumentNodes(
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
@@ -840,7 +855,11 @@ function parseDocumentNodes(
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let overflowX = "visible";
let overflowY = "visible";
let rendered = false;
let visuallySuppressed =
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
@@ -863,12 +882,19 @@ function parseDocumentNodes(
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
visuallySuppressed ||=
visibility === "hidden" ||
visibility === "collapse" ||
(Number.parseFloat(opacity) || 0) <= 0;
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
@@ -903,6 +929,9 @@ function parseDocumentNodes(
position,
pointerEvents,
cursor,
overflowX,
overflowY,
visuallySuppressed,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
@@ -0,0 +1,65 @@
import type { Rect } from "@browser-skill/vom";
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
const STACK_IOU_RATIO = 0.9;
const STACK_SIZE_RATIO = 0.9;
function area(rect: Rect): number {
return Math.max(0, rect.w) * Math.max(0, rect.h);
}
function intersectionArea(a: Rect, b: Rect): number {
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
return width * height;
}
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
if (a.frameId !== b.frameId) return false;
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
const aArea = area(a.visibleRect);
const bArea = area(b.visibleRect);
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
const union = aArea + bArea - intersection;
const widthRatio =
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
const heightRatio =
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
return (
union > 0 &&
intersection / union >= STACK_IOU_RATIO &&
widthRatio >= STACK_SIZE_RATIO &&
heightRatio >= STACK_SIZE_RATIO
);
}
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
const representative = [...members].sort(
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
)[0];
return {
frameId: representative.frameId,
members,
representative,
label: members.map((surface) => surface.label).find((label) => label !== undefined),
};
}
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
const groups: RenderedSurface[][] = [];
for (const surface of surfaces) {
const matching = groups.find((group) =>
group.some((member) => sameVisualStack(member, surface)),
);
if (matching) matching.push(surface);
else groups.push([surface]);
}
return groups
.map(toGroup)
.sort(
(a, b) =>
a.frameId.localeCompare(b.frameId) ||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
a.representative.visibleRect.x - b.representative.visibleRect.x,
);
}
@@ -0,0 +1,113 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurface } from "./types";
function clean(value: string | undefined): string | undefined {
const normalized = value?.replace(/\s+/g, " ").trim();
return normalized || undefined;
}
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
}
function intersectRect(a: Rect, b: Rect): Rect | null {
const x = Math.max(a.x, b.x);
const y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.w, b.x + b.w);
const bottom = Math.min(a.y + a.h, b.y + b.h);
if (right <= x || bottom <= y) return null;
return { x, y, w: right - x, h: bottom - y };
}
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
function clipToOverflowAncestors(
node: CapturedNode,
rect: Rect,
nodesByBackend: ReadonlyMap<number, CapturedNode>,
): Rect | null {
let clipped: Rect | null = rect;
let parentBackendNodeId = node.parentBackendNodeId;
let guard = 0;
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
const parent = nodesByBackend.get(parentBackendNodeId);
if (!parent) break;
if (parent.rect) {
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
if (clipX || clipY) {
const clipRect = {
x: clipX ? parent.rect.x : clipped.x,
y: clipY ? parent.rect.y : clipped.y,
w: clipX ? parent.rect.w : clipped.w,
h: clipY ? parent.rect.h : clipped.h,
};
clipped = intersectRect(clipped, clipRect);
}
}
parentBackendNodeId = parent.parentBackendNodeId;
guard += 1;
}
return clipped;
}
function excludedByDomPolicy(
node: CapturedNode,
nodesByBackend: Map<number, CapturedNode>,
excludedBackendNodeIds: ReadonlySet<number>,
): boolean {
let current: CapturedNode | undefined = node;
let guard = 0;
while (current && guard <= nodesByBackend.size) {
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
if (current.visuallySuppressed === true) return true;
const attrs = current.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
) {
return true;
}
current =
current.parentBackendNodeId === null
? undefined
: nodesByBackend.get(current.parentBackendNodeId);
guard += 1;
}
return false;
}
export function discoverRenderedSurfaces(
documents: CapturedFrameDocument<SemanticAxNode>[],
viewport: Viewport,
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
): RenderedSurface[] {
const surfaces: RenderedSurface[] = [];
for (const document of documents) {
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
for (const node of document.domNodes) {
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
const viewportRect = intersectViewport(node.rect, viewport);
const visibleRect = viewportRect
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
: null;
if (!visibleRect) continue;
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
surfaces.push({
renderingKind: "canvas",
frameId: document.frameId,
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
visibleRect,
paintOrder: node.paintOrder,
...(label ? { label } : {}),
});
}
}
return surfaces;
}
@@ -0,0 +1,4 @@
export { clusterRenderedSurfaces } from "./cluster";
export { discoverRenderedSurfaces } from "./discovery";
export { projectRenderedSurfaces } from "./project";
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
@@ -0,0 +1,100 @@
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurfaceGroup } from "./types";
function frameBackendKey(frameId: string, backendNodeId: number): string {
return `${frameId}\u0000${backendNodeId}`;
}
interface ProjectionIndex {
sceneByBackend: ReadonlyMap<string, number>;
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
domByFrame: ReadonlyMap<
string,
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
>;
rootSceneIdByFrame: ReadonlyMap<string, number>;
}
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
const document = index.documentByFrame.get(group.frameId);
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
let parentBackendNodeId = group.representative.parentBackendNodeId;
let guard = 0;
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
if (sceneId !== undefined) return sceneId;
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
guard += 1;
}
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
const owner = index.sceneByBackend.get(
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
);
if (owner !== undefined) return owner;
}
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
}
function buildProjectionIndex(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
): ProjectionIndex {
const rootSceneIdByFrame = new Map<string, number>();
for (const node of scene.nodes) {
if (
node.frameId &&
!rootSceneIdByFrame.has(node.frameId) &&
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
) {
rootSceneIdByFrame.set(node.frameId, node.id);
}
}
return {
sceneByBackend: new Map(
scene.nodes.flatMap((node) =>
node.backendNodeId === undefined
? []
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
),
),
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
domByFrame: new Map(
documents.map((document) => [
document.frameId,
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
]),
),
rootSceneIdByFrame,
};
}
export function projectRenderedSurfaces(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
groups: RenderedSurfaceGroup[],
): VomScene {
const index = buildProjectionIndex(scene, documents);
const visualSurfaces: VomVisualSurface[] = [];
for (const group of groups) {
const parentId = nearestSceneParent(index, group);
visualSurfaces.push({
parentId,
backendNodeId: group.representative.backendNodeId,
frameId: group.frameId,
renderingKind: "canvas",
visibleRect: group.representative.visibleRect,
...(group.label ? { label: group.label } : {}),
memberCount: group.members.length,
});
}
return {
...scene,
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
};
}
@@ -0,0 +1,18 @@
import type { Rect } from "@browser-skill/vom";
export interface RenderedSurface {
renderingKind: "canvas";
frameId: string;
backendNodeId: number;
parentBackendNodeId: number | null;
visibleRect: Rect;
paintOrder: number;
label?: string;
}
export interface RenderedSurfaceGroup {
frameId: string;
members: RenderedSurface[];
representative: RenderedSurface;
label?: string;
}
+1
View File
@@ -23,6 +23,7 @@ export type ErrorCode =
export type RpcErrorReason =
| "agent_window_scope"
| "element_not_visible"
| "ref_capability_denied"
| "ref_not_found"
| "selector_not_found"
| "target_not_fillable"
@@ -0,0 +1,105 @@
# Rendered Surface browser fixtures
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
## Serve
From the repository root:
```bash
python3 -m http.server 4173 --bind 127.0.0.1 \
--directory apps/extension/test-fixtures/rendered-surfaces
```
Use an Agent Window large enough to keep the primary matrix visible:
```bash
bsk session start --width 1440 --height 1100
bsk navigate http://127.0.0.1:4173/index.html --session <id>
bsk observe --session <id>
```
Always stop the session after the run.
## Pages
### `index.html`
Expected individual Surface labels:
- `fixture-basic-large`
- `fixture-tiny-8px`
- `fixture-pointer-events-none`
- `fixture-partial-ax`
- `fixture-contained-outer`
- `fixture-contained-inner`
- `fixture-different-parent-a`
- `fixture-different-parent-b`
Expected one grouped Surface:
- `fixture-layered-group`, with `layers=2`
Expected absent:
- `fixture-display-none`
- `fixture-visibility-hidden`
- `fixture-opacity-zero`
- `fixture-parent-opacity-zero`
- `fixture-hidden-attribute`
- `fixture-inert-parent`
- `fixture-aria-hidden-parent`
- `fixture-zero-size`
- `fixture-offscreen-right`
- `fixture-fully-overflow-clipped`
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
### `unlabeled.html`
This page contains eight Canvas elements but must render seven generic Surface markers because
the two coincident layers form one group.
- Every marker uses the stable fallback label `canvas visual surface`.
- Exactly one marker has `layers=2`.
- The two contained canvases remain separate markers.
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
- Blank `aria-label` and `title` values behave as no label.
- Fallback AX content does not suppress the corresponding Surface.
### `frames.html`
Expected Surface labels:
- `fixture-same-origin-frame`
- `fixture-cross-origin-frame`
- `fixture-nested-frame`
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
### `viewport.html`
Run with a `900x700` Agent Window.
- `fixture-partial-viewport` is partly visible and must be represented.
- `fixture-below-fold` is absent initially.
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
### `dynamic.html`
Use the normal DOM buttons and re-observe after each action:
- Initial state: `fixture-dynamic` absent.
- After **Insert canvas**: present.
- After **Resize to zero**: absent.
- After **Restore size**: present with a fresh ref.
- After **Remove canvas**: absent.
## Invariants
- `snapshot` must not emit any of these Surface markers.
- `observe` must emit only currently visible Canvas surfaces.
- Existing DOM/AX controls remain readable alongside Surface markers.
- Missing labels never prevent discovery; naming and discovery are independent.
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
- No fixture URL, id, label, or layout constant may appear in production discovery code.
@@ -0,0 +1,46 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface dynamic matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
button { padding: 8px 12px; }
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
</style>
</head>
<body>
<h1>Dynamic Canvas lifecycle</h1>
<div class="controls">
<button id="insert" type="button">Insert canvas</button>
<button id="zero" type="button">Resize to zero</button>
<button id="restore" type="button">Restore size</button>
<button id="remove" type="button">Remove canvas</button>
</div>
<div id="mount"></div>
<script>
const mount = document.querySelector("#mount");
const getCanvas = () => mount.querySelector("canvas");
document.querySelector("#insert").addEventListener("click", () => {
if (getCanvas()) return;
const canvas = document.createElement("canvas");
canvas.width = 260;
canvas.height = 120;
canvas.setAttribute("aria-label", "fixture-dynamic");
mount.append(canvas);
});
document.querySelector("#zero").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
});
document.querySelector("#restore").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.removeAttribute("style");
});
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
</script>
</body>
</html>
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 12px; background: #fdfefe; }
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
</style>
</head>
<body>
<canvas id="frame-canvas" width="220" height="90"></canvas>
<script>
const params = new URLSearchParams(location.search);
const canvas = document.querySelector("#frame-canvas");
if (params.get("canvas") === "0") canvas.remove();
else canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
if (params.get("nested") === "1") {
const iframe = document.createElement("iframe");
iframe.title = "nested fixture frame";
iframe.src = "frame.html?label=fixture-nested-frame";
document.body.append(iframe);
}
</script>
</body>
</html>
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface frame matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
</style>
</head>
<body>
<h1>Frame matrix</h1>
<div class="frames">
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame"></iframe>
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame"></iframe>
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
</div>
</body>
</html>
@@ -0,0 +1,118 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface discovery matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
.same-place { position: relative; width: 180px; height: 90px; }
.same-place > div { position: absolute; inset: 0; }
.same-place canvas { width: 180px; height: 90px; }
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
.tiny { width: 8px; height: 8px; border-width: 1px; }
</style>
</head>
<body>
<h1>Rendered Surface discovery matrix</h1>
<main class="matrix">
<section class="case">
<h2>Visible named canvas</h2>
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Tiny canvas remains addressable</h2>
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
</section>
<section class="case">
<h2>Visible but pointer-events none</h2>
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
</section>
<section class="case">
<h2>Partial AX/fallback content</h2>
<canvas aria-label="fixture-partial-ax" width="180" height="80">
<button type="button">Fallback button must not suppress the surface</button>
</canvas>
</section>
<section class="case">
<h2>Two real rendering layers</h2>
<div class="stack">
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Contained canvases are independent</h2>
<div class="contained">
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
</div>
</section>
<section class="case">
<h2>Same geometry, different parents</h2>
<div class="same-place">
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
</div>
</section>
<section class="case">
<h2>Partially overflow-clipped</h2>
<div class="clip-box">
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
</div>
</section>
<section class="case">
<h2>Fully overflow-clipped</h2>
<div class="clip-box">
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
</div>
</section>
<section class="case">
<h2>CSS-hidden variants</h2>
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
</section>
<section class="case">
<h2>Hidden ancestor variants</h2>
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
</section>
<section class="case">
<h2>Zero-size and offscreen</h2>
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
</section>
</main>
</body>
</html>
@@ -0,0 +1,68 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Unlabeled Rendered Surface matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
.case p { margin: 0 0 8px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
</style>
</head>
<body>
<h1>Unlabeled Rendered Surface matrix</h1>
<main class="matrix">
<section class="case">
<h2>Only engineering attributes</h2>
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Blank explicit names</h2>
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Nearby prose is not a Canvas name</h2>
<p>must-not-be-inferred-as-a-surface-label</p>
<canvas width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Fallback AX content does not name or suppress the Canvas</h2>
<canvas width="180" height="80">
<button type="button">Fallback control remains separate</button>
</canvas>
</section>
<section class="case">
<h2>Unnamed rendering layers still cluster</h2>
<div class="stack">
<canvas width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Unnamed contained canvases stay independent</h2>
<div class="contained">
<canvas width="190" height="100"></canvas>
<canvas width="70" height="40"></canvas>
</div>
</section>
</main>
</body>
</html>
@@ -0,0 +1,22 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface viewport matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { min-height: 1800px; padding: 16px; }
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
</style>
</head>
<body>
<h1>Viewport matrix</h1>
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
</body>
</html>
-1
View File
@@ -44,7 +44,6 @@
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
"!evals/browser/results/**",
"!**/target/**",
"!**/.wxt/**",
"!**/.output/**",
+285 -98
View File
@@ -1,142 +1,329 @@
---
name: browser-skill
description: |
Use when the user asks to automate their logged-in Chromium browser: visit
and read pages, fill forms, scrape data, click through flows, regression-test
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
the bsk CLI and browser extension.
Use when the user asks to perform browser automation tasks against their
logged-in browser: visit and read pages, fill forms, scrape data, click
through a flow, regression-test a PR's UI, validate a deployed page.
Requires the bsk CLI installed and the browser-skill extension loaded.
---
# browser-skill
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
are explicitly borrowed.
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
Do not use this skill for tasks with no browser, for extension installation, or when the user only
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
## When to use
## Required lifecycle
- Open pages, read titles/text, scrape structured data from sites the user can already access
- Fill forms, click through multi-step flows, smoke-test a UI change
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
Every browser task owns a bounded session:
## When NOT to use
```text
1. bsk session start # retain the printed 4-letter session id
2. bsk ... --session <id> # pass it to every session-scoped command
3. bsk session stop <id> # always run on success and error paths
- Tasks with **no browser** involved (files, APIs, databases only)
- Installing or configuring the extension (point the user to setup docs instead)
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
- Replacing the user's manual browsing when they only wanted an explanation
## Prerequisites
1. `bsk` on `PATH` (Rust CLI from browser-skill)
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
## Mandatory workflow
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
```
1. bsk session start → capture the 4-letter session id printed on stdout
2. … every tool command … → always pass --session <id>
3. bsk session stop <id> → REQUIRED when done (even on error paths)
```
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
When multiple browsers are connected, use `bsk browsers` and start with
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
Run `bsk doctor` when startup or transport problems persist after one retry.
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
## Work toward one observable goal
## Stop when the goal is met
- Derive a concrete success condition from the user's request or a supplied trace.
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
ambiguous result.
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
- If a human-only step appears or two attempts make no progress, request help instead of
brute-forcing.
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
With a trace, follow its semantic target information and values in order, but treat its refs as
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
task; it does not expand the user's goal or authorize additional actions.
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
## Observe, act, observe
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
Use this default loop:
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
```text
## Core interaction loop
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
```
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
Escalate page reading only as needed:
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
When `bsk observe` renders `@eN surface ... [visual-only; ...]`, the ref identifies a rendered canvas surface, not an interactable DOM control. Use `bsk screenshot --ref @eN --session <id>` to inspect its visible crop. Do not pass a visual surface ref to click, fill, hover, or select; those commands intentionally reject screenshot-only refs.
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
needed, obtain a fresh observation before acting on screenshot or HTML findings.
## Observation priority
## Respect the Agent Window boundary
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
borrowed across unrelated work.
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
## Ask the human when needed
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
controls can be highlighted. Use completion criteria only when the page has a clear stable success
signal.
## Sandbox rules
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
reasoning about the page or using refs.
| Rule | Detail |
|------|--------|
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
## Command inventory
## Global flags
This list of names is complete. Never invent a command outside it; read
`bsk <command...> --help` for flags instead of guessing them.
| Flag | Purpose |
|------|---------|
| `--json` | Machine-readable JSON on stdout (errors too) |
| `--quiet` | Suppress informational stderr |
| `-v` / `-vv` | More verbose logging |
```text
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
request-help record start|stop
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
## CLI command reference (one line each)
Details and flags: **`bsk <cmd> --help`**
### Diagnostics
| Command | Summary |
|---------|---------|
| `bsk status` | Connection health, connected browsers, active sessions |
| `bsk doctor` | Deep diagnostics and repair hints |
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
### Session
| Command | Summary |
|---------|---------|
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
| `bsk session stop --all` | Stop every active session |
| `bsk session list` | List active sessions |
### Window (require `--session <id>`)
| Command | Summary |
|---------|---------|
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
### Device emulation — `bsk emulate` (requires `--session <id>`)
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
```bash
bsk emulate --session <id> --device iphone-14
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
bsk emulate --session <id> --off
```
Required flags that are easy to get wrong:
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
```text
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
bsk screenshot --out <path> bsk emulate --device <preset-id>
### Tabs (require `--session <id>`)
| Command | Summary |
|---------|---------|
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
| `bsk tab close <tab-id>` | Close an agent tab |
| `bsk tab select <tab-id>` | Focus an agent tab |
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
### Observation (require `--session` unless noted)
| Command | Summary |
|---------|---------|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
### Console & network debugging (read-only; require `--session`)
| Command | Summary |
|---------|---------|
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
### Navigation
| Command | Summary |
|---------|---------|
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
| `bsk navigate-back` | History back one step |
| `bsk navigate-forward` | History forward one step |
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
### Interaction
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
### Scripting & timing
| Command | Summary |
|---------|---------|
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
### Ask the human for help — `bsk request-help`
When a step needs a human (captcha, login, OTP) or you want the user to
confirm an important action, pause and ask:
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
- `--prompt` (required): what the user should do.
- `--title` (optional): custom title for the overlay panel. When omitted,
the extension shows its default localized title.
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
whenever the prompt refers to a concrete element (a button to click, a
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
user is guided straight to the right spot instead of hunting for it. For
interaction scenarios, always include the relevant target(s); reserve a
prompt with no `--target` for cases where there is genuinely no specific
element to point at (e.g. "wait for the page to finish loading").
- `--timeout` (default `5m`): how long to wait.
- `--completion-criteria` (optional): JSON success detector. Use it only
when there is a concrete post-help success signal, e.g.
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
The target tab is brought to the foreground; the page stays interactive
while the agent control mask is hidden. The call blocks until the user
explicitly acts, the timeout expires, cancellation arrives, or explicit
completion criteria match. Page reloads, SPA route changes, and captcha
refreshes do not return control by themselves. The result `outcome` is one of:
- `continued` — the user finished and clicked Done / return control (treat as confirm).
- `cancelled` — the user clicked Cancel (treat as reject/abort).
- `timed_out` — nobody acted within the timeout.
- `completed` — the explicit `--completion-criteria` matched while the user had control.
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.
`request-help` does not refresh the page model after the user returns
control. After a `continued` or `completed` result, issue a separate
observation tool call (usually `bsk snapshot --session <id>`) before using
new refs or reasoning about the post-help page state.
#### Disabling request-help (unattended mode)
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
returns immediately with `outcome="disabled"` (no overlay, no waiting,
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
retry — complete the task autonomously or stop gracefully.
### Recording — `bsk record`
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
```bash
bsk record start --browser <instance-id-or-label> \
[--url https://…] [--purpose "publish a wiki doc"] \
[--max-page-tokens 3000] [--redact-values] \
[--output trace]
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
# Blocks until the user clicks Finish in the recording panel, then writes:
# trace/trace.json — action chain (+ state index when v3)
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
```
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
lowercase and hyphenated, such as `iphone-14`.
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
- `--purpose` is optional context metadata; it does **not** change what gets captured.
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
- `console` and `network` provide bounded, read-only debugging evidence.
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
them. Use `--off` to restore the real environment.
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
and never record banking, SSO, password-manager, or other sensitive pages.
## Error handling
## Recover without wandering
### Exit codes (`echo $?` after `bsk …`)
- Stale ref: observe again and retry the intended action once.
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
- Unsupported command: continue with available capabilities; suggest updating only when the missing
command is necessary.
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
| Code | Meaning | What to do |
|------|---------|------------|
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
details.
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
### When to run diagnostics
| Situation | Command |
|-----------|---------|
| Before first task in a session | `bsk status` — extension connected? |
| Any failure you cannot fix in one retry | `bsk doctor` |
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
## Red lines
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
---
**More detail for any command:** `bsk <cmd> --help`
+11 -18
View File
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
}
}
/// `bsk doctor` check: connected browsers should speak a protocol the
/// daemon accepts. A different protocol string is still a live
/// connection — report Ok so agents keep working. The detail is an
/// upgrade reminder, not a blocker.
/// `bsk doctor` check: every connected browser should use the same
/// protocol version as the daemon. Minor protocol drift is accepted by
/// the daemon but flagged here so the user can see who needs updating
/// (M10.4).
///
/// Review M2 (round-1 minor): when no browsers are connected, the
/// check has nothing to compare against, so it now reports
@@ -396,12 +396,13 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
})
.collect::<Vec<_>>()
.join(", ");
CheckResult::ok(
CheckResult::fail(
name,
format!(
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
"{} browser(s) have protocol minor drift from the daemon: {stale}",
status.version_skew_browsers.len()
),
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
)
}
@@ -580,7 +581,7 @@ mod m2_tests {
}
#[test]
fn browsers_check_reports_ok_when_skew_present() {
fn browsers_check_reports_fail_when_skew_present() {
let status = fake_status(
vec![BrowserStatusEntry {
instance_id: "alpha".into(),
@@ -604,17 +605,9 @@ mod m2_tests {
}],
);
let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok);
assert_eq!(check.status, CheckStatus::Fail);
assert!(check.detail.contains("alpha"));
assert!(
check
.detail
.contains("still usable — continue, and upgrade soon")
);
assert!(
check.hint.is_none(),
"a protocol-version note must not hint to stop"
);
assert!(check.hint.is_some());
}
#[test]
@@ -642,7 +635,7 @@ mod m2_tests {
}],
);
let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok);
assert_eq!(check.status, CheckStatus::Fail);
assert!(
check
.detail
+20
View File
@@ -40,6 +40,7 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
@@ -228,6 +229,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
summary: "snapshot ref does not support this operation",
hint: Some(
"visual surface refs are screenshot-only; run `bsk screenshot --ref <ref> --session <id>`",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
summary: "target element is not a <select>",
hint: Some(
@@ -377,6 +385,18 @@ mod tests {
assert!(!info.summary.contains("sandbox"));
}
#[test]
fn ref_capability_denied_explains_visual_surface_refs() {
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(info.summary, "snapshot ref does not support this operation");
assert!(
info.hint.unwrap().contains("screenshot-only"),
"expected visual-surface-specific hint"
);
assert_eq!(info.exit_code, 1);
}
#[test]
fn session_busy_overrides_timeout_copy() {
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
+2 -2
View File
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
let bold = "\x1b[1m";
let reset = "\x1b[0m";
eprintln!(
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
s.version_skew_browsers.len()
);
for skew in &s.version_skew_browsers {
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
skew.label.clone()
};
eprintln!(
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
skew.instance_id,
label,
display_protocol(&skew.client_protocol_version),
-259
View File
@@ -1,259 +0,0 @@
# Browser capability evaluation corpus
This directory is a deterministic, local browser evaluation environment for BrowserSkill, DSH,
and other command-line agents. Cases, fixture pages, direct `bsk` workflows, and assertions are
data-driven and discovered automatically. Adding a case does not require editing a central switch.
[中文说明](README.zh-CN.md)
## Design goals
- Agent-neutral: prompts describe observable page goals so the same cases can run through different
agents and adapters.
- Deterministic: pages run locally and use stable markers and run-scoped events.
- Extensible: manifests, fixture modules, and workflows are auto-discovered.
- Reproducible: generated cases record a seed and derive every DOM variation from it.
- Honest verification: page-observable results, response markers, and adapter evidence are reported
separately. Missing adapter evidence is `unverified`, never silently treated as passed.
- Privacy-safe: user badcases must be reduced to synthetic DOM and anonymized data before commit.
The corpus currently has six stable `core` cases and one seeded `matrix` case. The operation
inventory contains 28 browser operations; that count is capability coverage, not the number of
cases. Cases can grow independently and many cases may cover the same operation under different
page conditions.
## Quick start
These commands validate manifests and run unit/integration tests without launching a browser:
```sh
pnpm eval:browser validate
pnpm eval:browser list
pnpm eval:browser coverage
pnpm eval:browser:test
```
Build the repository CLI and exercise the real CLI, daemon, extension, Agent Window, and pages:
```sh
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
Every smoke case starts a fresh browser session and stops it in a `finally` cleanup path. Reports
and screenshots are written to the ignored `evals/browser/results/` directory.
`pnpm run eval:browser -- coverage` is also supported. The CLI intentionally accepts pnpm's leading
`--` separator.
## Suites and selection
| Suite | Intended use | Default execution |
| --- | --- | --- |
| `core` | Small stable acceptance set for everyday smoke and CI | Yes |
| `matrix` | Seeded DOM, timing, and layout variations | Explicit |
| `regression` | Minimized user badcases tied to an issue or fix | Explicit |
| `stress` | Higher volume or slower boundary testing | Explicit |
| `manual` | Human-dependent capabilities that cannot be deterministic | Explicit |
`smoke` and `run-agent` default to `core`, so adding a stress or regression case cannot silently
slow down existing CI. `list`, `coverage`, and `validate` inspect the full corpus by default.
```sh
pnpm eval:browser list --suite core
pnpm eval:browser list --tag form
pnpm eval:browser smoke --case form-controls
pnpm eval:browser smoke --case form-controls,generated-form
pnpm eval:browser smoke --case all
```
Filters from different dimensions are combined. Repeated/comma-separated suites or case ids are
alternatives; every requested tag must be present. `--task` remains a backward-compatible alias for
`--case`.
## Directory and discovery model
```text
evals/browser/
cases/
core/*.case.json
matrix/*.case.json
regression/<case-id>/
<case-id>.case.json
<case-id>.fixture.mjs
README.md
fixtures/pages/*.fixture.mjs
schemas/case.schema.json
lib/
tests/
```
- Any `cases/**/*.case.json` file is loaded as a case manifest.
- Any `fixtures/**/*.fixture.mjs` or `cases/**/*.fixture.mjs` file is registered as a fixture.
- Manifests are ordered by suite, optional numeric `order`, then id.
- `validate` rejects duplicate case ids, duplicate routes, missing fixture routes, unknown
operations/actions, invalid workflow variables, and adapter assertions with no evidence producer.
## Case manifest
A case owns all information needed by both an agent evaluation and direct CLI smoke:
```json
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "example-case",
"title": "Example browser behavior",
"order": 10,
"suite": "core",
"tags": ["form", "keyboard"],
"seed": 17,
"fixture": { "startPath": "/example" },
"prompts": { "en": "Open {url} ...", "zh-CN": "打开 {url}……" },
"coverage": ["page.navigate", "interact.press"],
"assertions": { "site": [], "response": [], "adapter": [] },
"smoke": { "steps": [{ "action": "navigate" }] }
}
```
Prompt placeholders are `{url}`, `{baseUrl}`, `{runId}`, and `{seed}`. Agent adapter arguments also
support `{prompt}`, `{caseId}` / `{taskId}`, and `{variant}`.
Assertions have three independent sources:
- `site`: events emitted by the local page, such as submitted values or a history navigation.
- `response`: stable text that must appear in the agent or CLI output.
- `adapter`: facts outside page JavaScript, such as a non-empty screenshot or confirmed session
cleanup.
The declarative smoke workflow supports navigation/history, waits, observe/snapshot/HTML,
screenshot/console/network, click/hover/fill/select/press, tabs, resize/emulation, and the manual
ownership/help operations. A step may `saveAs` its JSON result and later reference values such as
`{child.tab_id}`. A step may set `evidence` for an adapter assertion. See existing manifests for
complete examples.
## Adding a normal case
1. Add a `*.fixture.mjs` module exporting `{ id, routes, render(context) }`. It can live in shared
`fixtures/pages/` or beside a self-contained regression case.
2. Add a `*.case.json` manifest with prompts, operation coverage, assertions, and smoke steps.
3. Make all page events run-scoped using the shared `browserEval.send(...)` client helper.
4. Run `pnpm eval:browser validate` and `pnpm eval:browser:test`.
5. Run the new case against the real local build with `smoke --case <id>`.
Prefer one behavior or failure mechanism per case. Stable synthetic markers make failures easier to
diagnose than assertions against incidental prose.
## Turning a user badcase into a regression
Create a self-contained starting point:
```sh
pnpm eval:browser scaffold reported-timeout \
--title "Navigation settles after a late frame" \
--source "issue-123"
```
The command creates a manifest, fixture, and reproduction notes under
`cases/regression/reported-timeout/`. Then reduce the report to the smallest relevant mechanism:
- Keep DOM structure, timing, redirects, frames, or browser state that triggers the bug.
- Replace names, account data, URLs, tokens, screenshots, and page copy with synthetic values.
- Do not commit HAR files, cookies, credentials, production HTML, or proprietary assets.
- Record the issue/PR reference, seed when relevant, expected result, and the fixing PR.
- Confirm the new case fails on the known-bad build and passes on the fixed build when practical.
## Seeded matrix cases
`generated-form` varies label association, nesting depth, hydration delay, decoy controls, field
order, and element ids while keeping the task and oracle stable. Inspect a seed without a browser:
```sh
pnpm eval:browser generate --seed user-badcase-42
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
pnpm eval:browser smoke --case generated-form --seed 4,7,14
```
Repeated/comma-separated seeds run as independent results. Reports store every normalized seed, so
a failure can be replayed exactly and promoted into a named regression case if it represents a
distinct bug.
## Run any command-line agent
Copy `agents.example.json` to the ignored `agents.local.json` and adapt its command. Arguments are
passed directly to the executable without a shell.
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-local \
--suite core \
--repeat 3
```
An adapter may define `command`, `args`, `cwd`, `env`, `timeoutMs`, `variant`, free-form `metadata`,
and optional `metrics.errorPattern` / `metrics.toolCallPattern` regular expressions.
To compare 28 granular tools with domain tools, create two isolated adapter entries and run both
against the same selected cases and seeds:
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-granular,dsh-domain \
--suite core \
--repeat 10
```
Keep browser runs sequential to avoid session contention. Use fresh profiles and alternate adapter
order in larger experiments to reduce warm-cache bias.
## Reading results
```sh
pnpm eval:browser summarize evals/browser/results/*.json
```
The summary groups by adapter and variant and reports pass rate, fully verified rate, process
failures, matched error count, average tool calls, and average duration. The raw JSON retains case,
suite, tags, seed, prompt, command output, page event count, and individual oracle checks.
A healthy run has:
- no execution timeout or non-zero exit;
- no `executionError` in direct smoke;
- `verification.status` equal to `passed` (or intentionally `passed-with-unverified` for an agent
adapter that cannot expose external evidence);
- every session cleanup assertion verified;
- no unexpected increase in errors, tool calls, or duration versus the chosen baseline.
## Development baseline
Use these lanes as commit and PR gates:
| Change | Required checks |
| --- | --- |
| Any case, fixture, oracle, or harness change | `pnpm eval:browser:check` |
| CLI, daemon, extension, DSH plugin, or browser-operation change | The check above, `cargo build -p bsk`, then real-browser `smoke --suite core` |
| DOM discovery, form interaction, timing, or waiting change | The checks above, then `smoke --suite matrix --seed 4,7,14` |
| A bug fix with a known reproduction | Add a `regression` case and run that case on the fixed build |
Before committing this harness, the complete acceptance sequence is:
```sh
pnpm eval:browser:check
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
All commands must exit zero. The smoke summary must have 100% pass and fully verified rates, zero
execution failures/errors, and `bsk session list --json` must be `[]` afterward. Do not weaken an
oracle or replace a stable marker merely to make a regression pass; behavior changes should update
the case and expected baseline in the same reviewed change.
The direct smoke lane covers 25 of 28 operations. `tabs.borrow`, `tabs.return`, and
`assist.request-help` stay manual because they require a real user tab or human interaction and
would make the default suite destructive or non-deterministic.
-197
View File
@@ -1,197 +0,0 @@
# 浏览器能力测试语料库
这里是一套供 BrowserSkill、DSH 以及其他命令行 Agent 共用的本地确定性浏览器测试环境。
case、测试页面、`bsk` 直连流程和断言都会自动发现;新增 case 不需要修改中央 `switch`。
[English](README.md)
## 核心原则
- 与 Agent 无关:prompt 只描述可观察的页面目标,同一套 case 可以被不同 Agent 和
adapter 使用。
- 本地且确定:页面只依赖本地服务,使用稳定 marker 和按 run 隔离的事件。
- 可扩展:manifest、fixture 和 workflow 都从目录自动发现。
- 可复现:生成式页面的所有变化由 seed 决定,报告会保留 seed。
- 如实验证:页面事件、回答文本和 adapter 外部证据分别统计;拿不到的证据标为
`unverified`,不会冒充通过。
- 隐私安全:用户 badcase 必须先缩减为合成 DOM 和匿名数据,再进入仓库。
当前有 6 个稳定的 `core` case 和 1 个 seed 驱动的 `matrix` case。28 表示浏览器操作能力
清单,并不是 case 数量;同一种操作可以在不同页面结构、时序和浏览器状态下由多个 case
重复覆盖。
## 快速使用
不启动浏览器的校验:
```sh
pnpm eval:browser validate
pnpm eval:browser list
pnpm eval:browser coverage
pnpm eval:browser:test
```
使用仓库当前代码编译 `bsk`,再跑真实 CLI、daemon、扩展、小窗和页面链路:
```sh
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
每个 smoke case 都会创建独立 session,并在 `finally` 中关闭。JSON 报告和截图输出到已被
Git 忽略的 `evals/browser/results/`。
也可以写成 `pnpm run eval:browser -- coverage`;CLI 已兼容 pnpm 传入的前导 `--`。
## Suite 与筛选
| Suite | 用途 | 默认执行 |
| --- | --- | --- |
| `core` | 日常 smoke 和 CI 使用的小而稳定的验收集 | 是 |
| `matrix` | seed 驱动的 DOM、时序、布局组合 | 显式选择 |
| `regression` | 从用户 badcase 缩减并与 issue/修复关联的回归 | 显式选择 |
| `stress` | 更高次数或更慢的边界测试 | 显式选择 |
| `manual` | 必须依赖真人或真实用户标签页的能力 | 显式选择 |
`smoke` 和 `run-agent` 不带筛选时只跑 `core`,所以以后新增 stress/badcase 不会突然拖慢
已有 CI。`list`、`coverage`、`validate` 默认检查全部语料。
```sh
pnpm eval:browser list --suite core
pnpm eval:browser list --tag form
pnpm eval:browser smoke --case form-controls
pnpm eval:browser smoke --case form-controls,generated-form
pnpm eval:browser smoke --case all
```
不同维度的筛选会叠加;逗号分隔的 suite/case 是“或”,多个 tag 必须全部命中。旧参数
`--task` 仍作为 `--case` 的兼容别名。
## 自动发现结构
```text
evals/browser/
cases/
core/*.case.json
matrix/*.case.json
regression/<case-id>/
<case-id>.case.json
<case-id>.fixture.mjs
README.md
fixtures/pages/*.fixture.mjs
schemas/case.schema.json
lib/
tests/
```
- `cases/**/*.case.json` 都会作为 case manifest 加载。
- `fixtures/**/*.fixture.mjs` 和 `cases/**/*.fixture.mjs` 都会注册为页面 fixture。
- 顺序按 suite、可选的数字 `order`、id 排列。
- `validate` 会阻止重复 case id、重复路由、缺失 fixture、未知操作/action、无效 workflow
变量,以及没有任何步骤产出的 adapter evidence。
一个 manifest 同时描述:中英文 prompt、能力 coverage、页面事件/回答/adapter 三类断言,以及
`bsk` 直连 smoke 的声明式步骤。prompt 可用 `{url}`、`{baseUrl}`、`{runId}`、`{seed}`;
workflow 可以用 `saveAs` 保存 JSON 结果,并用 `{child.tab_id}` 之类的变量引用后续步骤。
完整写法可直接参考 `cases/core/` 和 `cases/matrix/`。
## 新增普通 case
1. 新增导出 `{ id, routes, render(context) }` 的 `*.fixture.mjs`,可放共享 fixture 目录,也可
和 regression case 放在一起。
2. 新增 `*.case.json`,写好 prompt、operation coverage、断言和 smoke steps。
3. 页面使用共享的 `browserEval.send(...)` 上报按 run 隔离的事件。
4. 执行 `pnpm eval:browser validate` 和 `pnpm eval:browser:test`。
5. 用 `smoke --case <id>` 验证真实本地编译链路。
建议一个 case 只表达一种行为或故障机制,并使用稳定的合成 marker,避免断言偶然文案。
## 把用户 badcase 固化成回归
先生成一个自包含模板:
```sh
pnpm eval:browser scaffold reported-timeout \
--title "Navigation settles after a late frame" \
--source "issue-123"
```
它会在 `cases/regression/reported-timeout/` 下生成 manifest、fixture 和复现说明。随后只保留
真正触发问题的 DOM 结构、时序、跳转、frame 或浏览器状态,并遵守:
- 所有姓名、账号、网址、token、截图和文案都替换为合成值。
- 不提交 HAR、cookie、凭据、生产 HTML 或私有素材。
- 记录 issue/PR 来源、相关 seed、期望行为和后续修复 PR。
- 条件允许时,证明新 case 在旧版本失败、修复版本通过。
## Seed 矩阵
`generated-form` 会确定性改变 label 关联方式、DOM 嵌套、hydration 延迟、干扰控件、字段
顺序和 element id,但任务和 oracle 保持不变:
```sh
pnpm eval:browser generate --seed user-badcase-42
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
pnpm eval:browser smoke --case generated-form --seed 4,7,14
```
逗号分隔或重复的 seed 会生成相互独立的结果。任何失败都能按报告中的规范化 seed 精确重放;
若它代表独立 bug,再提升为命名 regression case。
## 测任意 Agent 与比较 28/6 工具
把 `agents.example.json` 复制为已被 Git 忽略的 `agents.local.json`,分别配置两个隔离的 Agent
profile/build。参数以数组直接传给可执行程序,不经过 shell。
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-granular,dsh-domain \
--suite core \
--repeat 10
```
浏览器任务应串行运行,避免 session 竞争;更大实验中使用干净 profile 并轮换 adapter 顺序,
降低缓存偏差。
## 如何看结果
```sh
pnpm eval:browser summarize evals/browser/results/*.json
```
汇总按 adapter/variant 展示通过率、完全验证率、进程失败、错误数、平均工具调用数和耗时。
原始 JSON 会保留 case、suite、tag、seed、prompt、命令输出、页面事件数和每条 oracle 结果。
健康运行应满足:没有 timeout/非零退出,direct smoke 没有 `executionError`,验证状态为
`passed`(无法暴露外部证据的 Agent adapter 可明确为 `passed-with-unverified`),session cleanup
通过,并且错误数、调用数和耗时没有相对基线异常上涨。
## 开发基准线
后续开发统一按下面的门禁执行:
| 改动范围 | 必跑检查 |
| --- | --- |
| 任意 case、fixture、oracle 或 harness 改动 | `pnpm eval:browser:check` |
| CLI、daemon、扩展、DSH 插件或浏览器操作改动 | 上述检查、`cargo build -p bsk`、真实浏览器 `smoke --suite core` |
| DOM 发现、表单交互、时序或等待逻辑改动 | 上述检查,再跑 `smoke --suite matrix --seed 4,7,14` |
| 修复已有明确复现的 bug | 新增 `regression` case,并在修复版本单独运行该 case |
提交这套 harness 前,完整验收命令为:
```sh
pnpm eval:browser:check
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
所有命令必须以 0 退出;smoke 汇总必须达到 100% pass、100% fully verified、0 execution
failure/error,结束后 `bsk session list --json` 必须为 `[]`。不能为了让回归通过而削弱 oracle
或替换稳定 marker;如果产品行为确实改变,case 与预期基线必须在同一个受审查改动中更新。
直连 smoke 自动覆盖 28 项中的 25 项。`tabs.borrow`、`tabs.return`、`assist.request-help` 需要
真实用户标签页或真人交互,因此保留在 manual lane,避免默认测试具有破坏性或不确定性。
-17
View File
@@ -1,17 +0,0 @@
{
"agents": {
"dsh-local": {
"variant": "granular-28",
"command": "dsh",
"args": ["--profile", "headless", "{prompt}"],
"cwd": "../..",
"timeoutMs": 300000,
"metadata": {
"toolCount": 28
},
"metrics": {
"toolCallPattern": "browser[_-][a-z_-]+"
}
}
}
}
-9
View File
@@ -1,9 +0,0 @@
# Browser evaluation cases
Cases are discovered recursively from `*.case.json`; colocated `*.fixture.mjs` files are discovered
as fixtures. Use `core` for small deterministic acceptance checks, `matrix` for seeded variations,
and `regression/<case-id>/` for privacy-safe minimized user badcases.
Run `pnpm eval:browser validate` after every case change. See the parent
[`README.md`](../README.md) for the manifest contract, workflow actions, scaffolding, and acceptance
criteria.
@@ -1,48 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "diagnostics",
"title": "Console and network inspection",
"order": 50,
"suite": "core",
"tags": ["console", "network", "diagnostics"],
"fixture": { "startPath": "/diagnostics" },
"prompts": {
"en": "Open {url}. Use browser diagnostics to inspect the page console and recent network activity. Report the console marker and the token returned by the ping request, then close the browser session.",
"zh-CN": "打开 {url}。使用浏览器诊断能力检查页面 console 和最近的 network 活动,报告 console marker 以及 ping 请求返回的 token,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"inspect.console",
"inspect.network"
],
"assertions": {
"site": [
{
"label": "diagnostic ping completed",
"type": "network.ping",
"minCount": 1
}
],
"response": [
{ "label": "reported console marker", "includes": "CONSOLE-61" },
{ "label": "reported network token", "includes": "NETWORK-73" }
],
"adapter": [
{ "label": "console inspection was executed", "key": "consoleRead" },
{ "label": "network inspection was executed", "key": "networkRead" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait", "duration": "300ms" },
{ "action": "console", "evidence": "consoleRead" },
{ "action": "network", "evidence": "networkRead" },
{ "action": "observe" }
]
}
}
@@ -1,82 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "form-controls",
"title": "Form, select, keyboard, and screenshot",
"order": 20,
"suite": "core",
"tags": ["form", "keyboard", "screenshot", "select"],
"fixture": { "startPath": "/form" },
"prompts": {
"en": "Open {url}. Fill Text input with \"agent-parity\" and Textarea with \"BrowserSkill works\". Select the visible option \"Two\" from Dropdown. Capture a screenshot of the form, focus Submit, and press Enter to submit. Confirm that the result says \"Received!\", then close the browser session.",
"zh-CN": "打开 {url}。把 Text input 填成“agent-parity”,Textarea 填成“BrowserSkill works”,Dropdown 选择可见选项“Two”。截取表单截图,然后聚焦 Submit 并按 Enter 提交。确认结果页出现“Received!”,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.fill",
"interact.select",
"interact.press",
"inspect.screenshot"
],
"assertions": {
"site": [
{
"label": "Enter was pressed on Submit",
"type": "form.enter_pressed",
"minCount": 1
},
{
"label": "form values were submitted",
"type": "form.submitted",
"where": {
"data.text": "agent-parity",
"data.notes": "BrowserSkill works",
"data.choice": "two"
},
"minCount": 1
},
{
"label": "result page was displayed",
"type": "page.shown",
"where": { "data.path": "/result" },
"minCount": 1
}
],
"response": [{ "label": "confirmed result", "includes": "Received!" }],
"adapter": [
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{
"action": "fill",
"selector": "#text-input",
"value": "agent-parity"
},
{
"action": "fill",
"selector": "#notes",
"value": "BrowserSkill works"
},
{ "action": "select", "selector": "#choice", "values": ["two"] },
{
"action": "screenshot",
"artifactName": "form.png",
"evidence": "screenshotCreated"
},
{ "action": "press", "key": "Enter", "selector": "#submit" },
{
"action": "wait-site-event",
"type": "page.shown",
"where": { "path": "/result" }
},
{ "action": "wait", "duration": "200ms" },
{ "action": "observe" }
]
}
}
@@ -1,59 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "hover-inspect",
"title": "Hover, local HTML, and screenshot",
"order": 30,
"suite": "core",
"tags": ["hover", "html", "screenshot"],
"fixture": { "startPath": "/hover-inspect" },
"prompts": {
"en": "Open {url}. Hover over Account, then choose Profile beta from the revealed menu. Inspect the local HTML around the Inspection card and capture a screenshot. Report both marker codes you find, then close the browser session.",
"zh-CN": "打开 {url}。把鼠标悬停在“Account”上,然后点击展开菜单中的“Profile beta”。检查 Inspection card 附近的局部 HTML,并截取页面截图。报告找到的两个标记代码,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.hover",
"interact.click",
"inspect.html",
"inspect.screenshot"
],
"assertions": {
"site": [
{
"label": "Account menu was hovered",
"type": "hover.menu_opened",
"minCount": 1
},
{
"label": "Profile beta was selected",
"type": "hover.profile_clicked",
"minCount": 1
}
],
"response": [
{ "label": "reported hover marker", "includes": "HOVER-17" },
{ "label": "reported HTML marker", "includes": "INSPECT-27" }
],
"adapter": [
{ "label": "HTML inspection was executed", "key": "htmlInspected" },
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "hover", "selector": "#account-menu" },
{ "action": "click", "selector": "#profile-link" },
{ "action": "html", "evidence": "htmlInspected" },
{
"action": "screenshot",
"artifactName": "hover-inspect.png",
"evidence": "screenshotCreated"
}
]
}
}
@@ -1,43 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "mobile-emulation",
"title": "Mobile emulation",
"order": 60,
"suite": "core",
"tags": ["emulation", "mobile", "viewport"],
"fixture": { "startPath": "/responsive" },
"prompts": {
"en": "Open {url}, emulate an iPhone 14, and report the viewport marker shown on the page. Close the browser session when finished.",
"zh-CN": "打开 {url},模拟 iPhone 14,并报告页面显示的 viewport marker。完成后关闭浏览器会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "assist.emulate"],
"assertions": {
"site": [
{
"label": "iPhone-sized viewport was observed",
"type": "viewport.sample",
"where": { "data.width": 390, "data.height": 844 },
"minCount": 1
}
],
"response": [{ "label": "reported viewport marker", "includes": "VIEWPORT-84" }],
"adapter": [
{ "label": "device emulation was executed", "key": "emulatedDevice" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "resize", "width": 900, "height": 700 },
{
"action": "emulate",
"device": "iphone-14",
"evidence": "emulatedDevice"
},
{ "action": "wait", "duration": "300ms" },
{ "action": "html" }
]
}
}
@@ -1,67 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "navigation-history",
"title": "Navigation, history, and reload",
"order": 10,
"suite": "core",
"tags": ["navigation", "history", "reload"],
"fixture": { "startPath": "/navigation/start" },
"prompts": {
"en": "Open {url}. Read the main heading, follow the Details link, and report the code shown there. Then go back, go forward, reload the page, and close the browser session. Use only the browser capability for page operations.",
"zh-CN": "打开 {url}。读取页面主标题,点击“详情”,并报告详情页上的代码。然后依次后退、前进、刷新页面,最后关闭浏览器会话。页面操作只使用浏览器能力。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"page.back",
"page.forward",
"page.reload",
"interact.click"
],
"assertions": {
"site": [
{
"label": "details link was clicked",
"type": "navigation.detail.clicked",
"minCount": 1
},
{
"label": "back returned to the start page",
"type": "page.shown",
"where": { "data.path": "/navigation/start" },
"minCount": 2
},
{
"label": "forward returned to the detail page",
"type": "page.shown",
"where": { "data.path": "/navigation/detail" },
"minCount": 2
},
{
"label": "detail page was reloaded",
"type": "page.shown",
"where": {
"data.path": "/navigation/detail",
"data.navigationType": "reload"
},
"minCount": 1
}
],
"response": [{ "label": "reported detail code", "includes": "NAV-42" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "observe" },
{ "action": "snapshot" },
{ "action": "click", "selector": "#details-link" },
{ "action": "html" },
{ "action": "back" },
{ "action": "forward" },
{ "action": "reload" }
]
}
}
-71
View File
@@ -1,71 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "tabs",
"title": "Create, select, and close a tab",
"order": 40,
"suite": "core",
"tags": ["tabs", "ownership", "lifecycle"],
"fixture": { "startPath": "/tabs" },
"prompts": {
"en": "Open {url}. Create a new browser tab at {baseUrl}/tabs/child?run={runId}, read its token, click \"Mark child visited\", close the child tab, return to the original tab, and close the browser session.",
"zh-CN": "打开 {url}。新建一个浏览器标签页并访问 {baseUrl}/tabs/child?run={runId},读取其中的 token,点击“Mark child visited”,关闭子标签页,回到原标签页,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.click",
"tabs.create",
"tabs.select",
"tabs.close"
],
"assertions": {
"site": [
{
"label": "child tab page was displayed",
"type": "page.shown",
"where": { "data.path": "/tabs/child" },
"minCount": 1
},
{
"label": "child tab action ran",
"type": "tabs.child_marked",
"minCount": 1
}
],
"response": [{ "label": "reported child token", "includes": "TAB-29" }],
"adapter": [
{ "label": "child tab was closed", "key": "childTabClosed" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate", "saveAs": "original" },
{ "action": "tab-list" },
{
"action": "tab-create",
"url": "{baseUrl}/tabs/child?run={runId}",
"saveAs": "child"
},
{
"action": "navigate",
"url": "{baseUrl}/tabs/child?run={runId}",
"tabId": "{child.tab_id}"
},
{
"action": "click",
"selector": "#mark-child",
"tabId": "{child.tab_id}"
},
{ "action": "html", "tabId": "{child.tab_id}" },
{
"action": "tab-close",
"tabId": "{child.tab_id}",
"evidence": "childTabClosed"
},
{ "action": "tab-select", "tabId": "{original.tab_id}" }
]
}
}
-5
View File
@@ -1,5 +0,0 @@
# Seeded matrix cases
Matrix cases vary page structure or timing deterministically from a recorded seed. The task,
assertions, and success marker remain stable so a failed seed can be replayed exactly. A seed that
exposes a distinct product bug should be minimized and promoted to a named regression case.
@@ -1,77 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "generated-form",
"title": "Seeded form structure and hydration variants",
"order": 10,
"suite": "matrix",
"tags": ["dynamic-dom", "form", "hydration", "seeded"],
"seed": 20260827,
"fixture": { "startPath": "/matrix/generated-form" },
"prompts": {
"en": "Open {url}. Wait for the seeded form to finish hydrating. Fill Text input with \"seeded-agent\", select the visible option \"Two\", submit with Enter, and report the result marker and seed. Close the browser session.",
"zh-CN": "打开 {url}。等待 seeded form 完成 hydration,把 Text input 填成“seeded-agent”,选择可见选项“Two”,用 Enter 提交,并报告结果 marker 和 seed。最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"page.wait",
"inspect.observe",
"interact.fill",
"interact.select",
"interact.press"
],
"assertions": {
"site": [
{ "label": "generated form hydrated", "type": "matrix.ready", "minCount": 1 },
{
"label": "Enter was pressed on generated form",
"type": "matrix.enter_pressed",
"minCount": 1
},
{
"label": "generated values were submitted",
"type": "matrix.submitted",
"where": { "data.text": "seeded-agent", "data.choice": "two" },
"minCount": 1
},
{
"label": "matrix result page was displayed",
"type": "page.shown",
"where": { "data.path": "/matrix/generated-result" },
"minCount": 1
}
],
"response": [
{ "label": "reported matrix result", "includes": "MATRIX-OK" },
{ "label": "reported selected seed", "includes": "SEED-" }
],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{
"action": "wait-site-event",
"type": "matrix.ready",
"timeoutMs": 5000
},
{ "action": "observe" },
{ "action": "fill", "selector": "[name=text]", "value": "seeded-agent" },
{ "action": "select", "selector": "[name=choice]", "values": ["two"] },
{
"action": "press",
"key": "Enter",
"selector": "button[type=submit]"
},
{
"action": "wait-site-event",
"type": "page.shown",
"where": { "path": "/matrix/generated-result" }
},
{ "action": "wait", "duration": "200ms" },
{ "action": "observe" }
]
}
}
-12
View File
@@ -1,12 +0,0 @@
# Regression cases
Each user-reported badcase should live in its own directory with a manifest, synthetic fixture, and
README describing the symptom, minimal reproduction, expected behavior, source reference, and fix.
Never commit credentials, cookies, HAR files, production HTML, private screenshots, or proprietary
assets.
Create a starting point with:
```sh
pnpm eval:browser scaffold <case-id> --title "..." --source "issue-or-pr"
```
-467
View File
@@ -1,467 +0,0 @@
#!/usr/bin/env node
import { randomUUID } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { loadAgentConfig, runCommandTask } from "./lib/agent-runner.mjs";
import { runBskSmokeTask } from "./lib/bsk-runner.mjs";
import { buildOperationCoverage } from "./lib/coverage.mjs";
import { loadFixtureRegistry } from "./lib/fixture-registry.mjs";
import { verifyTask } from "./lib/oracle.mjs";
import { resultsDirectory } from "./lib/paths.mjs";
import { repositorySummary, validateRepositoryCases } from "./lib/repository-validation.mjs";
import { scaffoldCase } from "./lib/scaffold-case.mjs";
import { describeSeed, normalizeSeed } from "./lib/seeded-generator.mjs";
import { createEvalServer } from "./lib/server.mjs";
import { printSummary, summarizeReports } from "./lib/summary.mjs";
import { allTasks, getTask, listTasks, renderPrompt } from "./lib/tasks.mjs";
const booleanOptions = new Set(["help", "json"]);
function parseArguments(argv) {
const normalized = argv[0] === "--" ? argv.slice(1) : argv;
let [command, ...rest] = normalized;
const options = new Map();
const positionals = [];
if (command === "--help" || command === "-h") {
command = undefined;
options.set("help", [true]);
}
for (let index = 0; index < rest.length; index += 1) {
const argument = rest[index];
if (!argument.startsWith("--")) {
positionals.push(argument);
continue;
}
const equals = argument.indexOf("=");
const name = argument.slice(2, equals === -1 ? undefined : equals);
let value = equals === -1 ? undefined : argument.slice(equals + 1);
if (value === undefined && !booleanOptions.has(name)) {
const candidate = rest[index + 1];
if (candidate === undefined || candidate.startsWith("--")) {
throw new Error(`--${name} requires a value`);
}
value = candidate;
index += 1;
}
const values = options.get(name) ?? [];
values.push(value ?? true);
options.set(name, values);
}
return { command, options, positionals };
}
function option(parsed, name, fallback) {
return parsed.options.get(name)?.at(-1) ?? fallback;
}
function optionList(parsed, name) {
return (parsed.options.get(name) ?? [])
.flatMap((value) => String(value).split(","))
.map((value) => value.trim())
.filter(Boolean);
}
function numericOption(parsed, name, fallback) {
const value = Number(option(parsed, name, fallback));
if (!Number.isFinite(value) || value < 0) throw new Error(`--${name} must be non-negative`);
return value;
}
function seedOptions(parsed, fallback) {
const requested = optionList(parsed, "seed");
return requested.length > 0 ? unique(requested.map(normalizeSeed)) : [fallback];
}
function seedOption(parsed, fallback) {
const seeds = seedOptions(parsed, fallback);
if (seeds.length > 1) throw new Error("this command accepts only one --seed value");
return seeds[0];
}
function unique(values) {
return [...new Set(values)];
}
function selectCases(cases, parsed, { defaultSuite } = {}) {
const requestedIds = unique([...optionList(parsed, "case"), ...optionList(parsed, "task")]);
const requestedSuites = unique(optionList(parsed, "suite"));
const requestedTags = unique(optionList(parsed, "tag"));
const hasSelector =
requestedIds.length > 0 || requestedSuites.length > 0 || requestedTags.length > 0;
if (requestedIds.includes("all") && requestedIds.length > 1) {
throw new Error("--case all cannot be combined with named cases");
}
if (requestedSuites.includes("all") && requestedSuites.length > 1) {
throw new Error("--suite all cannot be combined with named suites");
}
const caseIds = requestedIds.includes("all") ? [] : requestedIds;
const suites = requestedSuites.includes("all")
? []
: requestedSuites.length > 0
? requestedSuites
: !hasSelector && defaultSuite
? [defaultSuite]
: [];
const knownIds = new Set(cases.map(({ id }) => id));
const missingIds = caseIds.filter((id) => !knownIds.has(id));
if (missingIds.length > 0) throw new Error(`unknown case(s): ${missingIds.join(", ")}`);
const knownSuites = new Set(cases.map(({ suite }) => suite));
const missingSuites = suites.filter((suite) => !knownSuites.has(suite));
if (missingSuites.length > 0) throw new Error(`unknown suite(s): ${missingSuites.join(", ")}`);
const knownTags = new Set(cases.flatMap(({ tags }) => tags));
const missingTags = requestedTags.filter((tag) => !knownTags.has(tag));
if (missingTags.length > 0) throw new Error(`unknown tag(s): ${missingTags.join(", ")}`);
const selected = cases.filter(
(testCase) =>
(caseIds.length === 0 || caseIds.includes(testCase.id)) &&
(suites.length === 0 || suites.includes(testCase.suite)) &&
requestedTags.every((tag) => testCase.tags.includes(tag)),
);
if (selected.length === 0) throw new Error("case selectors matched no cases");
return selected;
}
function reportFilename(mode) {
return `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${mode}.json`;
}
async function saveReport(report, outputDirectory) {
await mkdir(outputDirectory, { recursive: true });
const path = join(outputDirectory, reportFilename(report.mode));
await writeFile(path, `${JSON.stringify(report, null, 2)}\n`);
return path;
}
async function loadValidatedCorpus() {
const [cases, fixtureRegistry] = await Promise.all([allTasks(), loadFixtureRegistry()]);
const errors = validateRepositoryCases(cases, fixtureRegistry);
if (errors.length > 0) throw new Error(`browser eval validation failed:\n${errors.join("\n")}`);
return { cases, fixtureRegistry };
}
function selectionMetadata(cases) {
return {
cases: cases.map(({ id }) => id),
suites: unique(cases.map(({ suite }) => suite)),
tags: unique(cases.flatMap(({ tags }) => tags)).sort(),
};
}
function usage() {
console.log(`BrowserSkill local browser evaluation
Usage:
node evals/browser/cli.mjs validate [--json]
node evals/browser/cli.mjs list [--suite NAME] [--tag NAME] [--case ID] [--json]
node evals/browser/cli.mjs coverage [--suite NAME] [--tag NAME] [--case ID] [--json]
node evals/browser/cli.mjs serve [--host 127.0.0.1] [--port 4173]
node evals/browser/cli.mjs prompt <case> [--base-url URL] [--run-id ID] [--seed VALUE]
node evals/browser/cli.mjs verify <case> --base-url URL --run-id ID [--response TEXT]
node evals/browser/cli.mjs smoke [--bsk bsk] [--suite core] [--case ID] [--seed VALUE]
node evals/browser/cli.mjs run-agent --config FILE --agent NAME [--suite core] [--case ID]
node evals/browser/cli.mjs generate --seed VALUE [--json]
node evals/browser/cli.mjs scaffold <case-id> [--suite regression] [--title TEXT]
node evals/browser/cli.mjs summarize <report.json> [more reports...]
Selection:
--case ID[,ID] Select case ids; --task is a backward-compatible alias
--suite NAME[,NAME] Select suites
--tag NAME[,NAME] Require every selected tag
--case all Explicitly include every suite (run commands default to core)
--seed VALUE[,VALUE] Run one or more reproducible matrix seeds
run-agent options:
--agent NAME[,NAME] One or more entries from the config file
--variant NAME Override the report label (does not switch plugin code)
--locale en|zh-CN Prompt language (default zh-CN)
--repeat N Repeat every selected case (default 1)
--timeout MS Per-agent timeout (default 300000)
--out DIRECTORY Report/artifact directory
`);
}
async function commandValidate(parsed) {
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const summary = { ok: true, ...repositorySummary(cases, fixtureRegistry) };
if (option(parsed, "json", false)) console.log(JSON.stringify(summary, null, 2));
else {
console.log(`Validated ${summary.cases} cases and ${summary.fixtureRoutes} fixture routes.`);
console.log(
`Suites: ${Object.entries(summary.suites)
.map(([suite, count]) => `${suite}=${count}`)
.join(", ")}`,
);
}
}
async function commandList(parsed) {
const cases = selectCases(await allTasks(), parsed);
const tasks = listTasks(cases);
if (option(parsed, "json", false)) console.log(JSON.stringify(tasks, null, 2));
else {
for (const task of tasks) {
console.log(`${task.id}\t${task.suite}\t${task.tags.join(",")}\t${task.title}`);
}
}
}
async function commandCoverage(parsed) {
const cases = selectCases(await allTasks(), parsed);
const coverage = buildOperationCoverage(cases);
if (option(parsed, "json", false)) {
console.log(JSON.stringify(coverage, null, 2));
} else {
console.table(
coverage.map(({ operation, agentCases, smokeCases, directSmoke, note }) => ({
operation,
agent_cases: agentCases.join(", ") || "-",
smoke_cases: smokeCases.join(", ") || "-",
direct_smoke: directSmoke ? "yes" : "manual",
note: note ?? "",
})),
);
}
}
async function commandServe(parsed) {
const { fixtureRegistry } = await loadValidatedCorpus();
const server = createEvalServer({
host: option(parsed, "host", "127.0.0.1"),
port: numericOption(parsed, "port", 4173),
fixtureRegistry,
});
const info = await server.start();
console.log(`Browser eval server listening at ${info.baseUrl}`);
console.log("Press Ctrl-C to stop.");
await new Promise((resolveSignal) => {
process.once("SIGINT", resolveSignal);
process.once("SIGTERM", resolveSignal);
});
await server.stop();
}
async function commandPrompt(parsed) {
const cases = await allTasks();
const task = getTask(parsed.positionals[0], cases);
const prompt = renderPrompt(task, {
baseUrl: option(parsed, "base-url", "http://127.0.0.1:4173"),
runId: option(parsed, "run-id", randomUUID()),
locale: option(parsed, "locale", "zh-CN"),
seed: seedOption(parsed, task.seed),
});
console.log(prompt);
}
async function commandVerify(parsed) {
const task = getTask(parsed.positionals[0], await allTasks());
const baseUrl = option(parsed, "base-url");
const runId = option(parsed, "run-id");
if (!baseUrl || !runId) throw new Error("verify requires --base-url and --run-id");
let responseText = option(parsed, "response", "");
const responseFile = option(parsed, "response-file");
if (responseFile) responseText = await readFile(resolve(responseFile), "utf8");
const stateResponse = await fetch(`${baseUrl}/api/runs/${encodeURIComponent(runId)}`);
if (!stateResponse.ok) throw new Error(`oracle request failed with ${stateResponse.status}`);
const state = await stateResponse.json();
const verification = verifyTask(task, { events: state.events, responseText });
console.log(JSON.stringify(verification, null, 2));
if (verification.status === "failed") process.exitCode = 1;
}
async function commandRunAgent(parsed) {
const configPath = option(parsed, "config");
const agentNames = optionList(parsed, "agent");
if (!configPath || agentNames.length === 0) {
throw new Error("run-agent requires --config and at least one --agent");
}
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
const repeat = numericOption(parsed, "repeat", 1);
if (!Number.isInteger(repeat) || repeat < 1) throw new Error("--repeat must be at least 1");
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
const server = createEvalServer({ fixtureRegistry });
const serverInfo = await server.start();
const results = [];
const startedAt = new Date().toISOString();
try {
for (const agentName of agentNames) {
const { adapter, configDirectory } = await loadAgentConfig(configPath, agentName);
for (let iteration = 1; iteration <= repeat; iteration += 1) {
for (const task of selected) {
for (const seed of seedOptions(parsed, task.seed)) {
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
console.log(`[run] ${agentName} ${task.id}${seedLabel} (${iteration}/${repeat})`);
const result = await runCommandTask({
server,
serverInfo,
task,
adapter,
adapterName: agentName,
configDirectory,
iteration,
locale: option(parsed, "locale", "zh-CN"),
seed,
timeoutMs: numericOption(parsed, "timeout", 300_000),
variantOverride: option(parsed, "variant"),
});
results.push(result);
console.log(` ${result.verification.status} in ${result.execution.durationMs}ms`);
}
}
}
}
} finally {
await server.stop();
}
const report = {
schemaVersion: 1,
mode: "agent",
startedAt,
finishedAt: new Date().toISOString(),
selection: selectionMetadata(selected),
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
results,
};
const path = await saveReport(report, outputDirectory);
printSummary(summarizeReports([report]));
console.log(`Report: ${path}`);
if (
results.some(
(result) =>
result.verification.status === "failed" ||
result.execution.timedOut ||
result.execution.exitCode !== 0,
)
) {
process.exitCode = 1;
}
}
async function commandSmoke(parsed) {
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
const server = createEvalServer({ fixtureRegistry });
const serverInfo = await server.start();
const results = [];
const startedAt = new Date().toISOString();
try {
for (const task of selected) {
for (const seed of seedOptions(parsed, task.seed)) {
const runId = `bsk-${task.id}-${randomUUID().slice(0, 8)}`;
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
console.log(`[smoke] ${task.id}${seedLabel}`);
const result = await runBskSmokeTask({
bskCommand: option(parsed, "bsk", "bsk"),
outputDirectory,
server,
serverInfo,
task,
runId,
seed,
timeoutMs: numericOption(parsed, "timeout", 60_000),
});
results.push(result);
console.log(
` ${result.verification.status}${result.executionError ? `: ${result.executionError}` : ""}`,
);
}
}
} finally {
await server.stop();
}
const report = {
schemaVersion: 1,
mode: "bsk-smoke",
startedAt,
finishedAt: new Date().toISOString(),
selection: selectionMetadata(selected),
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
results,
};
const path = await saveReport(report, outputDirectory);
printSummary(summarizeReports([report]));
console.log(`Report: ${path}`);
if (results.some((result) => result.executionError || result.verification.status === "failed")) {
process.exitCode = 1;
}
}
async function commandGenerate(parsed) {
if (!parsed.options.has("seed")) throw new Error("generate requires --seed");
const dimensions = unique(optionList(parsed, "seed").map(normalizeSeed)).map(describeSeed);
const output = dimensions.length === 1 ? dimensions[0] : dimensions;
if (option(parsed, "json", false)) console.log(JSON.stringify(output, null, 2));
else console.table(dimensions);
}
async function commandScaffold(parsed) {
const id = parsed.positionals[0];
if (!id) throw new Error("scaffold requires a case id");
const result = await scaffoldCase({
id,
title: option(parsed, "title", id),
suite: option(parsed, "suite", "regression"),
source: option(parsed, "source"),
});
if (option(parsed, "json", false)) console.log(JSON.stringify(result, null, 2));
else {
console.log(`Created ${result.directory}`);
console.log(
"Edit the fixture, prompt, workflow, and assertions, then run eval:browser validate.",
);
}
}
async function commandSummarize(parsed) {
const paths = [...parsed.positionals, ...optionList(parsed, "input")];
if (paths.length === 0) throw new Error("summarize requires at least one report path");
const reports = await Promise.all(
paths.map(async (path) => JSON.parse(await readFile(resolve(path), "utf8"))),
);
const rows = summarizeReports(reports);
if (option(parsed, "json", false)) console.log(JSON.stringify(rows, null, 2));
else printSummary(rows);
}
async function main() {
const parsed = parseArguments(process.argv.slice(2));
if (!parsed.command || option(parsed, "help", false)) return usage();
switch (parsed.command) {
case "validate":
return commandValidate(parsed);
case "list":
return commandList(parsed);
case "coverage":
return commandCoverage(parsed);
case "serve":
return commandServe(parsed);
case "prompt":
return commandPrompt(parsed);
case "verify":
return commandVerify(parsed);
case "run-agent":
return commandRunAgent(parsed);
case "smoke":
return commandSmoke(parsed);
case "generate":
return commandGenerate(parsed);
case "scaffold":
return commandScaffold(parsed);
case "summarize":
return commandSummarize(parsed);
default:
throw new Error(`unknown command ${JSON.stringify(parsed.command)}`);
}
}
main().catch((error) => {
console.error(`browser eval: ${error instanceof Error ? error.message : String(error)}`);
process.exitCode = 1;
});
@@ -1,18 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "diagnostics",
routes: ["/diagnostics"],
render() {
return page({
title: "Diagnostics Lab",
body: `<section class="card"><h1>Diagnostics Lab</h1><p>Inspect console and network buffers.</p><p id="network-status" class="muted">Ping pending</p></section>`,
script: `
console.log("Browser eval console marker: CONSOLE-61");
fetch("/api/ping?run=" + encodeURIComponent(browserEval.runId))
.then((response) => response.json())
.then(({ token }) => { document.querySelector("#network-status").textContent = "Ping completed: " + token; });
`,
});
},
};
@@ -1,25 +0,0 @@
import { escapeHtml, page } from "../../lib/fixtures.mjs";
export default {
id: "form",
routes: ["/form", "/result"],
render({ pathname, runId, query, record }) {
if (pathname === "/form") {
return page({
title: "Browser Eval Form",
body: `<section class="card"><h1>Web form</h1><form action="/result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><label>Text input<input id="text-input" name="text" autocomplete="off"></label><label>Textarea<textarea id="notes" name="notes"></textarea></label><label>Dropdown<select id="choice" name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label><button id="submit" type="submit">Submit</button></form></section>`,
script: `document.querySelector("#submit").addEventListener("keydown", (event) => { if (event.key === "Enter") browserEval.send("form.enter_pressed"); });`,
});
}
const values = {
text: query.get("text") ?? "",
notes: query.get("notes") ?? "",
choice: query.get("choice") ?? "",
};
record("form.submitted", values, { path: pathname });
return page({
title: "Form Result",
body: `<section class="card"><h1>Received!</h1><p id="submitted-values" class="marker">${escapeHtml(values.text)} | ${escapeHtml(values.notes)} | ${escapeHtml(values.choice)}</p></section>`,
});
},
};
@@ -1,71 +0,0 @@
import { escapeHtml, page } from "../../lib/fixtures.mjs";
import { describeSeed } from "../../lib/seeded-generator.mjs";
function nest(content, depth) {
let nested = content;
for (let index = 0; index < depth; index += 1) {
nested = `<div data-depth="${index + 1}">${nested}</div>`;
}
return nested;
}
function textField(dimensions) {
const id = `matrix-text-${dimensions.idSuffix}`;
const input = `<input id="${id}" name="text" autocomplete="off">`;
if (dimensions.labelMode === "wrapped") return `<label>Text input${input}</label>`;
if (dimensions.labelMode === "for") return `<label for="${id}">Text input</label>${input}`;
return `<span id="${id}-label">Text input</span><input id="${id}" name="text" aria-labelledby="${id}-label" autocomplete="off">`;
}
function formMarkup(dimensions, runId) {
const text = textField(dimensions);
const select = `<label>Dropdown<select name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label>`;
const fields = dimensions.fieldOrder === "text-first" ? `${text}${select}` : `${select}${text}`;
const decoy =
dimensions.decoy === "disabled-input"
? `<label>Text input<input disabled value="decoy"></label>`
: dimensions.decoy === "duplicate-text"
? `<p class="muted">Text input is intentionally repeated in helper copy.</p>`
: "";
return nest(
`<form action="/matrix/generated-result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><input type="hidden" name="seed" value="${dimensions.seed}">${decoy}${fields}<button type="submit">Submit matrix form</button></form>`,
dimensions.nestingDepth,
);
}
export default {
id: "generated-form",
routes: ["/matrix/generated-form", "/matrix/generated-result"],
render({ pathname, runId, query, record }) {
const dimensions = describeSeed(query.get("seed") ?? 1);
if (pathname === "/matrix/generated-form") {
const markup = JSON.stringify(formMarkup(dimensions, runId));
const dimensionsJson = JSON.stringify(dimensions);
return page({
title: "Seeded Matrix Form",
body: `<section class="card"><h1>Seeded Matrix Form</h1><p class="marker">SEED-${dimensions.seed}</p><div id="matrix-root">Hydrating…</div></section>`,
script: `
const hydrate = () => {
const root = document.querySelector("#matrix-root");
root.innerHTML = ${markup};
root.querySelector("button[type=submit]").addEventListener("keydown", (event) => {
if (event.key === "Enter") browserEval.send("matrix.enter_pressed");
});
browserEval.send("matrix.ready", ${dimensionsJson});
};
setTimeout(hydrate, ${dimensions.hydrationDelayMs});
`,
});
}
const values = {
seed: dimensions.seed,
text: query.get("text") ?? "",
choice: query.get("choice") ?? "",
};
record("matrix.submitted", values, { path: pathname });
return page({
title: "Matrix Result",
body: `<section class="card"><h1>Matrix Result</h1><p class="marker">MATRIX-OK</p><p>${escapeHtml(values.text)} | ${escapeHtml(values.choice)} | seed ${values.seed}</p></section>`,
});
},
};
@@ -1,17 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "hover-inspect",
routes: ["/hover-inspect"],
render() {
return page({
title: "Hover and Inspect",
body: `<section class="card"><h1>Hover and Inspect</h1><div id="account-menu" tabindex="0"><button id="account-button" type="button">Account</button><div id="hover-panel"><button id="profile-link" type="button">Profile beta</button></div></div><p id="hover-result" class="marker" hidden>HOVER-17</p><article id="inspect-card" class="card" data-build="2026.08" data-marker="INSPECT-27"><h2>Inspection card</h2><p>Local HTML marker: <code>INSPECT-27</code></p></article></section>`,
script: `
const menu = document.querySelector("#account-menu");
menu.addEventListener("mouseenter", () => { menu.classList.add("open"); browserEval.send("hover.menu_opened"); });
document.querySelector("#profile-link").addEventListener("click", () => { document.querySelector("#hover-result").hidden = false; browserEval.send("hover.profile_clicked"); });
`,
});
},
};
@@ -1,12 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "fixture-index",
routes: ["/"],
render() {
return page({
title: "Browser Eval Fixtures",
body: `<section class="card"><h1>Browser Eval Fixtures</h1><p class="muted">Deterministic local pages for agent-neutral browser evaluation.</p></section>`,
});
},
};
@@ -1,19 +0,0 @@
import { page, withRun } from "../../lib/fixtures.mjs";
export default {
id: "navigation",
routes: ["/navigation/start", "/navigation/detail"],
render({ pathname, runId }) {
if (pathname === "/navigation/start") {
return page({
title: "Navigation Start",
body: `<section class="card"><h1>Navigation Start</h1><p>Use the link to open the deterministic detail page.</p><a id="details-link" class="button" href="${withRun("/navigation/detail", runId)}">Details</a></section>`,
script: `document.querySelector("#details-link").addEventListener("click", () => browserEval.send("navigation.detail.clicked"));`,
});
}
return page({
title: "Navigation Detail",
body: `<section class="card"><h1>Navigation Detail</h1><p>Stable detail code:</p><p id="navigation-code" class="marker">NAV-42</p></section>`,
});
},
};
@@ -1,21 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "responsive",
routes: ["/responsive"],
render() {
return page({
title: "Responsive Lab",
body: `<section class="card"><h1>Responsive Lab</h1><p id="viewport-marker" class="marker">VIEWPORT-84</p><p id="viewport-value"></p></section>`,
script: `
const sample = () => {
const data = { width: innerWidth, height: innerHeight, touchPoints: navigator.maxTouchPoints, userAgent: navigator.userAgent };
document.querySelector("#viewport-value").textContent = data.width + "x" + data.height;
browserEval.send("viewport.sample", data);
};
window.addEventListener("resize", sample);
sample();
`,
});
},
};
@@ -1,19 +0,0 @@
import { escapeHtml, page, withRun } from "../../lib/fixtures.mjs";
export default {
id: "tabs",
routes: ["/tabs", "/tabs/child"],
render({ pathname, runId }) {
if (pathname === "/tabs") {
return page({
title: "Tab Lab",
body: `<section class="card"><h1>Tab Lab</h1><p>Keep this original tab open while visiting the child page in a new tab.</p><p class="muted">Child URL: <code>${escapeHtml(withRun("/tabs/child", runId))}</code></p></section>`,
});
}
return page({
title: "Child Tab",
body: `<section class="card"><h1>Child Tab</h1><p id="tab-token" class="marker">TAB-29</p><button id="mark-child" type="button">Mark child visited</button><p id="child-status" hidden>Marked</p></section>`,
script: `document.querySelector("#mark-child").addEventListener("click", () => { document.querySelector("#child-status").hidden = false; browserEval.send("tabs.child_marked"); });`,
});
},
};
-99
View File
@@ -1,99 +0,0 @@
import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { countPattern, runProcess, substituteArgs } from "./process.mjs";
import { renderPrompt } from "./tasks.mjs";
function safeRunPart(value) {
return value.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 48);
}
export async function loadAgentConfig(configPath, agentName) {
const absolute = resolve(configPath);
const config = JSON.parse(await readFile(absolute, "utf8"));
const adapter = config.agents?.[agentName];
if (!adapter) {
const names = Object.keys(config.agents ?? {}).join(", ");
throw new Error(`unknown agent ${JSON.stringify(agentName)}; configured agents: ${names}`);
}
if (typeof adapter.command !== "string" || !Array.isArray(adapter.args)) {
throw new Error(`agent ${JSON.stringify(agentName)} must define command and args`);
}
return { adapter, configDirectory: dirname(absolute) };
}
export async function runCommandTask({
server,
serverInfo,
task,
adapter,
adapterName,
configDirectory,
iteration,
locale,
seed = task.seed,
timeoutMs,
variantOverride,
}) {
const variant = variantOverride ?? adapter.variant ?? "unspecified";
const runId = [
safeRunPart(adapterName),
safeRunPart(variant),
safeRunPart(task.id),
iteration,
randomUUID().slice(0, 8),
].join("-");
server.reset(runId);
const prompt = renderPrompt(task, { baseUrl: serverInfo.baseUrl, runId, locale, seed });
const values = {
prompt,
caseId: task.id,
taskId: task.id,
runId,
baseUrl: serverInfo.baseUrl,
seed: seed ?? "",
variant,
};
const args = substituteArgs(adapter.args, values);
const execution = await runProcess(adapter.command, args, {
cwd: adapter.cwd ? resolve(configDirectory, adapter.cwd) : configDirectory,
env: adapter.env,
timeoutMs: adapter.timeoutMs ?? timeoutMs,
});
await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
const responseText = execution.stdout;
const combinedOutput = `${execution.stdout}\n${execution.stderr}`;
const verification = verifyTask(task, {
events: server.snapshot(runId).events,
responseText,
});
const defaultErrorPattern = "(?:\\bError:|timed out|unfinished command|session already has)";
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
iteration,
adapter: adapterName,
variant,
adapterMetadata: adapter.metadata ?? {},
locale,
prompt,
execution,
metrics: {
errorCount: countPattern(
combinedOutput,
adapter.metrics?.errorPattern ?? defaultErrorPattern,
),
toolCallCount: countPattern(combinedOutput, adapter.metrics?.toolCallPattern),
responseChars: responseText.length,
},
eventCount: server.snapshot(runId).events.length,
verification,
};
}
-309
View File
@@ -1,309 +0,0 @@
import { mkdir, stat } from "node:fs/promises";
import { join } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { runProcess } from "./process.mjs";
import { taskUrl } from "./tasks.mjs";
function parseJsonOutput(step) {
try {
return JSON.parse(step.stdout);
} catch (error) {
throw new Error(`could not parse bsk JSON output: ${error.message}`);
}
}
function renderStepError(step) {
const raw = step.stderr.trim() || step.stdout.trim();
if (!raw) return `exit ${step.exitCode}`;
try {
const parsed = JSON.parse(raw);
return parsed.hint ? `${parsed.message} (hint: ${parsed.hint})` : parsed.message;
} catch {
return raw;
}
}
async function hasNonemptyFile(path) {
try {
return (await stat(path)).size > 0;
} catch {
return false;
}
}
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function resolveValue(value, variables) {
if (typeof value !== "string") return value;
const exact = value.match(/^\{([^{}]+)}$/);
if (exact) {
const resolved = readPath(variables, exact[1]);
if (resolved === undefined) throw new Error(`unknown workflow variable ${exact[1]}`);
return resolved;
}
return value.replaceAll(/\{([^{}]+)\}/g, (_match, path) => {
const resolved = readPath(variables, path);
if (resolved === undefined) throw new Error(`unknown workflow variable ${path}`);
return String(resolved);
});
}
function targetArgs(step, variables) {
if (step.selector) return ["--selector", String(resolveValue(step.selector, variables))];
if (step.ref) return ["--ref", String(resolveValue(step.ref, variables))];
return [];
}
function tabArgs(step, variables) {
return step.tabId === undefined ? [] : ["--tab-id", String(resolveValue(step.tabId, variables))];
}
function matchesWhere(value, where = {}) {
return Object.entries(where).every(([path, expected]) => readPath(value, path) === expected);
}
export async function runBskSmokeTask({
bskCommand,
outputDirectory,
server,
serverInfo,
task,
runId,
seed = task.seed,
timeoutMs = 60_000,
}) {
const steps = [];
const evidence = {};
const variables = {
baseUrl: serverInfo.baseUrl,
runId,
seed: seed ?? "",
url: taskUrl(task, serverInfo.baseUrl, runId, seed),
};
let sessionId;
let executionError;
await mkdir(outputDirectory, { recursive: true });
async function bsk(args, { timeout = timeoutMs } = {}) {
const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout });
steps.push(execution);
if (execution.exitCode !== 0 || execution.timedOut || execution.error) {
throw new Error(
`bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`,
);
}
return parseJsonOutput(execution);
}
async function waitForSiteEvent(step) {
const timeout = step.timeoutMs ?? 3_000;
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
if (
server
.snapshot(runId)
.events.some(
(event) => event.type === step.type && matchesWhere(event.data ?? {}, step.where ?? {}),
)
) {
return { observed: true };
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 50));
}
throw new Error(`local fixture did not report ${step.type} within ${timeout}ms`);
}
async function executeWorkflowStep(step) {
const session = ["--session", sessionId];
let result;
switch (step.action) {
case "navigate":
result = await bsk([
"navigate",
String(resolveValue(step.url ?? "{url}", variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "back":
case "forward":
result = await bsk([`navigate-${step.action}`, ...session, ...tabArgs(step, variables)]);
break;
case "reload":
result = await bsk(["reload", ...session, ...tabArgs(step, variables)]);
break;
case "wait":
result = await bsk(["wait-ms", String(resolveValue(step.duration, variables))]);
break;
case "wait-site-event":
result = await waitForSiteEvent(step);
break;
case "observe":
case "snapshot":
case "html": {
const command = step.action === "html" ? "get-html" : step.action;
result = await bsk([command, ...session, ...tabArgs(step, variables)]);
break;
}
case "screenshot": {
const name = resolveValue(step.artifactName ?? "screenshot.png", variables);
const path = join(outputDirectory, `${runId}-${name}`);
result = await bsk(["screenshot", ...session, ...tabArgs(step, variables), "--out", path]);
if (step.evidence) evidence[step.evidence] = await hasNonemptyFile(path);
break;
}
case "console":
case "network":
result = await bsk([step.action, ...session, ...tabArgs(step, variables)]);
break;
case "click":
case "hover":
result = await bsk([
step.action,
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "fill":
result = await bsk([
"fill",
...targetArgs(step, variables),
"--value",
String(resolveValue(step.value, variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "select":
result = await bsk([
"select",
...targetArgs(step, variables),
...step.values.flatMap((value) => ["--value", String(resolveValue(value, variables))]),
...session,
...tabArgs(step, variables),
]);
break;
case "press":
result = await bsk([
"press",
String(resolveValue(step.key, variables)),
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "tab-list":
result = await bsk(["tab", "list", ...session]);
break;
case "tab-create":
result = await bsk([
"tab",
"create",
...session,
...(step.url ? ["--url", String(resolveValue(step.url, variables))] : []),
]);
break;
case "tab-select":
case "tab-close":
result = await bsk([
"tab",
step.action.slice(4),
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "borrow":
case "return":
result = await bsk([
"tab",
step.action,
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "resize":
result = await bsk([
"window",
"resize",
...session,
"--width",
String(step.width),
"--height",
String(step.height),
]);
break;
case "emulate":
result = await bsk([
"emulate",
...session,
...tabArgs(step, variables),
"--device",
String(resolveValue(step.device, variables)),
]);
break;
case "request-help":
result = await bsk([
"request-help",
...session,
"--prompt",
String(resolveValue(step.prompt, variables)),
...(step.title ? ["--title", String(resolveValue(step.title, variables))] : []),
...(step.timeout ? ["--timeout", String(step.timeout)] : []),
]);
break;
default:
throw new Error(`unsupported smoke workflow action ${step.action}`);
}
if (step.saveAs) variables[step.saveAs] = result;
if (step.evidence && step.action !== "screenshot") evidence[step.evidence] = true;
}
server.reset(runId);
try {
if (task.smokeSteps.length === 0) throw new Error(`case ${task.id} has no smoke workflow`);
const started = await bsk(["session", "start", "--no-focus"], { timeout: 90_000 });
sessionId = started.session_id;
variables.sessionId = sessionId;
await bsk(["session", "list"]);
for (const step of task.smokeSteps) await executeWorkflowStep(step);
} catch (error) {
executionError = error instanceof Error ? error.message : String(error);
} finally {
if (sessionId) {
try {
const stopped = await bsk(["session", "stop", sessionId], { timeout: 90_000 });
evidence.sessionStopped =
stopped.stopped?.includes(sessionId) === true && (stopped.failed?.length ?? 0) === 0;
if (!evidence.sessionStopped) {
executionError ??= `bsk did not confirm that session ${sessionId} stopped`;
}
} catch (error) {
evidence.sessionStopped = false;
executionError ??= error instanceof Error ? error.message : String(error);
}
}
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 350));
const responseText = steps.map(({ stdout }) => stdout).join("\n");
const events = server.snapshot(runId).events;
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
adapter: "bsk-direct",
variant: "cli-direct",
executionError,
steps,
evidence,
eventCount: events.length,
verification: verifyTask(task, { events, responseText, adapterEvidence: evidence }),
};
}
-216
View File
@@ -1,216 +0,0 @@
import { readFile } from "node:fs/promises";
import { discoverFiles } from "./discovery.mjs";
import { OPERATION_NAMES, WORKFLOW_ACTIONS } from "./operations.mjs";
import { casesDirectory as defaultCasesDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const EVIDENCE_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
const TOP_LEVEL_KEYS = new Set([
"$schema",
"schemaVersion",
"id",
"title",
"order",
"suite",
"tags",
"seed",
"source",
"fixture",
"prompts",
"coverage",
"assertions",
"smoke",
]);
function isObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function checkString(errors, value, path) {
if (typeof value !== "string" || value.trim() === "") errors.push(`${path} must be a string`);
}
function validateAssertion(errors, assertion, path, kind) {
if (!isObject(assertion)) {
errors.push(`${path} must be an object`);
return;
}
checkString(errors, assertion.label, `${path}.label`);
if (kind === "site") {
checkString(errors, assertion.type, `${path}.type`);
if (!Number.isInteger(assertion.minCount) || assertion.minCount < 1) {
errors.push(`${path}.minCount must be a positive integer`);
}
if (assertion.where !== undefined && !isObject(assertion.where)) {
errors.push(`${path}.where must be an object`);
}
} else if (kind === "response") {
checkString(errors, assertion.includes, `${path}.includes`);
} else {
checkString(errors, assertion.key, `${path}.key`);
}
}
function validateWorkflowStep(errors, step, path) {
if (!isObject(step)) {
errors.push(`${path} must be an object`);
return;
}
if (!WORKFLOW_ACTIONS.has(step.action)) {
errors.push(`${path}.action is unsupported: ${JSON.stringify(step.action)}`);
}
if (step.saveAs !== undefined && !ID_PATTERN.test(step.saveAs)) {
errors.push(`${path}.saveAs must use lowercase kebab-case`);
}
if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) {
errors.push(`${path}.evidence must be an identifier`);
}
if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) {
errors.push(`${path} requires selector or ref`);
}
if (step.action === "fill") checkString(errors, step.value, `${path}.value`);
if (step.action === "select") {
if (!Array.isArray(step.values) || step.values.length === 0) {
errors.push(`${path}.values must be a non-empty array`);
} else if (step.values.some((value) => typeof value !== "string")) {
errors.push(`${path}.values must contain only strings`);
}
}
if (step.action === "press") checkString(errors, step.key, `${path}.key`);
if (step.action === "wait") checkString(errors, step.duration, `${path}.duration`);
if (step.action === "wait-site-event") checkString(errors, step.type, `${path}.type`);
if (["tab-select", "tab-close", "borrow", "return"].includes(step.action)) {
checkString(errors, step.tabId, `${path}.tabId`);
}
if (step.action === "resize") {
if (
!Number.isInteger(step.width) ||
step.width < 1 ||
!Number.isInteger(step.height) ||
step.height < 1
) {
errors.push(`${path}.width and height must be positive integers`);
}
}
if (step.action === "emulate") checkString(errors, step.device, `${path}.device`);
if (step.action === "request-help") checkString(errors, step.prompt, `${path}.prompt`);
if (step.timeoutMs !== undefined && (!Number.isInteger(step.timeoutMs) || step.timeoutMs < 1)) {
errors.push(`${path}.timeoutMs must be a positive integer`);
}
}
export function validateCaseManifest(manifest, { source = "case manifest" } = {}) {
const errors = [];
if (!isObject(manifest)) return [`${source} must contain a JSON object`];
for (const key of Object.keys(manifest)) {
if (!TOP_LEVEL_KEYS.has(key)) errors.push(`unknown top-level property ${key}`);
}
if (manifest.schemaVersion !== 1) errors.push("schemaVersion must equal 1");
if (!ID_PATTERN.test(manifest.id ?? "")) errors.push("id must use lowercase kebab-case");
checkString(errors, manifest.title, "title");
if (manifest.order !== undefined && (!Number.isInteger(manifest.order) || manifest.order < 0)) {
errors.push("order must be a non-negative integer");
}
if (!ID_PATTERN.test(manifest.suite ?? "")) errors.push("suite must use lowercase kebab-case");
if (!Array.isArray(manifest.tags) || manifest.tags.some((tag) => !ID_PATTERN.test(tag))) {
errors.push("tags must be an array of lowercase kebab-case strings");
} else if (new Set(manifest.tags).size !== manifest.tags.length) {
errors.push("tags must not contain duplicates");
}
if (!isObject(manifest.fixture)) errors.push("fixture must be an object");
else if (
typeof manifest.fixture.startPath !== "string" ||
!manifest.fixture.startPath.startsWith("/")
) {
errors.push("fixture.startPath must be an absolute URL path");
}
if (!isObject(manifest.prompts)) errors.push("prompts must be an object");
else {
checkString(errors, manifest.prompts.en, "prompts.en");
checkString(errors, manifest.prompts["zh-CN"], "prompts.zh-CN");
}
if (!Array.isArray(manifest.coverage)) errors.push("coverage must be an array");
else {
for (const operation of manifest.coverage) {
if (!OPERATION_NAMES.has(operation))
errors.push(`coverage contains unknown operation ${operation}`);
}
if (new Set(manifest.coverage).size !== manifest.coverage.length) {
errors.push("coverage must not contain duplicates");
}
}
if (!isObject(manifest.assertions)) errors.push("assertions must be an object");
else {
for (const kind of ["site", "response", "adapter"]) {
if (!Array.isArray(manifest.assertions[kind])) {
errors.push(`assertions.${kind} must be an array`);
} else {
manifest.assertions[kind].forEach((assertion, index) =>
validateAssertion(errors, assertion, `assertions.${kind}[${index}]`, kind),
);
}
}
}
if (!isObject(manifest.smoke) || !Array.isArray(manifest.smoke.steps)) {
errors.push("smoke.steps must be an array");
} else {
manifest.smoke.steps.forEach((step, index) =>
validateWorkflowStep(errors, step, `smoke.steps[${index}]`),
);
}
if (manifest.seed !== undefined && !Number.isSafeInteger(manifest.seed)) {
errors.push("seed must be a safe integer");
}
if (manifest.source !== undefined && !isObject(manifest.source)) {
errors.push("source must be an object");
}
return errors.map((error) => `${source}: ${error}`);
}
function normalizeCase(manifest, sourceFile) {
return Object.freeze({
...manifest,
startPath: manifest.fixture.startPath,
siteAssertions: manifest.assertions.site,
responseAssertions: manifest.assertions.response,
adapterAssertions: manifest.assertions.adapter,
smokeSteps: manifest.smoke.steps,
sourceFile,
});
}
export async function loadCaseManifests({ casesDirectory = defaultCasesDirectory } = {}) {
const files = await discoverFiles(casesDirectory, (path) => path.endsWith(".case.json"));
const cases = [];
const ids = new Map();
const errors = [];
for (const file of files) {
let manifest;
try {
manifest = JSON.parse(await readFile(file, "utf8"));
} catch (error) {
errors.push(`${file}: invalid JSON: ${error.message}`);
continue;
}
const manifestErrors = validateCaseManifest(manifest, { source: file });
errors.push(...manifestErrors);
if (typeof manifest?.id === "string") {
if (ids.has(manifest.id)) {
errors.push(`${file}: duplicate id also declared by ${ids.get(manifest.id)}`);
} else {
ids.set(manifest.id, file);
}
}
if (manifestErrors.length === 0) cases.push(normalizeCase(manifest, file));
}
if (files.length === 0) errors.push(`${casesDirectory}: no .case.json files found`);
if (errors.length > 0) throw new Error(`case validation failed:\n${errors.join("\n")}`);
cases.sort(
(left, right) =>
left.suite.localeCompare(right.suite) ||
(left.order ?? Number.MAX_SAFE_INTEGER) - (right.order ?? Number.MAX_SAFE_INTEGER) ||
left.id.localeCompare(right.id),
);
return Object.freeze(cases);
}
-26
View File
@@ -1,26 +0,0 @@
import { OPERATION_CATALOG, WORKFLOW_ACTION_OPERATIONS } from "./operations.mjs";
export function buildOperationCoverage(cases) {
return OPERATION_CATALOG.map(({ operation, manualNote }) => {
const agentCases = cases
.filter((testCase) => testCase.coverage.includes(operation))
.map(({ id }) => id);
let smokeCases;
if (["session.start", "session.stop", "session.list"].includes(operation)) {
smokeCases = cases.filter(({ smokeSteps }) => smokeSteps.length > 0).map(({ id }) => id);
} else {
smokeCases = cases
.filter(({ smokeSteps }) =>
smokeSteps.some((step) => WORKFLOW_ACTION_OPERATIONS[step.action] === operation),
)
.map(({ id }) => id);
}
return {
operation,
agentCases,
smokeCases,
directSmoke: smokeCases.length > 0,
note: smokeCases.length === 0 && manualNote ? `manual lane: ${manualNote}` : undefined,
};
});
}
-24
View File
@@ -1,24 +0,0 @@
import { readdir } from "node:fs/promises";
import { join } from "node:path";
export async function discoverFiles(root, predicate) {
const files = [];
async function visit(directory) {
let entries;
try {
entries = await readdir(directory, { withFileTypes: true });
} catch (error) {
if (error?.code === "ENOENT") return;
throw error;
}
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
const path = join(directory, entry.name);
if (entry.isDirectory()) await visit(path);
else if (entry.isFile() && predicate(path)) files.push(path);
}
}
await visit(root);
return files;
}
-68
View File
@@ -1,68 +0,0 @@
import { pathToFileURL } from "node:url";
import { discoverFiles } from "./discovery.mjs";
import { casesDirectory, fixturesDirectory } from "./paths.mjs";
export async function loadFixtureRegistry({ roots = [fixturesDirectory, casesDirectory] } = {}) {
const files = (
await Promise.all(
roots.map((root) => discoverFiles(root, (path) => path.endsWith(".fixture.mjs"))),
)
).flat();
const routes = new Map();
const definitions = [];
const errors = [];
for (const file of files.sort()) {
let definition;
try {
definition = (await import(pathToFileURL(file).href)).default;
} catch (error) {
errors.push(`${file}: could not import fixture: ${error.message}`);
continue;
}
if (!definition || typeof definition.id !== "string") {
errors.push(`${file}: default export must define id`);
continue;
}
if (!Array.isArray(definition.routes) || definition.routes.length === 0) {
errors.push(`${file}: default export must define non-empty routes`);
continue;
}
if (typeof definition.render !== "function") {
errors.push(`${file}: default export must define render(context)`);
continue;
}
definitions.push({ ...definition, sourceFile: file });
for (const route of definition.routes) {
if (typeof route !== "string" || !route.startsWith("/")) {
errors.push(`${file}: invalid route ${JSON.stringify(route)}`);
} else if (routes.has(route)) {
errors.push(
`${file}: route ${route} is already registered by ${routes.get(route).sourceFile}`,
);
} else {
routes.set(route, { definition, sourceFile: file });
}
}
}
if (errors.length > 0) throw new Error(`fixture validation failed:\n${errors.join("\n")}`);
return Object.freeze({
definitions: Object.freeze(definitions),
routes: Object.freeze([...routes.keys()].sort()),
hasRoute(pathname) {
return routes.has(pathname);
},
render(pathname, context) {
const entry = routes.get(pathname);
return entry ? entry.definition.render({ ...context, pathname }) : null;
},
});
}
export function validateCaseFixtureLinks(cases, fixtureRegistry) {
return cases
.filter(({ startPath }) => !fixtureRegistry.hasRoute(startPath))
.map(({ id, startPath }) => `case ${id} references unregistered fixture route ${startPath}`);
}
-78
View File
@@ -1,78 +0,0 @@
const stylesheet = `
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
body { margin: 0; background: #f4f7fb; color: #182033; }
main { width: min(760px, calc(100% - 40px)); margin: 48px auto; }
.card { background: white; border: 1px solid #d9e1ef; border-radius: 14px; padding: 24px; box-shadow: 0 8px 28px #20304a12; }
h1 { margin-top: 0; }
a, button { font: inherit; }
button, .button { display: inline-block; border: 0; border-radius: 8px; padding: 10px 16px; background: #3157d5; color: white; cursor: pointer; text-decoration: none; }
label { display: block; margin: 16px 0; font-weight: 600; }
input, textarea, select { display: block; box-sizing: border-box; width: 100%; margin-top: 6px; border: 1px solid #9caac2; border-radius: 7px; padding: 9px; font: inherit; }
textarea { min-height: 90px; }
.marker { border-left: 4px solid #3157d5; padding: 8px 12px; background: #eef2ff; font-family: ui-monospace, monospace; }
.muted { color: #5c667a; }
#account-menu { position: relative; display: inline-block; padding-bottom: 12px; }
#hover-panel { display: none; position: absolute; z-index: 2; top: 34px; left: 0; width: 190px; padding: 10px; background: white; border: 1px solid #9caac2; border-radius: 8px; box-shadow: 0 8px 24px #20304a22; }
#account-menu:hover #hover-panel, #account-menu.open #hover-panel { display: block; }
#hover-panel button { width: 100%; }
#inspect-card { margin-top: 80px; }
code { font-family: ui-monospace, monospace; }
`;
const clientScript = `
(() => {
const runId = new URLSearchParams(location.search).get("run");
const navigation = performance.getEntriesByType("navigation")[0];
const send = (type, data = {}) => {
if (!runId) return Promise.resolve();
return fetch("/api/events", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId, type, path: location.pathname, data }),
keepalive: true,
}).catch(() => undefined);
};
window.browserEval = { runId, send };
window.addEventListener("pageshow", (event) => {
send("page.shown", {
path: location.pathname,
persisted: event.persisted,
navigationType: navigation?.type ?? "unknown",
});
});
})();
`;
export function escapeHtml(value) {
return String(value)
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;");
}
export function withRun(path, runId, extra = {}) {
const url = new URL(path, "http://browser-eval.local");
url.searchParams.set("run", runId);
for (const [name, value] of Object.entries(extra)) {
if (value !== undefined) url.searchParams.set(name, String(value));
}
return `${url.pathname}${url.search}`;
}
export function page({ title, body, script = "" }) {
return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>${escapeHtml(title)}</title>
<style>${stylesheet}</style>
</head>
<body>
<main>${body}</main>
<script>${clientScript}</script>
${script ? `<script>${script}</script>` : ""}
</body>
</html>`;
}
-68
View File
@@ -1,68 +0,0 @@
export const OPERATION_CATALOG = [
{ operation: "session.start" },
{ operation: "session.stop" },
{ operation: "session.list" },
{ operation: "page.navigate" },
{ operation: "page.back" },
{ operation: "page.forward" },
{ operation: "page.reload" },
{ operation: "page.wait" },
{ operation: "inspect.observe" },
{ operation: "inspect.snapshot" },
{ operation: "inspect.html" },
{ operation: "inspect.screenshot" },
{ operation: "inspect.console" },
{ operation: "inspect.network" },
{ operation: "interact.click" },
{ operation: "interact.hover" },
{ operation: "interact.fill" },
{ operation: "interact.select" },
{ operation: "interact.press" },
{ operation: "tabs.list" },
{ operation: "tabs.create" },
{ operation: "tabs.select" },
{ operation: "tabs.close" },
{
operation: "tabs.borrow",
manualNote: "requires an existing user tab and ownership confirmation",
},
{ operation: "tabs.return", manualNote: "paired with borrowing a real user tab" },
{ operation: "assist.resize" },
{ operation: "assist.emulate" },
{ operation: "assist.request-help", manualNote: "intentionally waits for human interaction" },
];
export const WORKFLOW_ACTION_OPERATIONS = {
navigate: "page.navigate",
back: "page.back",
forward: "page.forward",
reload: "page.reload",
wait: "page.wait",
observe: "inspect.observe",
snapshot: "inspect.snapshot",
html: "inspect.html",
screenshot: "inspect.screenshot",
console: "inspect.console",
network: "inspect.network",
click: "interact.click",
hover: "interact.hover",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
"tab-list": "tabs.list",
"tab-create": "tabs.create",
"tab-select": "tabs.select",
"tab-close": "tabs.close",
borrow: "tabs.borrow",
return: "tabs.return",
resize: "assist.resize",
emulate: "assist.emulate",
"request-help": "assist.request-help",
};
export const WORKFLOW_ACTIONS = new Set([
...Object.keys(WORKFLOW_ACTION_OPERATIONS),
"wait-site-event",
]);
export const OPERATION_NAMES = new Set(OPERATION_CATALOG.map(({ operation }) => operation));
-63
View File
@@ -1,63 +0,0 @@
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function eventMatches(event, assertion) {
if (event.type !== assertion.type) return false;
return Object.entries(assertion.where ?? {}).every(
([path, expected]) => readPath(event, path) === expected,
);
}
export function verifyTask(task, { events = [], responseText = "", adapterEvidence = {} } = {}) {
const site = task.siteAssertions.map((assertion) => {
const actual = events.filter((event) => eventMatches(event, assertion)).length;
return {
kind: "site",
label: assertion.label,
status: actual >= assertion.minCount ? "passed" : "failed",
expected: `at least ${assertion.minCount}`,
actual,
};
});
const response = task.responseAssertions.map((assertion) => ({
kind: "response",
label: assertion.label,
status: responseText.includes(assertion.includes) ? "passed" : "failed",
expected: `response includes ${JSON.stringify(assertion.includes)}`,
}));
const adapter = task.adapterAssertions.map((assertion) => {
if (!Object.hasOwn(adapterEvidence, assertion.key)) {
return {
kind: "adapter",
label: assertion.label,
status: "unverified",
expected: assertion.key,
};
}
return {
kind: "adapter",
label: assertion.label,
status: adapterEvidence[assertion.key] ? "passed" : "failed",
expected: assertion.key,
actual: adapterEvidence[assertion.key],
};
});
const checks = [...site, ...response, ...adapter];
const failed = checks.filter(({ status }) => status === "failed").length;
const unverified = checks.filter(({ status }) => status === "unverified").length;
return {
taskId: task.id,
status: failed > 0 ? "failed" : unverified > 0 ? "passed-with-unverified" : "passed",
summary: {
passed: checks.length - failed - unverified,
failed,
unverified,
total: checks.length,
},
checks,
};
}
-8
View File
@@ -1,8 +0,0 @@
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export const evalDirectory = dirname(dirname(fileURLToPath(import.meta.url)));
export const casesDirectory = join(evalDirectory, "cases");
export const fixturesDirectory = join(evalDirectory, "fixtures");
export const schemasDirectory = join(evalDirectory, "schemas");
export const resultsDirectory = join(evalDirectory, "results");
-71
View File
@@ -1,71 +0,0 @@
import { spawn } from "node:child_process";
const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
function appendCaptured(current, chunk) {
if (current.length >= MAX_CAPTURE_BYTES) return current;
return `${current}${chunk}`.slice(0, MAX_CAPTURE_BYTES);
}
export function runProcess(command, args, { cwd, env, timeoutMs = 300_000 } = {}) {
return new Promise((resolve) => {
const startedAt = Date.now();
let stdout = "";
let stderr = "";
let timedOut = false;
let settled = false;
const child = spawn(command, args, {
cwd,
env: { ...process.env, ...env },
shell: false,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout.on("data", (chunk) => {
stdout = appendCaptured(stdout, chunk.toString());
});
child.stderr.on("data", (chunk) => {
stderr = appendCaptured(stderr, chunk.toString());
});
const timeout = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
}, timeoutMs);
const finish = (result) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
resolve({
command,
args,
stdout,
stderr,
timedOut,
durationMs: Date.now() - startedAt,
...result,
});
};
child.once("error", (error) => finish({ exitCode: null, signal: null, error: error.message }));
child.once("close", (exitCode, signal) => finish({ exitCode, signal }));
});
}
export function substituteArgs(args, values) {
return args.map((argument) => {
let result = argument;
for (const [key, value] of Object.entries(values)) {
result = result.replaceAll(`{${key}}`, String(value));
}
return result;
});
}
export function countPattern(text, pattern) {
if (!pattern) return null;
const matches = text.match(new RegExp(pattern, "g"));
return matches?.length ?? 0;
}
@@ -1,64 +0,0 @@
import { relative, sep } from "node:path";
import { validateCaseFixtureLinks } from "./fixture-registry.mjs";
import { casesDirectory } from "./paths.mjs";
function templateRoots(value) {
if (typeof value === "string") {
return [...value.matchAll(/\{([^{}]+)\}/g)].map((match) => match[1].split(".")[0]);
}
if (Array.isArray(value)) return value.flatMap(templateRoots);
if (value && typeof value === "object") return Object.values(value).flatMap(templateRoots);
return [];
}
export function validateRepositoryCases(cases, fixtureRegistry) {
const errors = [...validateCaseFixtureLinks(cases, fixtureRegistry)];
for (const testCase of cases) {
const relativePath = relative(casesDirectory, testCase.sourceFile);
const suiteDirectory = relativePath.split(sep)[0];
if (suiteDirectory !== testCase.suite) {
errors.push(
`case ${testCase.id} declares suite ${testCase.suite} but is stored under ${suiteDirectory}`,
);
}
const availableVariables = new Set(["baseUrl", "runId", "seed", "sessionId", "url"]);
const evidence = new Set(["sessionStopped"]);
for (const [index, step] of testCase.smokeSteps.entries()) {
for (const root of templateRoots(step)) {
if (!availableVariables.has(root)) {
errors.push(
`case ${testCase.id} smoke.steps[${index}] references unavailable variable ${root}`,
);
}
}
if (step.saveAs) availableVariables.add(step.saveAs);
if (step.evidence) evidence.add(step.evidence);
}
for (const assertion of testCase.adapterAssertions) {
if (!evidence.has(assertion.key)) {
errors.push(
`case ${testCase.id} expects adapter evidence ${assertion.key} but no smoke step produces it`,
);
}
}
}
return errors;
}
export function repositorySummary(cases, fixtureRegistry) {
const suites = Object.fromEntries(
[...new Set(cases.map(({ suite }) => suite))]
.sort()
.map((suite) => [suite, cases.filter((testCase) => testCase.suite === suite).length]),
);
const tags = [...new Set(cases.flatMap(({ tags }) => tags))].sort();
return {
cases: cases.length,
suites,
tags: tags.length,
fixtureModules: fixtureRegistry.definitions.length,
fixtureRoutes: fixtureRegistry.routes.length,
};
}
-120
View File
@@ -1,120 +0,0 @@
import { mkdir, realpath, stat, writeFile } from "node:fs/promises";
import { join, relative, sep } from "node:path";
import { casesDirectory, evalDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
async function exists(path) {
try {
await stat(path);
return true;
} catch (error) {
if (error?.code === "ENOENT") return false;
throw error;
}
}
async function moduleSpecifier(fromDirectory, target) {
const path = relative(await realpath(fromDirectory), await realpath(target))
.split(sep)
.join("/");
return path.startsWith(".") ? path : `./${path}`;
}
export async function scaffoldCase({
id,
title = id,
suite = "regression",
source,
rootCasesDirectory = casesDirectory,
}) {
if (!ID_PATTERN.test(id ?? "")) throw new Error("case id must use lowercase kebab-case");
if (!ID_PATTERN.test(suite ?? "")) throw new Error("suite must use lowercase kebab-case");
const directory = join(rootCasesDirectory, suite, id);
if (await exists(directory)) throw new Error(`case directory already exists: ${directory}`);
await mkdir(directory, { recursive: true });
const marker = `CASE-${id.toUpperCase().replaceAll("-", "_")}`;
const startPath = `/cases/${id}`;
const manifest = {
$schema: "../../../schemas/case.schema.json",
schemaVersion: 1,
id,
title,
order: 10,
suite,
tags: ["badcase", "regression"],
...(source ? { source: { type: "user-badcase", reference: source } } : {}),
fixture: { startPath },
prompts: {
en: `Open {url}, complete the minimal regression scenario, report ${marker}, and close the browser session.`,
"zh-CN": `打开 {url},完成最小回归场景,报告 ${marker},最后关闭浏览器会话。`,
},
coverage: ["session.start", "session.stop", "page.navigate", "inspect.observe"],
assertions: {
site: [
{
label: "regression fixture was displayed",
type: "page.shown",
where: { "data.path": startPath },
minCount: 1,
},
],
response: [{ label: "reported regression marker", includes: marker }],
adapter: [{ label: "browser session was closed", key: "sessionStopped" }],
},
smoke: { steps: [{ action: "navigate" }, { action: "observe" }] },
};
const fixtureHelper = await moduleSpecifier(
directory,
join(evalDirectory, "lib", "fixtures.mjs"),
);
const fixture = `import { escapeHtml, page } from ${JSON.stringify(fixtureHelper)};
const title = ${JSON.stringify(title)};
export default {
id: ${JSON.stringify(id)},
routes: [${JSON.stringify(startPath)}],
render() {
return page({
title,
body: \`<section class="card"><h1>\${escapeHtml(title)}</h1><p class="marker">${marker}</p><p>Replace this page with a privacy-safe minimal reproduction.</p></section>\`,
});
},
};
`;
const readme = `# ${title}
## Original symptom
Describe what the user observed without including private data or proprietary page content.
## Minimal reproduction
Explain the smallest DOM, timing, or browser-state condition that reproduces the failure.
## Expected behavior
Describe the observable success condition and update the case assertions accordingly.
## Regression history
- Source: ${source ?? "add issue, PR, or anonymized user report reference"}
- Fix: add the fixing PR when available
`;
const files = {
manifest: join(directory, `${id}.case.json`),
fixture: join(directory, `${id}.fixture.mjs`),
readme: join(directory, "README.md"),
};
await Promise.all([
writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }),
writeFile(files.fixture, fixture, { flag: "wx" }),
writeFile(files.readme, readme, { flag: "wx" }),
]);
return { directory, files };
}
-40
View File
@@ -1,40 +0,0 @@
function hashSeed(value) {
const text = String(value);
let hash = 2166136261;
for (const character of text) {
hash ^= character.codePointAt(0);
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
function mulberry32(seed) {
let state = seed >>> 0;
return () => {
state += 0x6d2b79f5;
let value = state;
value = Math.imul(value ^ (value >>> 15), value | 1);
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
};
}
export function normalizeSeed(value) {
const numeric = Number(value);
return Number.isSafeInteger(numeric) ? numeric : hashSeed(value);
}
export function describeSeed(value) {
const seed = normalizeSeed(value);
const random = mulberry32(seed);
const choose = (values) => values[Math.floor(random() * values.length)];
return {
seed,
labelMode: choose(["wrapped", "for", "aria"]),
nestingDepth: 1 + Math.floor(random() * 4),
hydrationDelayMs: choose([0, 40, 120, 250]),
decoy: choose(["none", "disabled-input", "duplicate-text"]),
fieldOrder: choose(["text-first", "select-first"]),
idSuffix: hashSeed(`${seed}:id`).toString(36).slice(0, 6),
};
}
-152
View File
@@ -1,152 +0,0 @@
import { createServer } from "node:http";
import { loadFixtureRegistry } from "./fixture-registry.mjs";
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]{1,120}$/;
function validRunId(value) {
return typeof value === "string" && RUN_ID_PATTERN.test(value);
}
function json(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "application/json; charset=utf-8",
});
response.end(`${JSON.stringify(value, null, 2)}\n`);
}
function html(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "text/html; charset=utf-8",
});
response.end(value);
}
async function readJson(request) {
let raw = "";
for await (const chunk of request) {
raw += chunk;
if (raw.length > 64 * 1024) {
throw new Error("request body exceeds 64 KiB");
}
}
return raw ? JSON.parse(raw) : {};
}
export function createEvalServer({ host = "127.0.0.1", port = 0, fixtureRegistry } = {}) {
const runs = new Map();
let sequence = 0;
let server;
let registry = fixtureRegistry;
function eventsFor(runId) {
if (!runs.has(runId)) runs.set(runId, []);
return runs.get(runId);
}
function record(runId, type, data = {}, metadata = {}) {
if (!validRunId(runId)) return null;
const event = {
sequence: ++sequence,
timestamp: new Date().toISOString(),
type,
data,
...metadata,
};
eventsFor(runId).push(event);
return event;
}
function snapshot(runId) {
return { runId, events: [...(runs.get(runId) ?? [])] };
}
function reset(runId) {
runs.set(runId, []);
}
async function handle(request, response) {
const origin = `http://${request.headers.host ?? `${host}:${port}`}`;
const url = new URL(request.url ?? "/", origin);
if (request.method === "GET" && url.pathname === "/healthz") {
return json(response, 200, { ok: true });
}
if (request.method === "POST" && url.pathname === "/api/events") {
const body = await readJson(request);
if (!validRunId(body.runId) || typeof body.type !== "string") {
return json(response, 400, { error: "runId and type are required" });
}
const event = record(body.runId, body.type, body.data ?? {}, {
path: typeof body.path === "string" ? body.path : undefined,
});
return json(response, 202, { accepted: true, sequence: event.sequence });
}
const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/);
if (runMatch) {
const runId = decodeURIComponent(runMatch[1]);
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
if (request.method === "GET") return json(response, 200, snapshot(runId));
if (request.method === "DELETE") {
reset(runId);
return json(response, 200, { runId, reset: true });
}
}
if (request.method === "GET" && url.pathname === "/api/ping") {
const runId = url.searchParams.get("run");
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
record(runId, "network.ping", { token: "NETWORK-73" }, { path: url.pathname });
return json(response, 200, { ok: true, token: "NETWORK-73" });
}
if (request.method !== "GET") return json(response, 405, { error: "method not allowed" });
const runId = url.searchParams.get("run") ?? "manual";
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
const fixture = registry.render(url.pathname, {
runId,
query: url.searchParams,
record: (type, data, metadata) => record(runId, type, data, metadata),
});
if (!fixture) return html(response, 404, "<!doctype html><h1>Not found</h1>");
record(runId, "page.request", { path: url.pathname }, { path: url.pathname });
return html(response, 200, fixture);
}
return {
record,
reset,
snapshot,
async start() {
if (server) throw new Error("eval server is already running");
registry ??= await loadFixtureRegistry();
server = createServer((request, response) => {
handle(request, response).catch((error) => {
json(response, 500, { error: error instanceof Error ? error.message : String(error) });
});
});
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, host, resolve);
});
const address = server.address();
if (!address || typeof address === "string") throw new Error("unexpected server address");
return { host, port: address.port, baseUrl: `http://${host}:${address.port}` };
},
async stop() {
if (!server) return;
const active = server;
server = undefined;
await new Promise((resolve, reject) => {
active.close((error) => (error ? reject(error) : resolve()));
active.closeAllConnections?.();
});
},
};
}
-73
View File
@@ -1,73 +0,0 @@
function durationFor(result) {
if (result.execution?.durationMs !== undefined) return result.execution.durationMs;
return (result.steps ?? []).reduce((sum, step) => sum + step.durationMs, 0);
}
export function summarizeReports(reports) {
const groups = new Map();
for (const report of reports) {
for (const result of report.results ?? []) {
const adapter = result.adapter ?? report.adapter ?? "unknown";
const variant = result.variant ?? report.variant ?? "unspecified";
const key = `${adapter}\u0000${variant}`;
if (!groups.has(key)) {
groups.set(key, {
adapter,
variant,
runs: 0,
passed: 0,
fullyVerified: 0,
executionFailures: 0,
errors: 0,
durationMs: 0,
toolCalls: 0,
toolCallSamples: 0,
});
}
const group = groups.get(key);
group.runs += 1;
group.durationMs += durationFor(result);
group.errors += result.metrics?.errorCount ?? (result.executionError ? 1 : 0);
if (result.metrics?.toolCallCount !== null && result.metrics?.toolCallCount !== undefined) {
group.toolCalls += result.metrics.toolCallCount;
group.toolCallSamples += 1;
}
const executionFailed =
Boolean(result.executionError) ||
result.execution?.timedOut ||
(result.execution?.exitCode !== undefined && result.execution.exitCode !== 0);
if (executionFailed) group.executionFailures += 1;
if (result.verification?.status !== "failed" && !executionFailed) group.passed += 1;
if (result.verification?.status === "passed" && !executionFailed) group.fullyVerified += 1;
}
}
return [...groups.values()].map((group) => ({
adapter: group.adapter,
variant: group.variant,
runs: group.runs,
passRate: group.runs ? group.passed / group.runs : 0,
fullyVerifiedRate: group.runs ? group.fullyVerified / group.runs : 0,
executionFailures: group.executionFailures,
errors: group.errors,
averageToolCalls: group.toolCallSamples
? Math.round((group.toolCalls / group.toolCallSamples) * 10) / 10
: null,
averageDurationMs: group.runs ? Math.round(group.durationMs / group.runs) : 0,
}));
}
export function printSummary(rows) {
const printable = rows.map((row) => ({
adapter: row.adapter,
variant: row.variant,
runs: row.runs,
pass: `${Math.round(row.passRate * 100)}%`,
verified: `${Math.round(row.fullyVerifiedRate * 100)}%`,
exec_failures: row.executionFailures,
errors: row.errors,
avg_tools: row.averageToolCalls ?? "-",
avg_ms: row.averageDurationMs,
}));
console.table(printable);
}
-48
View File
@@ -1,48 +0,0 @@
import { loadCaseManifests } from "./case-loader.mjs";
let casesPromise;
export function allTasks({ reload = false } = {}) {
if (reload || !casesPromise) casesPromise = loadCaseManifests();
return casesPromise;
}
export function listTasks(cases) {
return cases.map(({ id, title, suite, tags, seed }) => ({ id, title, suite, tags, seed }));
}
export function getTask(taskId, cases) {
const task = cases.find(({ id }) => id === taskId);
if (!task) {
const ids = cases.map(({ id }) => id).join(", ");
throw new Error(`unknown case ${JSON.stringify(taskId)}; expected one of: ${ids}`);
}
return task;
}
export function taskUrl(task, baseUrl, runId, seed = task.seed) {
const url = new URL(task.startPath, baseUrl);
url.searchParams.set("run", runId);
if (seed !== undefined) url.searchParams.set("seed", String(seed));
return url.toString();
}
function expandTemplate(template, values) {
return Object.entries(values).reduce(
(result, [name, value]) => result.replaceAll(`{${name}}`, String(value)),
template,
);
}
export function renderPrompt(task, { baseUrl, runId, locale = "zh-CN", seed = task.seed }) {
const template = task.prompts[locale];
if (!template) {
throw new Error(`unsupported locale ${JSON.stringify(locale)}; expected en or zh-CN`);
}
return expandTemplate(template, {
baseUrl,
runId: encodeURIComponent(runId),
seed: seed ?? "",
url: taskUrl(task, baseUrl, runId, seed),
});
}
-76
View File
@@ -1,76 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://browser-skill.local/schemas/browser-eval-case.schema.json",
"title": "BrowserSkill browser evaluation case",
"type": "object",
"required": [
"schemaVersion",
"id",
"title",
"suite",
"tags",
"fixture",
"prompts",
"coverage",
"assertions",
"smoke"
],
"properties": {
"$schema": { "type": "string" },
"schemaVersion": { "const": 1 },
"id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"title": { "type": "string", "minLength": 1 },
"order": { "type": "integer", "minimum": 0 },
"suite": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"tags": {
"type": "array",
"items": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"uniqueItems": true
},
"fixture": {
"type": "object",
"required": ["startPath"],
"properties": { "startPath": { "type": "string", "pattern": "^/" } },
"additionalProperties": false
},
"prompts": {
"type": "object",
"required": ["en", "zh-CN"],
"properties": {
"en": { "type": "string", "minLength": 1 },
"zh-CN": { "type": "string", "minLength": 1 }
},
"additionalProperties": false
},
"coverage": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
"assertions": {
"type": "object",
"required": ["site", "response", "adapter"],
"properties": {
"site": { "type": "array", "items": { "type": "object" } },
"response": { "type": "array", "items": { "type": "object" } },
"adapter": { "type": "array", "items": { "type": "object" } }
},
"additionalProperties": false
},
"smoke": {
"type": "object",
"required": ["steps"],
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"required": ["action"],
"properties": { "action": { "type": "string" } },
"additionalProperties": true
}
}
},
"additionalProperties": false
},
"seed": { "type": "integer" },
"source": { "type": "object" }
},
"additionalProperties": false
}
-328
View File
@@ -1,328 +0,0 @@
import assert from "node:assert/strict";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import test from "node:test";
import { runCommandTask } from "../lib/agent-runner.mjs";
import { loadCaseManifests, validateCaseManifest } from "../lib/case-loader.mjs";
import { buildOperationCoverage } from "../lib/coverage.mjs";
import { loadFixtureRegistry } from "../lib/fixture-registry.mjs";
import { verifyTask } from "../lib/oracle.mjs";
import { runProcess } from "../lib/process.mjs";
import { repositorySummary, validateRepositoryCases } from "../lib/repository-validation.mjs";
import { scaffoldCase } from "../lib/scaffold-case.mjs";
import { describeSeed } from "../lib/seeded-generator.mjs";
import { createEvalServer } from "../lib/server.mjs";
import { summarizeReports } from "../lib/summary.mjs";
import { allTasks, getTask, renderPrompt } from "../lib/tasks.mjs";
const cases = await allTasks();
const fixtureRegistry = await loadFixtureRegistry();
async function withServer(run) {
const server = createEvalServer({ fixtureRegistry });
const info = await server.start();
try {
await run(server, info);
} finally {
await server.stop();
}
}
test("case manifests are discovered, ordered, and grouped into suites", () => {
assert.deepEqual(
cases.map(({ id }) => id),
[
"navigation-history",
"form-controls",
"hover-inspect",
"tabs",
"diagnostics",
"mobile-emulation",
"generated-form",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
const invalid = {
schemaVersion: 1,
id: "invalid-case",
title: "Invalid",
suite: "regression",
tags: ["regression"],
fixture: { startPath: "/invalid" },
prompts: { en: "Open {url}", "zh-CN": "打开 {url}" },
coverage: ["page.teleport"],
assertions: { site: [], response: [], adapter: [] },
smoke: { steps: [{ action: "select", selector: "#choice", values: [] }] },
typo: true,
};
const errors = validateCaseManifest(invalid);
assert.ok(errors.some((error) => error.includes("unknown top-level property typo")));
assert.ok(errors.some((error) => error.includes("unknown operation page.teleport")));
assert.ok(errors.some((error) => error.includes("values must be a non-empty array")));
});
test("fixture server exposes deterministic task and seeded matrix markers", async () => {
await withServer(async (_server, { baseUrl }) => {
const fixtures = [
["/navigation/detail?run=test-markers", "NAV-42"],
["/form?run=test-markers", '<option value="two">Two</option>'],
["/hover-inspect?run=test-markers", "INSPECT-27"],
["/tabs/child?run=test-markers", "TAB-29"],
["/diagnostics?run=test-markers", "CONSOLE-61"],
["/responsive?run=test-markers", "VIEWPORT-84"],
["/matrix/generated-form?run=test-markers&seed=17", "SEED-17"],
];
for (const [path, marker] of fixtures) {
const response = await fetch(`${baseUrl}${path}`);
assert.equal(response.status, 200);
assert.match(await response.text(), new RegExp(marker));
}
});
});
test("server records and resets run-scoped events", async () => {
await withServer(async (server, { baseUrl }) => {
const response = await fetch(`${baseUrl}/api/events`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId: "event-test", type: "example", data: { value: 7 } }),
});
assert.equal(response.status, 202);
assert.equal(server.snapshot("event-test").events[0].data.value, 7);
const resetResponse = await fetch(`${baseUrl}/api/runs/event-test`, { method: "DELETE" });
assert.equal(resetResponse.status, 200);
assert.deepEqual(server.snapshot("event-test").events, []);
});
});
test("form case oracle checks submitted values and separates adapter evidence", () => {
const task = getTask("form-controls", cases);
const events = [
{ type: "form.enter_pressed", data: {} },
{
type: "form.submitted",
data: { text: "agent-parity", notes: "BrowserSkill works", choice: "two" },
},
{ type: "page.shown", data: { path: "/result" } },
];
const partial = verifyTask(task, { events, responseText: "The page says Received!" });
assert.equal(partial.status, "passed-with-unverified");
assert.equal(partial.summary.unverified, 2);
const complete = verifyTask(task, {
events,
responseText: "The page says Received!",
adapterEvidence: { screenshotCreated: true, sessionStopped: true },
});
assert.equal(complete.status, "passed");
});
test("navigation case fails when reload is missing", () => {
const task = getTask("navigation-history", cases);
const verification = verifyTask(task, {
events: [
{ type: "navigation.detail.clicked", data: {} },
{ type: "page.shown", data: { path: "/navigation/start" } },
{ type: "page.shown", data: { path: "/navigation/start" } },
{ type: "page.shown", data: { path: "/navigation/detail" } },
{ type: "page.shown", data: { path: "/navigation/detail" } },
],
responseText: "NAV-42",
});
assert.equal(verification.status, "failed");
assert.equal(
verification.checks.find(({ label }) => label === "detail page was reloaded").status,
"failed",
);
});
test("prompt rendering is run-scoped, localized, and seed-aware", () => {
const task = getTask("generated-form", cases);
const input = { baseUrl: "http://127.0.0.1:4173", runId: "prompt-test", seed: 17 };
assert.match(renderPrompt(task, { ...input, locale: "en" }), /seed=17/);
assert.match(renderPrompt(task, { ...input, locale: "zh-CN" }), /seeded form/);
});
test("command adapter carries a run from process launch through oracle verification", async () => {
await withServer(async (server, serverInfo) => {
const script = `
const [baseUrl, runId] = process.argv.slice(1);
const send = (type, data) => fetch(baseUrl + "/api/events", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId, type, data }),
});
await send("navigation.detail.clicked", {});
await send("page.shown", { path: "/navigation/start" });
await send("page.shown", { path: "/navigation/start" });
await send("page.shown", { path: "/navigation/detail" });
await send("page.shown", { path: "/navigation/detail" });
await send("page.shown", { path: "/navigation/detail", navigationType: "reload" });
console.log("NAV-42");
`;
const result = await runCommandTask({
server,
serverInfo,
task: getTask("navigation-history", cases),
adapter: {
command: process.execPath,
args: ["--input-type=module", "-e", script, "{baseUrl}", "{runId}"],
variant: "fake-test",
},
adapterName: "fake-agent",
configDirectory: process.cwd(),
iteration: 1,
locale: "en",
timeoutMs: 5_000,
});
assert.equal(result.execution.exitCode, 0);
assert.equal(result.verification.status, "passed-with-unverified");
assert.equal(result.eventCount, 6);
});
});
test("report summary keeps partial verification distinct from full verification", () => {
const rows = summarizeReports([
{
results: [
{
adapter: "agent-a",
variant: "granular-28",
execution: { exitCode: 0, durationMs: 100 },
metrics: { errorCount: 0, toolCallCount: 4 },
verification: { status: "passed-with-unverified" },
},
{
adapter: "agent-a",
variant: "granular-28",
execution: { exitCode: 0, durationMs: 300 },
metrics: { errorCount: 1, toolCallCount: 6 },
verification: { status: "passed" },
},
],
},
]);
assert.equal(rows[0].passRate, 1);
assert.equal(rows[0].fullyVerifiedRate, 0.5);
assert.equal(rows[0].averageDurationMs, 200);
assert.equal(rows[0].averageToolCalls, 5);
assert.equal(rows[0].errors, 1);
});
test("coverage inventory contains all 28 operations and three manual lanes", () => {
const coverage = buildOperationCoverage(cases);
const operations = coverage.map(({ operation }) => operation);
assert.equal(operations.length, 28);
assert.equal(new Set(operations).size, 28);
assert.deepEqual(
coverage.filter(({ directSmoke }) => !directSmoke).map(({ operation }) => operation),
["tabs.borrow", "tabs.return", "assist.request-help"],
);
assert.deepEqual(coverage.find(({ operation }) => operation === "interact.fill").smokeCases, [
"form-controls",
"generated-form",
]);
});
test("seeded dimensions are reproducible and vary across seeds", () => {
assert.deepEqual(describeSeed("badcase-17"), describeSeed("badcase-17"));
assert.notDeepEqual(describeSeed("badcase-17"), describeSeed("badcase-18"));
assert.equal(describeSeed(17).seed, 17);
});
test("badcase scaffold creates a loadable, escaped, self-contained case", async () => {
const temporaryRoot = await mkdtemp(join(tmpdir(), "browser-eval-scaffold-"));
try {
const title = "Quoted `title` ${notCode} <unsafe>";
const result = await scaffoldCase({
id: "reported-timeout",
title,
suite: "regression",
source: "issue-123",
rootCasesDirectory: temporaryRoot,
});
const loaded = await loadCaseManifests({ casesDirectory: temporaryRoot });
assert.equal(loaded[0].id, "reported-timeout");
assert.equal(loaded[0].source.reference, "issue-123");
const generatedRegistry = await loadFixtureRegistry({ roots: [temporaryRoot] });
const html = generatedRegistry.render("/cases/reported-timeout", {
runId: "scaffold-test",
query: new URLSearchParams(),
record() {},
});
assert.match(html, /Quoted `title` \$\{notCode\} &lt;unsafe&gt;/);
assert.match(await readFile(result.files.readme, "utf8"), /issue-123/);
} finally {
await rm(temporaryRoot, { recursive: true, force: true });
}
});
test("CLI accepts pnpm separator and supports suite and tag filters", async () => {
const cli = resolve("evals/browser/cli.mjs");
const coverageResult = await runProcess(process.execPath, [cli, "--", "coverage", "--json"], {
timeoutMs: 5_000,
});
assert.equal(coverageResult.exitCode, 0, coverageResult.stderr);
assert.equal(JSON.parse(coverageResult.stdout).length, 28);
const coreResult = await runProcess(
process.execPath,
[cli, "list", "--suite", "core", "--json"],
{
timeoutMs: 5_000,
},
);
assert.equal(coreResult.exitCode, 0, coreResult.stderr);
assert.equal(JSON.parse(coreResult.stdout).length, 6);
const formResult = await runProcess(process.execPath, [cli, "list", "--tag", "form", "--json"], {
timeoutMs: 5_000,
});
assert.equal(formResult.exitCode, 0, formResult.stderr);
assert.deepEqual(
JSON.parse(formResult.stdout).map(({ id }) => id),
["form-controls", "generated-form"],
);
});
test("CLI validation and seed generation are machine-readable", async () => {
const cli = resolve("evals/browser/cli.mjs");
const validation = await runProcess(process.execPath, [cli, "validate", "--json"], {
timeoutMs: 5_000,
});
assert.equal(validation.exitCode, 0, validation.stderr);
assert.equal(JSON.parse(validation.stdout).ok, true);
const generated = await runProcess(
process.execPath,
[cli, "generate", "--seed", "known-badcase", "--json"],
{ timeoutMs: 5_000 },
);
assert.equal(generated.exitCode, 0, generated.stderr);
assert.equal(typeof JSON.parse(generated.stdout).hydrationDelayMs, "number");
const generatedSet = await runProcess(
process.execPath,
[cli, "generate", "--seed", "4,7,14", "--json"],
{ timeoutMs: 5_000 },
);
assert.equal(generatedSet.exitCode, 0, generatedSet.stderr);
assert.deepEqual(
JSON.parse(generatedSet.stdout).map(({ seed }) => seed),
[4, 7, 14],
);
});
-3
View File
@@ -4,9 +4,6 @@
"packageManager": "pnpm@10.17.0",
"scripts": {
"cli:build": "cargo run -p bsk-protocol --bin dump-schema --locked && pnpm -r build",
"eval:browser": "node evals/browser/cli.mjs",
"eval:browser:check": "node evals/browser/cli.mjs validate && node --test evals/browser/tests/*.test.mjs",
"eval:browser:test": "node --test evals/browser/tests/*.test.mjs",
"ext:build": "pnpm --filter @browser-skill/extension build",
"ext:build:zip": "pnpm --filter @browser-skill/extension zip",
"ext:dev": "pnpm --filter @browser-skill/extension dev",
+33 -34
View File
@@ -5,44 +5,44 @@ npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wx
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
## Tools
| Tool | Actions | Purpose |
| Tool | bsk command | Purpose |
| --- | --- | --- |
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
DeepSeek Harness lifecycle before it can be added safely.
| `browser_session_start` | `bsk session start` | Open an Agent Window session; optional initial URL, window size, and device emulation preset. Returns the session id and makes it the current session. |
| `browser_session_stop` | `bsk session stop` | Stop a session (the current one by default) and close its Agent Window. Only plugin-created sessions can be stopped. |
| `browser_session_list` | — (registry only) | List the sessions this plugin created, marking the current one. Foreign daemon sessions are never shown. |
| `browser_navigate` | `bsk navigate` | Navigate the active tab, with `waitUntil` / timeout control. |
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
## Agent skill (progressive disclosure)
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
tool. The body is assembled at build time from two parts, so there is exactly one source of
truth: a dsh-specific prelude (`skill/prelude.md` — tool↔CLI map, owned-session semantics,
plugin-only overrides) followed verbatim by the canonical CLI skill (`skill/SKILL.md` at the
repo root — the same file `crates/bsk-cli/build.rs` mirrors for the CLI package; workflows,
stop-when-done rules, refs usage, sandbox rules). Registration and every pre-step catalog
snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
## Multi-session model
One agent conversation can drive several browser sessions at once:
- `browser_session` with `action: start` returns the session id and makes it the **current session**.
- `browser_session_start` returns the session id and makes it the **current session**.
- Every operation tool accepts an optional `session` argument. When omitted, the call acts on the
current session (the one most recently started or used); when given, that session becomes current.
- Every tool result echoes the session it actually acted on, so the model never has to guess.
@@ -51,9 +51,9 @@ One agent conversation can drive several browser sessions at once:
**Ownership boundary**: the bsk daemon may be shared with other agents, terminals, or dsh
instances. The plugin therefore only ever sees and operates on sessions it created itself —
an explicit `session` argument naming a foreign or unknown id is rejected, the `list` action on
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
can never touch a session owned by another program.
an explicit `session` argument naming a foreign or unknown id is rejected, `browser_session_list`
shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup can never touch
a session owned by another program.
## Installation
@@ -88,7 +88,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
# lazyTools: true # reveal browser_* tools only after the skill is invoked
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the eleven
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
@@ -110,7 +110,7 @@ like the BrowserSkill extension without leaking a single selector into the host
shell's theme cannot bleed back in.
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
`browser_session_start`; disappears when all sessions stop (or the plugin unloads).
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
stage while the next one loads, and is kept on errors so the card does not flash),
@@ -152,11 +152,11 @@ shell's theme cannot bleed back in.
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`lib/client.js`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
custom view keeps the terminal block (command + output) and, when the settled result carries an
image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
Every other `browser_*` tool keeps the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
@@ -177,15 +177,14 @@ pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
## Publishing
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id, client
bundle registration, and `cordis.patch.yml` name are the same specifier so
dsh can import and materialize the plugin without a ModuleLoader id mismatch.
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
`dsh-plugin-browserskill`.
Trigger it by pushing a tag that matches `package.json`'s `version`:
```sh
git tag dsh-plugin-v0.1.2
git push origin dsh-plugin-v0.1.2
git tag dsh-plugin-v0.1.0
git push origin dsh-plugin-v0.1.0
```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.1.2",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
"version": "0.1.1",
"license": "MIT",
"publishConfig": {
"access": "public",
@@ -50,10 +50,6 @@
"@deepseek-ai/dsh-client-runtime",
"@deepseek-ai/dsh-client-ui-tool",
"@deepseek-ai/dsh-client-ui-layout"
],
"external": [
"@deepseek-ai/dsh-client-ui-attachment",
"@deepseek-ai/dsh-client-ui-primitives"
]
}
},
@@ -111,8 +107,6 @@
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
"@deepseek-ai/dsh-scope": "^0.1.0-rc.6",
"@deepseek-ai/dsh-skill": "^0.1.0-rc.6",
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
"@deepseek-ai/schemastery": "^3.18.1",
"@tailwindcss/cli": "^4.3.3",
@@ -1,15 +1,20 @@
// Embed the DeepSeek Harness-specific browser skill at build time. The model-facing
// skill deliberately documents only injected browser_* tools; the repository's CLI
// skill is a separate interface and must never be concatenated here.
// Assemble the plugin's skill content: a dsh-specific routing prelude, the
// canonical CLI skill body verbatim (repo-root skill/SKILL.md — the same source
// crates/bsk-cli's build.rs mirrors), then a dsh routing reminder postlude. The
// original's YAML frontmatter is stripped: name/description are supplied by the
// registration.
// Output: src/skill-content.generated.ts (gitignored, built before tsdown and
// before vitest), so registration is a pure in-memory read.
// before vitest) so the markdown ships inlined in the bundle and registration
// is a pure in-memory read — never a per-step disk access.
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const here = dirname(fileURLToPath(import.meta.url));
const pkg = join(here, "..");
const source = readFileSync(join(pkg, "skill", "SKILL.md"), "utf8");
const prelude = readFileSync(join(pkg, "skill", "prelude.md"), "utf8");
const upstream = readFileSync(join(pkg, "..", "..", "skill", "SKILL.md"), "utf8");
const postlude = readFileSync(join(pkg, "skill", "postlude.md"), "utf8");
function stripFrontmatter(markdown) {
if (!markdown.startsWith("---\n")) return markdown;
@@ -17,47 +22,18 @@ function stripFrontmatter(markdown) {
return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\s*\n/, "");
}
const SUPPORTED_BROWSER_TOOLS = [
"browser_session",
"browser_page",
"browser_inspect",
"browser_interact",
"browser_tabs",
"browser_assist",
];
function validateModelFacingSkill(markdown, supportedTools) {
if (/\bbsk\b/i.test(markdown)) {
throw new Error("DSH skill must not expose the internal CLI name");
}
if (/```(?:bash|sh|shell)\b/i.test(markdown)) {
throw new Error("DSH skill must not contain command-line code blocks");
}
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
const supported = new Set(supportedTools);
const unknown = [...mentioned].filter((name) => !supported.has(name));
const missing = supportedTools.filter((name) => !mentioned.has(name));
if (unknown.length > 0) {
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
}
if (missing.length > 0) {
throw new Error(`DSH skill omits supported browser tools: ${missing.join(", ")}`);
}
}
const NAME = "browser-skill";
const DESCRIPTION =
"Browser automation against the user's logged-in Chromium through this plugin's injected " +
"browser_* tools. Use to visit and read pages, fill forms, click through flows, inspect tabs, " +
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
"Browser automation against the user's logged-in Chromium through this plugin's injected browser_* tools; never invoke bsk through bash or shell. " +
"tools (bsk engine): visit and read pages, fill forms, click through flows, scrape data, " +
"smoke-test UI changes. Use when a task needs a real browser; requires the bsk CLI and the " +
"browser-skill extension connected.";
const content = `${stripFrontmatter(source).trim()}\n`;
validateModelFacingSkill(content, SUPPORTED_BROWSER_TOOLS);
const content = `${prelude.trim()}\n\n${stripFrontmatter(upstream).trim()}\n\n${postlude.trim()}\n`;
const banner =
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
"// Source: skill/SKILL.md (DeepSeek Harness six-tool instructions).\n";
"// Sources: skill/prelude.md + ../../../skill/SKILL.md + skill/postlude.md.\n";
writeFileSync(
join(pkg, "src", "skill-content.generated.ts"),
@@ -1,103 +0,0 @@
---
name: browser-skill
description: Browser automation through six injected domain tools.
---
# browser-skill for DeepSeek Harness
Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is
isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed.
Loading this skill reveals six tools for the rest of the conversation:
- `browser_session` owns session lifecycle.
- `browser_page` handles navigation and lifecycle waits.
- `browser_inspect` reads semantic, visual, console, and network state.
- `browser_interact` performs normal page interactions.
- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing.
- `browser_assist` handles human help, window size, and device emulation.
Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and
parameters; do not guess fields. All browser work must use the injected tools directly so session
ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke
another process to control the browser.
## Mandatory workflow
Every task owns a bounded plugin session:
```text
browser_session({ action: "start", ... })
... use the returned sessionId for browser work ...
browser_session({ action: "stop", session: sessionId })
```
Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another
program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep
the session open. Stopping also returns borrowed tabs.
## Work toward one observable goal
- Derive a concrete success condition from the user's request.
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
ambiguous result.
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
- If a human-only step appears or two attempts make no progress, request help instead of
brute-forcing.
## Observe, act, observe
Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states,
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action.
Escalate reading only as needed:
1. `observe` for normal understanding and interaction refs.
2. `snapshot` when a stricter static accessibility tree is more useful.
3. `html` for exact markup or hidden metadata that semantic views cannot provide.
4. `screenshot` for layout, styling, canvas, images, or requested visual evidence.
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
needed, obtain a fresh observation before acting on screenshot or HTML findings.
Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative
waits when no navigation is expected. After any page change, discard old refs and observe again.
## Respect the Agent Window boundary
Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs.
Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never
invent a tab id or keep a personal tab borrowed across unrelated work.
## Ask the human when needed
Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or
another step the user must complete. Give a precise prompt and highlight fresh targets when concrete
controls are involved. Use completion criteria only for a clear stable success signal.
Resume only after the user continues or the criteria complete. Treat cancellation as rejection and
timeout as a blocker rather than retrying. Observe again after control returns before reasoning about
the new state or using refs.
The same tool can resize the Agent Window or emulate a device when the task requires visual or
responsive testing. Emulation is scoped to one tab.
## Debug and recover without wandering
Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics.
Continue from returned sequence cursors instead of rereading the same buffer.
- Stale ref: observe again and retry the intended action once.
- Unknown tab: list tabs instead of guessing.
- Unknown session: list owned sessions or start one; never try foreign ids.
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
- Unrecoverable failure: report the blocker and stop the owned session.
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
invent tools or route around those limits.
@@ -0,0 +1,10 @@
---
## DSH routing reminder
The CLI-oriented reference above describes behavior, not the execution mechanism in
DeepSeek Harness. For every browser operation, call the injected `browser_*` model
tool. Do not translate these examples into `bash` commands and do not execute `bsk`
directly; the plugin invokes `bsk` internally. In particular, always begin with
`browser_session_start`, pass its returned session id to subsequent `browser_*`
tools, and finish with `browser_session_stop`.
@@ -0,0 +1,71 @@
# browser-skill (dsh edition)
## DSH tool-routing requirement (highest priority)
This skill is running inside DeepSeek Harness, where the plugin has injected
structured `browser_*` model tools. **Always call those injected `browser_*` tools
for browser work. Never use `bash`, `shell`, `exec`, or another tool to invoke the
`bsk` CLI directly.** The CLI command names shown below are only conceptual
mappings for understanding the tools; they are not instructions to run commands.
If a `browser_*` tool is available, using the CLI directly is incorrect because it
bypasses plugin session ownership, observation UI, tool cards, and cleanup.
Use the injected tool whose name matches the operation, for example:
`browser_session_start`, `browser_navigate`, `browser_observe`, `browser_snapshot`,
`browser_click`, `browser_fill`, `browser_press`, `browser_screenshot`, and
`browser_session_stop`. Pass arguments as the tool schema specifies (not CLI flags).
Drive the user's **real Chromium browser** (logins, cookies, Agent Window isolation)
through this plugin's `browser_*` tools. Each injected tool is implemented by the
plugin using the `bsk` engine internally — the model must not invoke that engine
itself. The workflow, judgment rules, and safety constraints remain identical to
the canonical CLI skill.
> **Tool availability**: invoking this skill (which you just did) makes the plugin's
> `browser_*` tools available for the rest of the session — they are injected on
> successful skill load. Until then they are deliberately hidden, so do not attempt
> a browser action before loading this skill.
## Tool ↔ CLI map
The tools below are thin, structured wrappers over the bsk CLI verbs of the same
name; `session` parameters take the 4-letter session id (no `--session` flag
spelling here):
| dsh tool | bsk equivalent |
|---|---|
| `browser_session_start` / `browser_session_stop` / `browser_session_list` | `bsk session start/stop/list` |
| `browser_navigate` | `bsk navigate` |
| `browser_snapshot` / `browser_observe` | `bsk snapshot` / `bsk observe` |
| `browser_click` / `browser_fill` / `browser_press` | `bsk click` / `bsk fill` / `bsk press` |
| `browser_screenshot` | `bsk screenshot` |
| `browser_emulate` | `bsk emulate` |
(`bsk console` / `bsk network` have no tool counterpart in this plugin yet.)
Plugin-specific semantics that override or narrow the CLI rules below:
- **Owned sessions only**: this plugin sees and acts on just the sessions IT
started via `browser_session_start`. `browser_session_list` shows only those
(with the current marker); passing a session id created by another program
(or person) sharing the bsk daemon is refused outright — never retry those
with the same id.
- **Current session**: omitted `session` parameters resolve to the most
recently started or used OWNED session. With several sessions in flight,
pass `session` explicitly to avoid surprises.
- **Screenshots**: when the model route accepts images, `browser_screenshot`
returns the PNG inline as an image attachment; otherwise it returns the
saved file path.
- **`browser_emulate mobile`**: requires `width`+`height` — the bsk daemon
refuses `--mobile` without viewport dimensions.
- **Live observation**: in the dsh Web UI, every owned session is watched by
the floating overlay (thumbnails, action, interrupt) — observational side
traffic is free and never steals your command turn; just keep acting.
The canonical CLI skill body follows verbatim (source of truth: `skill/SKILL.md`
in the BrowserSkill repo, mirrored by build into `crates/bsk-cli/skill/SKILL.md`).
Read `bsk <verb> --session <id>` as "the `browser_<verb>` tool with
`session: <id>`"; everything else — when to use, the stop-when-done rules, the
interaction loop, refs, sandbox rules — applies unchanged.
---
@@ -1,108 +0,0 @@
/**
* Archive-triggered bsk session cleanup. DSH conversation archival writes
* the workspace registry's durable global state, which the storage domain
* broadcasts as `domain/changed` ({domain: 'workspace', table: ''}) with
* the new `archivedSessionIds`. The watcher diffs that set and stops every
* bsk session owned by a freshly archived conversation — archived sessions
* are hidden from every surface, so their Agent Windows would otherwise
* linger with no way back to them.
*
* Ownership lineage: browser_session action=start records the calling agent's
* session id PLUS every ancestor along `header.parentSession` (a subagent's
* browsers are reaped when any ancestor is archived, the root conversation
* included). Lineage is resolved through the host session store; an
* unloaded ancestor simply ends the walk.
*/
import type { Context } from "@deepseek-ai/cordis";
import type { ObservationService } from "./observation";
import type { SessionRegistry } from "./sessions";
/** The wire shape of a `domain/changed` frame (see dsh-storage-domain). */
interface DomainChange {
domain?: string;
table?: string;
value?: unknown;
}
/** The workspace registry's durable global singleton slice we read. */
interface WorkspaceGlobal {
archivedSessionIds?: unknown;
}
/** Structural host session store face: only the lineage read is needed. */
interface SessionStoreLike {
get(id: string): { header: { parentSession?: string } } | undefined;
}
/** Cap on lineage walks — defensive against a malformed parent chain. */
const MAX_LINEAGE_DEPTH = 16;
/**
* The DSH session ids that own a tool call's browser sessions: the calling
* agent's own session plus every ancestor along the seed lineage. Empty
* when the call carried no agent identity (those sessions outlive any
* archive cleanup by design — nothing can name their owner).
*/
export function ownerSessionIds(ctx: Context, agentId: string | undefined): string[] {
if (agentId === undefined) return [];
const store = ctx.get("sessions") as SessionStoreLike | undefined;
const ids: string[] = [];
let current: string | undefined = agentId;
for (let depth = 0; current !== undefined && depth < MAX_LINEAGE_DEPTH; depth += 1) {
if (ids.includes(current)) break; // a cycle in stored headers must not loop
ids.push(current);
current = store?.get(current)?.header.parentSession;
}
return ids;
}
/**
* Watch conversation archival and stop the bsk sessions it opened. Returns
* the disposer (plugin unload). In compositions without the workspace
* domain (headless), the event simply never fires — and a context without
* the events mixin degrades to a no-op like the other optional seams.
*/
export function armArchiveCleanup(
ctx: Context,
registry: SessionRegistry,
observation: ObservationService,
): () => void {
// 'domain/changed' lives outside the vendored Events type map, so the
// listener goes through a structural view of the events mixin.
const on = (
ctx as { on?: (event: string, listener: (change: DomainChange) => void) => () => void }
).on;
if (typeof on !== "function") return () => {};
/** Archived ids already accounted for; lazily seeded from the registry. */
let seen: Set<string> | undefined;
const initialize = (): Set<string> => {
if (seen === undefined) {
const registryService = ctx.get("workspaceRegistry") as
| { archivedSessionIds?: readonly string[] }
| undefined;
seen = new Set(registryService?.archivedSessionIds ?? []);
}
return seen;
};
return on.call(ctx, "domain/changed", (change: DomainChange) => {
if (change?.domain !== "workspace" || change?.table !== "") return;
const archived = (change.value as WorkspaceGlobal | undefined)?.archivedSessionIds;
if (!Array.isArray(archived)) return;
const previous = initialize();
const fresh = archived.filter(
(id): id is string => typeof id === "string" && !previous.has(id),
);
seen = new Set(archived.filter((id): id is string => typeof id === "string"));
for (const dshSessionId of fresh) {
for (const sessionId of registry.ownedByDsh(dshSessionId)) {
// stopSession owns the full teardown (kill in-flight tools, queue
// the daemon stop, drop registry + observation entries); a failure
// just leaves the session for idle timeout or unload cleanup.
void observation.stopSession(sessionId).catch(() => {});
}
}
});
}
@@ -1,334 +0,0 @@
/**
* The six model-facing BrowserSkill tools. Each action dispatches to a private
* operation handler, so the model receives only six schemas while every action
* retains the existing ownership, cancellation, queuing, observation,
* screenshot, and presentation behavior.
*/
import { defineTool, type ParameterSchemaSpec, type ToolDefinition } from "@deepseek-ai/dsh-tools";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
import { createBrowserOperationDefinitions, type ToolDeps } from "./tools";
const DEVICE_PRESETS = [
"iphone-14",
"iphone-14-pro-max",
"iphone-se",
"pixel-7",
"galaxy-s23",
"ipad-mini",
"galaxy-tab-s8",
] as const;
const TARGET_PARAM = {
type: "string",
description: "Snapshot ref such as @e3, or a CSS selector.",
} as const;
const DOMAIN_RESULT = { type: "json" } as const;
interface BrowserToolSpec {
name: string;
description: string;
actions: Record<string, string>;
parameters: ParameterSchemaSpec;
}
function operationArgs(args: unknown): Record<string, unknown> {
const { action: _action, ...rest } = args as Record<string, unknown>;
return rest;
}
function definitionFor(
definitions: ReadonlyMap<string, ToolDefinition>,
actions: Record<string, string>,
action: unknown,
): ToolDefinition {
const operationName = typeof action === "string" ? actions[action] : undefined;
const definition = operationName === undefined ? undefined : definitions.get(operationName);
if (definition === undefined) throw new Error(`unsupported browser action: ${String(action)}`);
return definition;
}
function defineBrowserTool(
spec: BrowserToolSpec,
definitions: ReadonlyMap<string, ToolDefinition>,
): ToolDefinition {
const actionNames = Object.keys(spec.actions);
return defineTool({
name: spec.name,
description: spec.description,
parameters: {
action: {
type: "string",
required: true,
enum: actionNames,
description:
"Operation to perform. Other arguments are action-dependent and are validated by that operation.",
},
...spec.parameters,
},
output: {
schema: DOMAIN_RESULT,
render(args, value) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.output.render(operationArgs(args), value);
},
},
isConcurrencySafe(args) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.isConcurrencySafe?.(operationArgs(args)) === true;
},
async execute(args, exec) {
const definition = definitionFor(definitions, spec.actions, args.action);
return (await definition.execute(operationArgs(args), exec)) as never;
},
presentCall(args) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.presentCall?.(operationArgs(args));
},
presentResult(args, result) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.presentResult?.(operationArgs(args), result);
},
});
}
const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{
name: "browser_session",
description:
"Manage plugin-owned browser sessions. Actions: start opens an Agent Window; stop closes an " +
"owned session; list returns owned sessions. For start, url/device/width/height/noFocus/browser " +
"are optional. For stop, session is optional and defaults to the current owned session.",
actions: {
start: "session.start",
stop: "session.stop",
list: "session.list",
},
parameters: {
session: SESSION_PARAM,
url: { type: "string", description: "Initial URL for start." },
width: { type: "integer", description: "Agent Window width; start requires height too." },
height: { type: "integer", description: "Agent Window height; start requires width too." },
noFocus: { type: "boolean", description: "Start the Agent Window in the background." },
browser: { type: "string", description: "Browser instance id for start." },
device: { type: "string", enum: DEVICE_PRESETS, description: "Device preset for start." },
},
},
{
name: "browser_page",
description:
"Navigate and wait on the active Agent Window tab. Actions: navigate, back, forward, reload, " +
"wait. navigate requires url; reload optionally accepts hard; navigation actions accept " +
"waitUntil/timeoutMs. Observe again after a meaningful page change before reusing refs.",
actions: {
navigate: "page.navigate",
back: "page.back",
forward: "page.forward",
reload: "page.reload",
wait: "page.wait",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
url: { type: "string", description: "Destination URL; required for navigate." },
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
hard: { type: "boolean", description: "Bypass cache for reload." },
},
},
{
name: "browser_inspect",
description:
"Read page state without arbitrary script execution. Actions: observe, snapshot, html, " +
"screenshot, console, network. Prefer observe, then snapshot, then bounded html; use screenshot " +
"for visual evidence. console/network support cursor fields since/limit/maxTextChars.",
actions: {
observe: "inspect.observe",
snapshot: "inspect.snapshot",
html: "inspect.html",
screenshot: "inspect.screenshot",
console: "inspect.console",
network: "inspect.network",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
maxDepth: { type: "integer", description: "Tree depth cap for observe/snapshot." },
maxTokens: { type: "integer", description: "Token cap for observe/snapshot." },
ref: { type: "string", description: "Fresh ref for scoped html or cropped screenshot." },
maxBytes: { type: "integer", description: "HTML byte cap." },
since: { type: "integer", description: "Console/network sequence cursor." },
limit: { type: "integer", description: "Console/network entry cap." },
maxTextChars: { type: "integer", description: "Console/network per-entry text cap." },
includeStack: { type: "boolean", description: "Include console stack frames." },
},
},
{
name: "browser_interact",
description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.",
actions: {
click: "interact.click",
hover: "interact.hover",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
target: TARGET_PARAM,
button: { type: "string", enum: ["left", "middle", "right"], description: "Click button." },
clickCount: { type: "integer", description: "Click count." },
value: { type: "string", description: "Text for fill." },
noClear: { type: "boolean", description: "Append instead of clearing for fill." },
modifiers: {
type: "array",
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
description: "Modifiers held during hover.",
},
settleMs: { type: "integer", description: "Hover settle delay." },
timeoutMs: TIMEOUT_MS_PARAM,
values: {
type: "array",
items: { type: "string" },
description: "Option values for select; at least one is required.",
},
key: { type: "string", description: "Key or combo for press." },
holdMs: { type: "integer", description: "Key hold duration for press." },
},
},
{
name: "browser_tabs",
description:
"Manage tabs visible to an owned session. Actions: list, create, select, close, borrow, return. " +
"select/close/borrow/return require tabId from list/create. Borrow moves a user tab into the " +
"Agent Window; return it as soon as the task finishes.",
actions: {
list: "tabs.list",
create: "tabs.create",
select: "tabs.select",
close: "tabs.close",
borrow: "tabs.borrow",
return: "tabs.return",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
scope: {
type: "string",
enum: ["user", "agent", "all"],
description: "Tab scope for list.",
},
url: { type: "string", description: "Initial URL for create." },
active: { type: "boolean", description: "Focus the created tab (default true)." },
index: { type: "integer", description: "Insertion index for create." },
},
},
{
name: "browser_assist",
description:
"Display and human-assistance operations. Actions: resize, emulate, request-help. resize " +
"requires width/height; emulate accepts device or width/height/mobile, or off alone; " +
"request-help requires prompt and can wait for explicit completion criteria.",
actions: {
resize: "assist.resize",
emulate: "assist.emulate",
"request-help": "assist.request-help",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
width: { type: "integer", description: "Window/viewport width." },
height: { type: "integer", description: "Window/viewport height." },
device: { type: "string", enum: DEVICE_PRESETS, description: "Emulation device preset." },
mobile: { type: "boolean", description: "Enable a mobile viewport with width/height." },
off: { type: "boolean", description: "Clear emulation; use alone." },
prompt: { type: "string", description: "Instructions shown to the user for request-help." },
title: { type: "string", description: "Optional request-help title." },
targets: {
type: "array",
items: { type: "string" },
description: "Refs/selectors highlighted for request-help.",
},
timeoutMs: TIMEOUT_MS_PARAM,
completionCriteria: {
type: "object",
additionalProperties: false,
description: "Automatic completion detector for request-help.",
properties: {
any: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
},
},
all: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
},
},
stableForMs: { type: "integer" },
},
},
},
},
];
function indexOperations(definitions: ToolDefinition[]): ReadonlyMap<string, ToolDefinition> {
const indexed = new Map<string, ToolDefinition>();
for (const definition of definitions) {
if (indexed.has(definition.name)) {
throw new Error(`duplicate browser operation definition: ${definition.name}`);
}
indexed.set(definition.name, definition);
}
const routed = new Set(BROWSER_TOOL_SPECS.flatMap((spec) => Object.values(spec.actions)));
const missing = [...routed].filter((name) => !indexed.has(name));
const unreachable = [...indexed.keys()].filter((name) => !routed.has(name));
if (missing.length > 0 || unreachable.length > 0) {
throw new Error(
[
missing.length > 0 ? `missing operations: ${missing.join(", ")}` : undefined,
unreachable.length > 0 ? `unreachable operations: ${unreachable.join(", ")}` : undefined,
]
.filter(Boolean)
.join("; "),
);
}
return indexed;
}
/** Register the complete six-tool browser suite; returns its combined disposer. */
export function registerBrowserTools(deps: ToolDeps): () => void {
const definitions = indexOperations(createBrowserOperationDefinitions(deps));
const disposers = BROWSER_TOOL_SPECS.map((spec) =>
deps.ctx.tools.register(defineBrowserTool(spec, definitions)),
).filter((dispose): dispose is () => void => typeof dispose === "function");
return () => {
for (const dispose of disposers.splice(0)) dispose();
};
}
@@ -30,51 +30,17 @@
color: var(--card-foreground);
}
/* Sidebar carrier: fill the better-sidebar tab area and let the body flex
(the tab content container owns the outer sizing). */
.sidebar-tab {
display: flex;
flex-direction: column;
height: 100%;
min-height: 0;
}
.sidebar-tab > * {
flex: 1;
min-height: 0;
}
/* The BSK product mark in the sidebar tab strip (the source artwork already
carries rounded corners — just soften the square to match). */
.brand-icon {
display: block;
border-radius: 22%;
}
.header {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 8px;
cursor: move;
user-select: none;
touch-action: none;
border-bottom: 1px solid var(--border);
}
/* Only the floating card drags by its header — the sidebar tab does not. */
.header[data-draggable] {
cursor: move;
}
/* In the sidebar the status row is chrome, not a card edge: drop its bottom
border. A fixed-height bordered row can never track the shell header's
hairline across the panel boundary (its height varies per window), and
the BSK warm border tint clashes with the sidebar's neutral hairlines —
borderless, the stage's own background provides the separation. */
.sidebar-tab .header {
border-bottom: none;
}
.status-text {
flex: 1;
min-width: 0;
@@ -165,12 +131,6 @@
border-top: 1px solid var(--border);
}
.actions-group {
display: inline-flex;
align-items: center;
gap: 4px;
}
.tool-wrap {
position: relative;
display: inline-flex;
@@ -205,14 +165,8 @@
color: var(--destructive);
}
/* Stop session: danger-tinted like interrupt. */
.tool-stop {
color: var(--destructive);
}
/* Hover the wrap (not just :hover on the button) so the tooltip target
lights the icon up too. The corner resize handles sit above this bar, so
the outer sliver of the corner-most buttons starts a resize instead. */
/* Hover the wrap (not just :hover on the button) so the tooltip target and
the left-corner resize overlay still light the icon up. */
.tool-wrap:hover .tool-button:not(:disabled) {
color: var(--foreground);
background: var(--accent);
@@ -227,15 +181,6 @@
background: var(--accent);
}
/* The armed (confirm) state of the stop button: solid destructive fill so
the second click reads as final. Written at the wrap-hover specificity so
the hover rules above never wash it out. */
.tool-button.tool-stop-armed,
.tool-wrap:hover .tool-button.tool-stop-armed {
color: var(--destructive-foreground);
background: var(--destructive);
}
.hint {
position: absolute;
bottom: calc(100% + 4px);
@@ -258,16 +203,13 @@
right: 0;
}
/* Invisible hit targets — no grip glyph; all four corners resize. Must sit
above the bottom .actions bar (z-index 3): it spans the full card width
and would otherwise swallow every pointer event aimed at the sw/se
corners. The top corners already win because .header is not positioned. */
/* Invisible hit targets — no grip glyph; all four corners resize. */
.resize-handle {
position: absolute;
width: 16px;
height: 16px;
touch-action: none;
z-index: 4;
z-index: 2;
}
.resize-handle[data-corner="nw"] {
@@ -3,18 +3,13 @@
// composer's way); collapsed = a status capsule; "pop out" upgrades the same
// content to a native Document PiP window (user gesture required by the
// browser). Multi-session renders a meeting-style strip under the focus view.
// When the dsh-better-sidebar plugin is installed the tracking view moves
// into a sidebar tab instead (see observation-sidebar.tsx) and this floating
// card hides itself via the sidebar-mode flag. Visuals follow the
// BrowserSkill product family: @browser-skill/ui components and oklch tokens
// on a .bsk-obs scope root, so the card reads as BSK's own surface without
// leaking styles into (or inheriting themes from) the shell.
// Visuals follow the BrowserSkill product family: @browser-skill/ui components
// and oklch tokens on a .bsk-obs scope root, so the card reads as BSK's own
// surface without leaking styles into (or inheriting themes from) the shell.
import { cn } from "@browser-skill/ui";
import {
RiArrowDownSLine,
RiCheckLine,
RiCloseCircleLine,
RiErrorWarningLine,
RiPictureInPicture2Line,
RiPushpinFill,
@@ -30,8 +25,6 @@ const IconPip = asIcon(RiPictureInPicture2Line);
const IconDown = asIcon(RiArrowDownSLine);
const IconWarn = asIcon(RiErrorWarningLine);
const IconPin = asIcon(RiPushpinFill);
const IconCloseSession = asIcon(RiCloseCircleLine);
const IconCheck = asIcon(RiCheckLine);
import {
type ReactNode,
@@ -46,12 +39,11 @@ import { createPortal } from "react-dom";
import type { SessionObservation } from "../observation";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { focusOf, statusOf, useObservationView, usePip } from "./observation-view";
import { getSidebarMode, subscribeSidebarMode } from "./sidebar-mode";
// The pure view helpers live in observation-view (shared with the sidebar
// tab); re-export so existing imports of this module keep working.
export { focusOf, statusOf };
/** Minimal Document PiP surface (TS lib.dom lacks it). */
interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
interface Point {
x: number;
@@ -69,6 +61,11 @@ const EDGE_MARGIN = 16;
/** Default dock: top-right, just under the shell's top bar (no spacing tokens exist in dsh 0.1). */
const TOP_OFFSET = 64;
function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
function clampSize(size: Size, viewport: Size): Size {
const maxW = viewport.w * 0.8;
const maxH = viewport.h * 0.8;
@@ -117,6 +114,23 @@ function formatElapsed(sinceMs: number, nowMs: number): string {
return `${mm}:${ss}`;
}
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Compact toolbar icon: no label, hover bubble for the name / semantics. */
function IconAction(props: {
label: string;
@@ -152,83 +166,6 @@ function IconAction(props: {
);
}
/** How long the stop button stays armed before the confirm click expires. */
const STOP_ARM_MS = 3000;
/**
* Stop-session button with a lightweight two-click confirm: the first click
* arms the button (solid red, check icon, short expiry), the second executes
* the stop. No dialog, no layout shift — and a stray single click can never
* close an Agent Window.
*/
function StopSessionAction(props: {
sessionId: string | undefined;
onStop: (sessionId: string) => Promise<boolean>;
}) {
const { sessionId, onStop } = props;
const [hover, setHover] = useState(false);
const [armed, setArmed] = useState(false);
const [stopping, setStopping] = useState(false);
useEffect(() => {
if (!armed) return;
const timer = setTimeout(() => setArmed(false), STOP_ARM_MS);
return () => clearTimeout(timer);
}, [armed]);
// The focused session can vanish mid-arm (stopped elsewhere): disarm.
useEffect(() => {
if (sessionId === undefined) setArmed(false);
}, [sessionId]);
const disabled = sessionId === undefined || stopping;
const label = stopping
? "Stopping session…"
: armed
? `Confirm stop session ${sessionId ?? ""}`
: `Stop session ${sessionId ?? ""}`;
const hint = stopping
? "Closing the Agent Window…"
: armed
? `Click again to stop session ${sessionId ?? ""} and close its Agent Window.`
: `Stop session ${sessionId ?? ""} and close its Agent Window.`;
return (
<span
className={css["tool-wrap"]}
onPointerEnter={() => setHover(true)}
onPointerLeave={() => setHover(false)}
>
<button
type="button"
className={cn(css["tool-button"], css["tool-stop"], armed && css["tool-stop-armed"])}
disabled={disabled}
aria-label={label}
aria-pressed={armed}
data-armed={armed || undefined}
onClick={() => {
if (sessionId === undefined || stopping) return;
if (!armed) {
setArmed(true);
return;
}
setArmed(false);
setStopping(true);
const target = sessionId;
void onStop(target).finally(() => setStopping(false));
}}
>
{armed ? <IconCheck size={16} /> : <IconCloseSession size={16} />}
</button>
{hover ? (
<span className={css.hint} role="tooltip">
{hint}
</span>
) : null}
</span>
);
}
/** Flat status dot, specced after the BSK popup's ConnectionStatusIndicator. */
function StatusDot({ state }: { state: "active" | "idle" | "error" | "dead" }) {
const color =
@@ -309,8 +246,8 @@ function StripItem(props: {
);
}
/** The floating card / PiP / sidebar-tab shared content. */
export function OverlayBody(props: {
/** The floating card / PiP shared content. */
function OverlayBody(props: {
store: ObservationClientStore;
focus: SessionObservation | undefined;
sessions: readonly SessionObservation[];
@@ -373,7 +310,6 @@ export function OverlayBody(props: {
<div
className={css.header}
data-testid="obs-header"
data-draggable={onHeaderPointerDown !== undefined || undefined}
onPointerDown={onHeaderPointerDown}
role="presentation"
>
@@ -427,21 +363,15 @@ export function OverlayBody(props: {
</div>
) : null}
<div className={css.actions}>
<div className={css["actions-group"]}>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
<StopSessionAction
sessionId={focus?.sessionId}
onStop={(sessionId) => store.stopSession(sessionId)}
/>
</div>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
{onPopOut !== undefined ? (
<IconAction
label="Pop out into a mini window"
@@ -457,17 +387,25 @@ export function OverlayBody(props: {
);
}
/** Clone the host document's style/link nodes into a PiP window. */
function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store);
// While the better-sidebar plugin carries the tracking view, this floating
// card (and its capsule) stays out of the way; an already-open PiP window
// keeps its portal until the user closes it.
const sidebarMode = useSyncExternalStore(subscribeSidebarMode, getSidebarMode);
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.start();
return () => store.stop();
}, [store]);
const [collapsed, setCollapsed] = useState(false);
const [pos, setPos] = useState<Point | null>(null);
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
const { pipWindow, pipSupported, popOut } = usePip();
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const [pinnedId, setPinnedId] = useState<string | null>(null);
const dragRef = useRef<{
kind: "move" | "resize";
corner?: ResizeCorner;
@@ -476,6 +414,23 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
base: Point & Size;
} | null>(null);
// Elapsed-time ticker: 1s while anything is active.
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
// Focus: the pinned session wins while it still exists; otherwise auto-follow.
const pinned =
pinnedId !== null ? snapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(snapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
const onPointerMove = useCallback((event: PointerEvent) => {
@@ -556,21 +511,34 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
document.addEventListener("pointerup", onPointerUp);
};
const popOut = async (): Promise<void> => {
const pip = pipApi();
if (pip === undefined) return;
try {
const win = await pip.requestWindow({ width: size.w, height: size.h });
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
} catch {
// requestWindow rejects without a user gesture (or when the window was
// closed mid-request): stay on the in-page card, no state change.
}
};
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinnedId}
pinnedId={pinned !== undefined ? pinnedId : null}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={
pipWindow === null && pipSupported
? () => popOut({ width: size.w, height: size.h })
: undefined
}
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
/>
@@ -580,12 +548,6 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
return createPortal(body, pipWindow.document.body);
}
// The sidebar tab is the carrier now — no floating card, no capsule.
if (sidebarMode) return null;
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0) return null;
if (collapsed) {
const state = focus !== undefined ? statusOf(focus) : "idle";
return (
@@ -1,4 +1,4 @@
/* browser_inspect screenshot toolview: disclosure row over a terminal block plus the
/* browser_screenshot toolview: disclosure row over a terminal block plus the
screenshot image. Semantic tokens only (--dsw-alias-*), no literal colors. */
.card {
@@ -1,6 +1,8 @@
// browser_inspect keyed toolview: all inspect actions retain a terminal card;
// screenshot results additionally render the image attachment. Pure function
// of the frozen call/result slice, so live and replay paths stay identical.
// browser_screenshot keyed toolview: the terminal card's command line and
// output stay exactly as the host renders them, and a result carrying an
// image block additionally renders the screenshot itself (thumbnail with the
// shared click-to-open lightbox). Pure function of the frozen call/result
// slice — live and replay paths render identically.
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
import {
@@ -17,13 +19,13 @@ import {
} from "@deepseek-ai/dsh-client-ui-primitives";
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
import { useState } from "react";
import css from "./BrowserInspectToolView.module.css";
import css from "./ScreenshotToolView.module.css";
/** Resolved loader bound to the owning session at registration time. */
export type BrowserInspectImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
export type ScreenshotImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
export type BrowserInspectToolViewProps = ToolCallViewProps & {
loadImage: BrowserInspectImageLoader;
export type ScreenshotToolViewProps = ToolCallViewProps & {
loadImage: ScreenshotImageLoader;
};
type ViewState = "running" | "ok" | "error";
@@ -34,7 +36,6 @@ interface ViewModel {
readonly output: string | null;
readonly image: ImageAttachmentRef | null;
readonly summary: string;
readonly title: string;
}
const TERMINAL_LABELS: Partial<TerminalBlockLabels> = {
@@ -65,71 +66,18 @@ function firstLine(text: string): string {
}
/** Rebuild the command line from the logged arguments (mirrors the host presenter). */
const INSPECT_COMMANDS: Record<string, string> = {
observe: "observe",
snapshot: "snapshot",
html: "get-html",
screenshot: "screenshot",
console: "console",
network: "network",
};
function titleOf(action: string): string {
return action === "html" ? "HTML" : `${action.slice(0, 1).toUpperCase()}${action.slice(1)}`;
}
interface InspectArgs {
action?: unknown;
session?: unknown;
tabId?: unknown;
maxDepth?: unknown;
maxTokens?: unknown;
ref?: unknown;
maxBytes?: unknown;
since?: unknown;
limit?: unknown;
maxTextChars?: unknown;
includeStack?: unknown;
}
function appendNumber(parts: string[], flag: string, value: unknown): void {
if (typeof value === "number" && Number.isFinite(value)) parts.push(flag, String(value));
}
function commandOf(argsRaw: string, callId: string): { command: string; title: string } {
function commandOf(argsRaw: string, callId: string): string {
try {
const args = JSON.parse(argsRaw) as InspectArgs;
const action = typeof args.action === "string" ? args.action : "inspect";
const command = INSPECT_COMMANDS[action] ?? action;
const parts = ["bsk", command, "--session"];
const args = JSON.parse(argsRaw) as { session?: unknown; ref?: unknown };
const parts = ["bsk", "screenshot", "--session"];
parts.push(
typeof args.session === "string" && args.session !== "" ? args.session : "(current)",
);
if (action === "observe" || action === "snapshot") {
appendNumber(parts, "--max-depth", args.maxDepth);
appendNumber(parts, "--max-tokens", args.maxTokens);
}
if (action === "html" || action === "console" || action === "network") {
appendNumber(parts, "--tab-id", args.tabId);
}
if ((action === "html" || action === "screenshot") && typeof args.ref === "string") {
if (args.ref !== "") parts.push("--ref", args.ref);
}
if (action === "html") appendNumber(parts, "--max-bytes", args.maxBytes);
if (action === "console" || action === "network") {
appendNumber(parts, "--since", args.since);
appendNumber(parts, "--limit", args.limit);
appendNumber(parts, "--max-text-chars", args.maxTextChars);
}
if (action === "console" && args.includeStack === true) parts.push("--include-stack");
return { command: parts.join(" "), title: titleOf(action) };
if (typeof args.ref === "string" && args.ref !== "") parts.push("--ref", args.ref);
return parts.join(" ");
} catch {
// Streaming can expose a truncated JSON prefix; it is still the best label.
return {
command:
argsRaw === "" ? `browser_inspect (${callId})` : `browser_inspect ${firstLine(argsRaw)}`,
title: "Inspect",
};
return argsRaw === "" ? `bsk screenshot (${callId})` : `bsk screenshot ${firstLine(argsRaw)}`;
}
}
@@ -137,9 +85,9 @@ function commandOf(argsRaw: string, callId: string): { command: string; title: s
export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
const settled = "kind" in block;
const argsRaw = (settled ? block.call?.argsRaw : block.argsRaw) ?? "";
const { command, title } = commandOf(argsRaw, block.callId);
const command = commandOf(argsRaw, block.callId);
if (!settled) {
return { state: "running", command, output: null, image: null, summary: command, title };
return { state: "running", command, output: null, image: null, summary: command };
}
const textBlock = block.content.find((item) => item.type === "text");
const imageBlock = block.content.find((item) => item.type === "image");
@@ -148,7 +96,7 @@ export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
imageBlock !== undefined && imageBlock.type === "image" ? imageBlock.attachment : null;
const state: ViewState = block.isError ? "error" : "ok";
const summary = output !== null ? firstLine(output) : command;
return { state, command, output, image, summary, title };
return { state, command, output, image, summary };
}
function dotState(state: ViewState): StateDotState {
@@ -163,24 +111,19 @@ function dotState(state: ViewState): StateDotState {
}
/**
* Render one browser_inspect call: a disclosure row over a terminal block,
* plus the screenshot image when that action returns an image attachment.
* Render one browser_screenshot call: a disclosure row over a terminal block
* (command + output) plus the screenshot image when the result carries one.
*/
export function BrowserInspectToolView({
block,
cwd,
inspect,
loadImage,
}: BrowserInspectToolViewProps) {
export function ScreenshotToolView({ block, cwd, inspect, loadImage }: ScreenshotToolViewProps) {
const model = viewModelOf(block);
const [expanded, setExpanded] = useState(false);
const expandable = model.state === "running" || model.output !== null || model.image !== null;
const open = expanded && expandable;
return (
<div className={css.card} data-tool="browser_inspect" data-state={model.state}>
<div className={css.card} data-tool="browser_screenshot" data-state={model.state}>
<DisclosureRow
icon={<StateDot state={dotState(model.state)} />}
title={model.title}
title="Screenshot"
open={open}
expandable={expandable}
onToggle={() => setExpanded((value) => !value)}
@@ -1,8 +0,0 @@
/**
* The BrowserSkill product mark (apps/extension/assets/logo.png, downscaled
* to 32px and inlined): the sidebar tab icon, so the tracking view reads as
* BSK's own surface next to better-sidebar's built-in "browser" tab.
* Regenerate with: resize the source to 32x32 PNG and replace the payload.
*/
export const BSK_LOGO_URL =
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAJGklEQVR42m2X248c2V3HP79zTlV19/TcPOOZWY+vsdmQFdngTQJswgs3hQ15iCJslAey+8xDAIHgH+CBNxIpj0gkEXnAKyKQWITEJlKWXSssEMfJRmuMHa/W9njHMz3Tl+nprsv5/XiomlugpdN16lR1n9/1e75fAbAbeLlO3Pz2ly4vjO7+sRbF58oiP49FEREOPwaIYQjS3IvUy0fP66kAZoYJIM7SpPVY0uy14cyzXzn9u9+4c+MG/vp1otiNa16uvxr7X/+1L3bK3lcTyU9P80ilR/uKCGYHOxxd6s2kmdaGIVYb1qyY1e8HD63UUblstwirf9J95fW/sRvXvAD0v/6Z6/O68Xf5/pS8kigiTsTkuCfIwfzAxQN37chjaAyt594JIXE4gbKMhog6Nd/qthi7lVe6L3/nG7Jz4/fPJ9u3b4U4WSwiKog/EcsTLh+PSuO3GXYs/D44QnBgyniQs7OV0+8bOxPPM6c9l660VEyF0Nmrlp9/IYT+/S/PUJ4a5C46EW8HnjVhBKjXaHJfz9Xq95xzhMThHZR5xXBrwtPNnP6eRzvzhJWzrF1d5UI759//9iYra5lrt1ycDeXsaOf+n4Y4HX8+r9SciasLS/6P12LHoyF47wipYKpM9gqePpyyvVMxjBn+1DILH17l0rkZlhYcLetT3n/A5FGfTtvTCh4ibjpVI81/J0il62V1rNStLp6j4jowxwiJh2iMevtsbU3pjYQim6Nz9hJLH13i8mrGbJgSxjuUvQfEjT2GOw4Zddh0BakPtFPH/jRKqaCSrwetKi9mKIIYhyFuOg6TurCcF7ae5ty5t48/vcTiz3+Ei+szLC06WnGA7TykfNCnmObk5nAtj+53yDcTZteMwZOChbkMVa3/14DCJBBBFBxyopndUUBqwwxu/XjAJ778Bc4ul+R3fgyj9yg/GDMpFZxHgoe0hQ9QDRzFI48zqHzJeKhcvNKiLAyiQ1BwgsOcqNZ9Xo96rgaVgmqdiBACH740y/1/+j4b7w1x80tUkwLM4VptJAQMcN6o+p7iYQIqhBQmsUIrR7cd0NI12ODAwJnWTpvZoRGqdRrS1CECeR7Z2NzHRBjeecgbX/kut39UUlx5ATe3gFUlAM5Tb/4oHCJl0oadUcFsK8E5sFh7ZaqgEFBr3KyhxAySxDMplDv3BuztlmhhZC6QqONDrXkW28Kbf/1d7A9+i6vPr7N/e5vQDZQ7jvxxUqdPwFRwHaP3sGR1boZYGabWwHdd7MHMMBUMh5mRJp6tnZx33tllJbS53J1jZj6BKpKPpkhZ8e5mj3Nf/FV+8ZOnmNz6Pr6TUvUcxeOAc0cN6xwUoWIyUhZXMqqq6bKmrQ0hWBQzdXX+nGOwV3LnJ30+trjE8mxGUUVG/Qn7owkt7/hpb5fwmat84qXz5P95EwmOqh8oNkKTSkFqICTJYDgpCOppZYGyjIfgdoApQRScCtEgDZ5b93dxuWd3OuWD8T5pqbSLyEyWcHdrh+S3r/Kpz15i/z/eQhIh9hOKjQRTyBYgFhAndSpDW+gNC+ZbGR6jjM3BVsNsXQNqhqpiOPIicuXcLFvdKT7xtNXTG+X0Bopt9Wi9dJUXX7rI5O23kNQRB4F8I0CEzgr0/B7t2YB/2iJOQDrK8HHJ+dkZqqI+JfUAYRXMGc6pM1MBBa2Ubttz5ews51c6XFjr8OJHT5N0HPIbz/Ppz11h/+03IXVUg4T8ccAidFaNp7LHzdd2uXdvj866gQq5L8nHMD+TUJUG0aAyJBqidZQcUfFqeK2vlEo1rYh5iUXl6eMB5eoKv37tOUb/9q+4LBD7gXKj3ry7Ak/iHrdeH/ArC7P07ykPekMWzkFvkNMmkHkgagNoDciogYKzaGg0YgSNYLEJj0ndfiGj6o/Z3i5or5+j2BTKJwlawcwKbMQ9fvK9Ib+QtgitQMg8d384wVYLdocls1mCad1+Fg1TDgeV4UxFTB2ow44NjQ5rwcwZ+MjqmLe/dZOBLOHHCeUUZteED+Ie73xvyHNpmyx1TERozXieW1/mO/+yzWhTWVtsU5aCqKvRT4+GmRDEHGJ6yPPMBHNGeqbCdwqs02X9l14g/a8H3PzWm/zyp1ZZXgn8tDfgnbeGPN/qIEDSyhh7w2OcPZWRsESWCq3EoVFPksZDMuUIps7qLjDUQBxkZ0tcu8KSGdIXPo3OLbOSdfj4YMIPbg6ZXxux9wRmWimFKe0k0J7v8H5vQHchpSoiC536bKhKEDlCJzM7JLJqhqtxoEYoUcEFw/Yduh8Jl38OP7eAlBOm29ssziV87JlluqMOL15ZZWVlhu0yMrfYxQQmZUU3TbCmnog1mRE1xOp9xOoaEwVRI9QnUGOECLoPNlcRlucJ6xeIgx3K/3kX23pINWnRdnBxtYNWkbkkMJ5JSLKEwaSgRFlsJWh1mNBDfsEBzZGaYIiBIgRtImCmmArhlBFOR1Q9xZ3b6OOHWFnifEq1U/84TpXgHL4hUh7hv3t91pY6eCeUpTYs2o6RisYEO6B5hpoSRCWaanNAGP5UVZ+r+YB4dxcbZ4QVx/Q9wSYg/uDYNvKyoorwgyfbtDuBC6c65HmFa5Jsh56f3NgO4iGeYC68n8Ty2WkphAUVv2hoLuggoXgUcJkRh0rVF7wHi3Vw1YxuSMh1zMpcmw8tdynzCmmCL1JjzQG3lpNU37xAdNnjYC79xywUfzbJNboEX214iqeeuN/IrkKw6BAvNUiJ1a2K0HKOT144jQBFoTWRayrc9LhWa1TTUTNqJ038wCf/LO//xZfWWw9u/zCt8uWpSiTixZ3UeJzQPkerdkyq1RryZwVN87CJBALR0ExULMtG4/VnPy4AG3/4m19YGG/9fT7NmZpEhzg5lCNHOo//774u7UYp2WEt1XrSftYMTcF32ynbrfmXz3ztjW86u3bNn/nq69/eaZ/6PUlbm/PB+YSapUozMG2uB0M5fI7WSKoKasjBu6qHczEjwWQ2iJc03XmaLb5y5mtvfNOuNeLUruHlVeK7f/7yxbXBu39U5sXnq7I8L2pH6rwRgCZyUpeKnZSOB5K2EbT1l5hPk0c+zV7bXXzmry7/5T/cvXENf/1V4v8CSYIrURx3kHgAAAAASUVORK5CYII=";
@@ -1,9 +1,7 @@
/**
* dsh-plugin-browserskill browser half: the `browser_inspect` keyed toolview
* plus the observation overlay (live thumbnails + interrupt). The
* default carrier is a floating card on the `shell.overlay` seat; when the
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
* moves into a sidebar tab instead (see observation-sidebar.tsx).
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
* toolview plus the observation overlay (live thumbnails + interrupt) floating
* over the shell via the `shell.overlay` seat.
*/
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
@@ -16,10 +14,9 @@ import { createElement } from "react";
// <style> tags; selectors stay unhashed so `cn(..., "bsk-obs")` roots match).
import "./bsk-tokens.nomodule.css";
import "./bsk-ui.nomodule.css";
import { BrowserInspectToolView } from "./BrowserInspectToolView";
import { ObservationOverlay } from "./ObservationOverlay";
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
import { ScreenshotToolView } from "./ScreenshotToolView";
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
export const inject = ["slots", "sessions"];
@@ -65,9 +62,9 @@ export function apply(ctx: ClientContext): void {
const sessions = ctx.get("sessions") as unknown as ISessions;
ctx.slots.inject("tool.call.toolview", () =>
ctx.slots.register(
{ name: "tool.call.toolview", key: "browser_inspect" },
{ name: "tool.call.toolview", key: "browser_screenshot" },
(props: ToolCallViewProps) =>
createElement(BrowserInspectToolView, {
createElement(ScreenshotToolView, {
...props,
loadImage: (attachment: ImageAttachmentRef) =>
loadSessionImage(sessions, props.sessionId, attachment),
@@ -84,14 +81,4 @@ export function apply(ctx: ClientContext): void {
createElement(ObservationOverlay, { store }),
),
);
// Optional carrier upgrade: when the dsh-better-sidebar plugin is installed,
// its service moves the tracking view into a sidebar tab (the floating
// overlay hides itself through the sidebar-mode flag). In profiles without
// the sidebar plugin this fiber never runs and nothing changes.
ctx.inject(["betterSidebar"], (injected) =>
registerObservationSidebar(
(injected as unknown as { betterSidebar: BetterSidebarLike }).betterSidebar,
store,
),
);
}
@@ -1,256 +0,0 @@
// better-sidebar carrier for the observation view. When the
// dsh-better-sidebar plugin is installed its client publishes a
// `betterSidebar` cordis service; the client entry then runs the
// registration below and the tracking view moves from the floating card
// into a single-instance sidebar tab (Document PiP pop-out stays available
// from inside the tab). Detection is purely service-based — profiles
// without the sidebar plugin never start this fiber and keep the floating
// overlay.
import { createElement, type ReactNode } from "react";
import { createPortal } from "react-dom";
import { BSK_LOGO_URL } from "./brand-icon";
import { OverlayBody } from "./ObservationOverlay";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { useObservationView, usePip, visibleToScope } from "./observation-view";
import { setSidebarMode } from "./sidebar-mode";
/** The tab title — "Browser Skill", distinct from the sidebar's built-in "browser" tab. */
const TAB_TITLE = "Browser Skill";
/** Tab strip icon: the BrowserSkill product mark at the requested size. */
function TabIcon({ size }: { size: number }) {
return (
<img
src={BSK_LOGO_URL}
width={size}
height={size}
alt=""
aria-hidden
className={css["brand-icon"]}
/>
);
}
/**
* Structural mirrors of dsh-better-sidebar's client service surface (only
* the slices this integration touches — see the upstream
* lib/types/client/service.d.ts). Declared locally so the plugin carries no
* dependency on the sidebar package; the service contract has been stable
* since v0.4.0 and newer capabilities arrive behind its `features` list.
*/
export interface SidebarTabLike {
id: string;
type: string;
title: string;
}
export interface SidebarLeafLike {
kind: "leaf";
id: string;
tabs: SidebarTabLike[];
active: string | null;
}
export interface SidebarSplitLike {
kind: "split";
id: string;
dir: "row" | "col";
sizes: number[];
children: SidebarNodeLike[];
}
export type SidebarNodeLike = SidebarLeafLike | SidebarSplitLike;
export interface SidebarStateLike {
splits: SidebarNodeLike;
bottomSplits: SidebarNodeLike;
/** Whether the right panel is expanded (the merged drawer on narrow screens). */
panelOpen?: boolean;
}
export interface SidebarSnapshotLike {
sessionId?: string;
state?: SidebarStateLike;
}
/** Props every tab component receives (the slices we read). */
export interface TabComponentPropsLike {
/** The conversation this sidebar instance belongs to. */
scope: { sessionId: string };
}
export interface TabDescriptorLike {
id: string;
title: string | (() => string);
icon?: ReactNode | ((size: number) => ReactNode);
order?: number;
/** Single-instance: opening focuses the existing tab instead of duplicating. */
single?: boolean;
/** Small pill on the tab strip; null/undefined hides it. */
badge?: (
ctx: unknown,
scope: { sessionId: string },
state: unknown,
) => string | number | null | undefined;
component: (props: TabComponentPropsLike) => ReactNode;
}
export interface BetterSidebarLike {
registerTab(descriptor: TabDescriptorLike): () => void;
openTab(seed: {
type: string;
title?: string;
/** Content seed (lands on tab.path); content opens expand the panel. */
path?: string;
}): void;
getSnapshot(): SidebarSnapshotLike;
/** Sidebar state feed (v0.12+): session switches, state and prefs changes. */
subscribeState?: (listener: () => void) => () => void;
isTabEnabled(id: string): boolean;
}
/** The registered tab type id (also the SidebarTab.type value). */
export const OBSERVATION_TAB_TYPE = "browserskill:observation";
/**
* Inert content seed carried on auto-opened tabs: its mere presence makes
* the sidebar treat the open as a content open (expanding the hosting panel
* so the tracking view lands in sight). Never read by our component.
*/
export const OBSERVATION_TAB_PATH = "browser-skill:observation";
/**
* The observation tab body: the same OverlayBody the floating card renders,
* minus the card chrome (no drag header, no collapse — the sidebar tab bar
* owns those), plus the PiP pop-out upgrade. The view is scoped to the
* sidebar's conversation: only browser sessions started by it (or its
* descendants) show here — the floating card keeps the global view.
*/
export function ObservationSidebarTab({
store,
scopeId,
}: {
store: ObservationClientStore;
scopeId: string;
}) {
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store, scopeId);
const { pipWindow, pipSupported, popOut } = usePip();
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinnedId}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined}
/>
);
if (pipWindow !== null) {
return createPortal(body, pipWindow.document.body);
}
return <div className={css["sidebar-tab"]}>{body}</div>;
}
function* leafNodes(node: SidebarNodeLike): Generator<SidebarLeafLike> {
if (node.kind === "leaf") {
yield node;
return;
}
for (const child of node.children) yield* leafNodes(child);
}
/** Whether a tab of our type is already open in either sidebar workbench. */
export function observationTabOpen(state: SidebarStateLike | undefined): boolean {
if (state === undefined) return false;
for (const root of [state.splits, state.bottomSplits]) {
for (const leaf of leafNodes(root)) {
if (leaf.tabs.some((tab) => tab.type === OBSERVATION_TAB_TYPE)) return true;
}
}
return false;
}
/**
* Register the observation sidebar tab and flip the carrier flag. Returns
* the disposer the cordis fiber invokes when the sidebar service goes away
* (plugin unload/HMR): the floating overlay then resumes as the carrier.
*/
export function registerObservationSidebar(
service: BetterSidebarLike,
store: ObservationClientStore,
): () => void {
setSidebarMode(true);
// Hold the feed for the whole sidebar lifetime so the auto-open watcher
// sees new sessions even while the tab itself is closed.
store.acquire();
const disposeTab = service.registerTab({
id: OBSERVATION_TAB_TYPE,
title: TAB_TITLE,
icon: (size: number) => createElement(TabIcon, { size }),
single: true,
// The badge counts only sessions visible to the tab strip's own
// conversation (global sessions from other conversations stay hidden,
// mirroring the tab body's scoped view).
badge: (_ctx, scope) => {
const count = store
.getSnapshot()
.sessions.filter((s) => visibleToScope(s, scope.sessionId)).length;
return count > 0 ? count : null;
},
component: (props) =>
createElement(ObservationSidebarTab, { store, scopeId: props.scope.sessionId }),
});
// Auto-open the tab when a browser session VISIBLE TO the active
// conversation appears (and right away when one is already live). The
// evaluation runs on observation publishes AND on sidebar state changes:
// the latter covers the sidebar store's async init (its state is
// undefined for the first beats after page load) and conversation
// switches. The open carries a content seed (`path`): only content opens
// land in sight — the sidebar expands the hosting panel for them. While
// the panel is OPEN an existing tab is never re-focused — the user may
// be reading another page on purpose; while it is collapsed a NEW
// session (0→N) nudges the tab back into sight, mirroring how the
// floating card used to reappear.
const activeVisibleCount = (): number => {
const activeId = service.getSnapshot().sessionId;
if (activeId === undefined) return 0;
return store.getSnapshot().sessions.filter((s) => visibleToScope(s, activeId)).length;
};
let previousVisible = activeVisibleCount();
const evaluate = (): void => {
const count = activeVisibleCount();
const { state, sessionId } = service.getSnapshot();
if (
state !== undefined &&
sessionId !== undefined &&
service.isTabEnabled(OBSERVATION_TAB_TYPE)
) {
const open = observationTabOpen(state);
if (count > 0 && !open) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
} else if (previousVisible === 0 && count > 0 && open && state.panelOpen === false) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
}
}
previousVisible = count;
};
evaluate();
const unsubscribe = store.subscribe(evaluate);
const unsubscribeState = service.subscribeState?.(evaluate);
return () => {
unsubscribe();
unsubscribeState?.();
disposeTab();
store.release();
setSidebarMode(false);
};
}
@@ -53,7 +53,6 @@ export interface OverlaySnapshot {
const STATE_URL = "/bsk-observation/state";
const EVENTS_URL = "/bsk-observation/events";
const INTERRUPT_URL = "/bsk-observation/interrupt";
const STOP_URL = "/bsk-observation/stop";
function revoke(url: string | undefined): void {
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
@@ -79,8 +78,6 @@ export class ObservationClientStore {
};
private available = true;
private started = false;
/** Refcount of mounted consumers (overlay card, sidebar tab, sidebar fiber). */
private consumers = 0;
constructor(private readonly deps: ObservationClientDeps) {}
@@ -192,23 +189,6 @@ export class ObservationClientStore {
this.publish();
}
/**
* Hold the feed for one consumer's lifetime: the stream starts with the
* first holder and stops with the last release. Several carriers can share
* the store (the floating card, the better-sidebar tab, and the sidebar
* integration fiber) without one unmount killing the others' updates.
*/
acquire(): void {
this.consumers += 1;
if (this.consumers === 1) this.start();
}
release(): void {
if (this.consumers === 0) return;
this.consumers -= 1;
if (this.consumers === 0) this.stop();
}
/** Forget one session's tracked + held frames, revoking their blob URLs. */
private dropThumb(sessionId: string): void {
const attachmentId = this.thumbBySession.get(sessionId);
@@ -312,23 +292,4 @@ export class ObservationClientStore {
return false;
}
}
/**
* Stop one session and close its Agent Window. The session's removal
* arrives through the SSE remove event — no local state is touched here.
*/
async stopSession(sessionId: string): Promise<boolean> {
try {
const res = await this.deps.fetchFn(STOP_URL, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ sessionId }),
});
if (!res.ok) return false;
const body = (await res.json()) as { stopped?: boolean };
return body.stopped === true;
} catch {
return false;
}
}
}
@@ -1,148 +0,0 @@
/**
* Shared view logic for the observation carriers (the floating overlay card
* and the better-sidebar tab): the store-backed view model (snapshot, focus
* pinning, elapsed ticker) and the Document PiP pop-out. Extracted from
* ObservationOverlay so both carriers run the same focus/interrupt behavior
* without duplicating hooks.
*/
import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
import type { SessionObservation } from "../observation";
import type { ObservationClientStore, OverlaySnapshot } from "./observation-store";
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Minimal Document PiP surface (TS lib.dom lacks it). */
export interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
export function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
/** Clone the host document's style/link nodes into a PiP window. */
export function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export interface ObservationView {
readonly snapshot: OverlaySnapshot;
/** Pinned session wins while it still exists; otherwise auto-follow. */
readonly focus: SessionObservation | undefined;
readonly pinnedId: string | null;
readonly onTogglePin: (sessionId: string) => void;
/** 1s ticker while anything is active (drives the elapsed readout). */
readonly now: number;
}
/**
* Whether one session is visible on a surface scoped to one DSH
* conversation: sessions started by that conversation or its descendants
* (the lineage ancestors ride along on the entry). Untracked sessions (no
* owner recorded) are hidden from scoped surfaces but stay in global views.
*/
export function visibleToScope(obs: SessionObservation, scopeId: string): boolean {
return obs.dshSessionIds?.includes(scopeId) === true;
}
/**
* The store-backed observation view model. Holds the feed for the component
* lifetime (refcounted — overlapping carriers never kill each other's
* stream). `scopeId` narrows the view to one DSH conversation's sessions
* (the better-sidebar tab); undefined keeps the global view (floating
* card, PiP).
*/
export function useObservationView(
store: ObservationClientStore,
scopeId?: string,
): ObservationView {
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.acquire();
return () => store.release();
}, [store]);
const scopedSnapshot: OverlaySnapshot =
scopeId === undefined
? snapshot
: {
...snapshot,
sessions: snapshot.sessions.filter((s) => visibleToScope(s, scopeId)),
};
const [pinnedId, setPinnedId] = useState<string | null>(null);
const pinned =
pinnedId !== null ? scopedSnapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(scopedSnapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const anyActive = scopedSnapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
return {
snapshot: scopedSnapshot,
focus,
pinnedId: pinned !== undefined ? pinnedId : null,
onTogglePin,
now,
};
}
export interface PipHandle {
/** The open PiP window, or null while the content renders in its carrier. */
readonly pipWindow: Window | null;
/** Whether the browser exposes Document PiP (Chrome-only today). */
readonly pipSupported: boolean;
/**
* Upgrade the content into a native PiP window. Rejects silently without a
* user gesture (or when closed mid-request): the carrier keeps the content.
*/
readonly popOut: (size?: { width: number; height: number }) => void;
}
export function usePip(): PipHandle {
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const popOut = useCallback((size?: { width: number; height: number }): void => {
const pip = pipApi();
if (pip === undefined) return;
void pip
.requestWindow(size)
.then((win) => {
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
})
.catch(() => {});
}, []);
// Unmounting the carrier (tab close, conversation switch, plugin HMR) does
// not destroy the PiP browsing context: close it ourselves or it survives
// as a blank window. close() on an already-closed window is a no-op.
useEffect(() => () => pipWindow?.close(), [pipWindow]);
return { pipWindow, pipSupported: pipApi() !== undefined, popOut };
}
@@ -1,26 +0,0 @@
/**
* Carrier switch for the observation view: while the better-sidebar
* integration fiber is alive (the dsh-better-sidebar plugin provides its
* `betterSidebar` service), the tracking view lives in a sidebar tab and the
* floating overlay card/capsule hides itself. The flag is a tiny external
* store so the overlay can read it through useSyncExternalStore and flip
* without a remount when the sidebar plugin (un)loads.
*/
let active = false;
const listeners = new Set<() => void>();
export function getSidebarMode(): boolean {
return active;
}
export function subscribeSidebarMode(listener: () => void): () => void {
listeners.add(listener);
return () => listeners.delete(listener);
}
export function setSidebarMode(next: boolean): void {
if (active === next) return;
active = next;
for (const listener of [...listeners]) listener();
}
+9 -65
View File
@@ -1,11 +1,10 @@
/**
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
* file path plus pixel metadata); when the host mounts a durable attachment
* store AND the calling route declares image input, the capture bytes are
* additionally committed through `ctx.attachments` so the render step can
* attach the image itself to the tool result. Any uncertainty (no store,
* unknown route, text-only model, unrecognized bytes) falls back to the
* path-only form.
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
* path plus pixel metadata); when the host mounts a durable attachment store
* AND the calling route declares image input, the PNG bytes are additionally
* committed through `ctx.attachments` so the render step can attach the image
* itself to the tool result. Any uncertainty (no store, unknown route,
* text-only model) falls back to the path-only form.
*/
import type { Context } from "@deepseek-ai/cordis";
@@ -34,54 +33,6 @@ interface LlmLike {
): Promise<{ inputModalities?: readonly string[] }>;
}
/**
* Sniff the image media type from magic bytes. The capture path can hand back
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
* returning JPEG regardless of the requested format), so the declared type can
* never be trusted from the file extension alone.
* @param data - first bytes of the capture.
* @returns the detected media type, or undefined when unrecognized.
*/
function sniffImageMediaType(data: Uint8Array): string | undefined {
if (data.length >= 4) {
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
return "image/png";
}
// JPEG SOI: FF D8 FF
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
return "image/jpeg";
}
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
if (
data[0] === 0x52 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x46 &&
data.length >= 12 &&
data[8] === 0x57 &&
data[9] === 0x45 &&
data[10] === 0x42 &&
data[11] === 0x50
) {
return "image/webp";
}
// GIF: "GIF87a" or "GIF89a"
if (
data[0] === 0x47 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x38 &&
data.length >= 6 &&
(data[4] === 0x37 || data[4] === 0x39) &&
data[5] === 0x61
) {
return "image/gif";
}
}
return undefined;
}
/**
* Commit screenshot bytes to the host attachment store when the composition
* supports durable images on the current model route.
@@ -95,14 +46,7 @@ export async function trySaveScreenshot(
): Promise<ImageAttachmentRef | undefined> {
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
if (attachments === undefined) return undefined;
// Declare the real media type from the bytes, never a hard-coded "image/png".
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
// silently degrades to path-only; sniffing first lets JPEG captures (returned
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
// inline correctly.
const mediaType = sniffImageMediaType(data);
if (mediaType === undefined) return undefined;
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
if (
data.byteLength >
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
@@ -111,9 +55,9 @@ export async function trySaveScreenshot(
}
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
try {
return await attachments.saveImage({ data, mediaType, name });
return await attachments.saveImage({ data, mediaType: "image/png", name });
} catch {
// A store hiccup must not fail the tool call; the path-only form still works.
// A store hiccup must not fail the tool call; the PNG path still works.
return undefined;
}
}
+15 -24
View File
@@ -12,18 +12,16 @@
import type { Context } from "@deepseek-ai/cordis";
import Schema from "@deepseek-ai/schemastery";
import { armArchiveCleanup } from "./archive-cleanup";
import { registerBrowserTools } from "./browser-tools";
import { armLazyTools } from "./lazy-tools";
import { ObservationService } from "./observation";
import { registerObservationRoutes } from "./observation-http";
import { KeyedExecutor } from "./queue";
import { type BskRunner, createBskRunner } from "./runner";
import { SessionRegistry } from "./sessions";
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
import type { PluginConfig } from "./tools";
import { registerBskSkill } from "./skill";
import { type PluginConfig, registerTools } from "./tools";
export const name = "@wxg-prc-cpg/browser-skill-dsh-plugin";
export const name = "dsh-plugin-browserskill";
export const inject = ["tools"];
/** Runtime configuration schema (validated and defaulted by Cordis). */
@@ -96,14 +94,11 @@ export function apply(
// ONLY model-visible advertisement — the tool suite reveals itself on a
// successful skill invocation (or a session whose history already has one).
const unregisterSkill = registerBskSkill(ctx);
// A same-name CLI skill left in ~/.agents is discovered in the nearer preset
// layer and shadows the global registration above. Re-register through every
// exact agent context at startup so DSH always loads the browser_* protocol
// instructions; the shared CLI skill remains untouched for other agents.
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
const registerSuite = () =>
registerBrowserTools({ ctx, runner, registry, config: resolved, observation, queue });
const removeSuite = resolved.lazyTools ? armLazyTools(ctx, registerSuite) : registerSuite();
const removeSuite = resolved.lazyTools
? armLazyTools(ctx, () =>
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
)
: registerTools({ ctx, runner, registry, config: resolved, observation, queue });
// Route registration rides ctx.inject: the webServer service may be provided
// AFTER this plugin loads, and in headless compositions it never appears (the
// callback simply never runs, leaving the rest of the plugin unaffected).
@@ -112,10 +107,6 @@ export function apply(
removeRoutes = registerObservationRoutes(injected, observation);
return () => removeRoutes();
});
// Reap a conversation's browsers when the conversation itself is archived:
// archived sessions are hidden from every surface, so their Agent Windows
// would otherwise linger unreachable until idle timeout or unload.
const disarmArchiveCleanup = armArchiveCleanup(ctx, registry, observation);
// Non-blocking install probe: warn early when bsk is missing instead of
// failing the first tool call with a bare spawn error. Uses --version on
@@ -132,28 +123,27 @@ export function apply(
);
// Unload cleanup: kill in-flight children, then stop every session this
// plugin OWNS (created via browser_session action=start). Referenced or unknown
// plugin OWNS (created via browser_session_start). Referenced or unknown
// sessions belonging to other programs on the shared daemon are never
// touched; per-stop failures (already stopped externally, daemon restart)
// are swallowed so one stale handle cannot abort the rest.
ctx.effect(() => {
return () => {
removeSuite();
disarmAgentSkill();
unregisterSkill();
removeRoutes();
disarmArchiveCleanup();
observation.dispose();
runner.killAll();
const stops = registry
.ownedIds()
.map((sessionId) => observation.stopSession(sessionId).catch(() => false));
return Promise.all(stops).then(() => observation.dispose());
.map((sessionId) =>
runner.run(["session", "stop", sessionId], { timeoutMs: 15_000 }).catch(() => {}),
);
return Promise.all(stops).then(() => {});
};
});
}
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
export { registerBrowserTools } from "./browser-tools";
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
export { ObservationService } from "./observation";
export { registerObservationRoutes } from "./observation-http";
@@ -162,3 +152,4 @@ export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
export { BskError, createBskRunner } from "./runner";
export { SessionRegistry } from "./sessions";
export type { PluginConfig, ToolDeps } from "./tools";
export { registerTools } from "./tools";

Some files were not shown because too many files have changed in this diff Show More