Compare commits

..
105 changed files with 796 additions and 6687 deletions

No files matched your search

-177
View File
@@ -325,180 +325,3 @@ jobs:
echo "Chrome Web Store submission created successfully:"
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
publish-edge-addons:
needs: [resolve, build-extension]
runs-on: ubuntu-latest
environment: edge-addons
permissions:
contents: read
steps:
- name: Download build artifact
uses: actions/download-artifact@v8
with:
name: extension-zip
path: dist
- name: Validate Microsoft Edge Add-ons configuration
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
run: |
set -euo pipefail
if [ -z "$API_KEY" ]; then
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
exit 1
fi
if [ -z "$CLIENT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
exit 1
fi
if [ -z "$PRODUCT_ID" ]; then
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
exit 1
fi
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
exit 1
fi
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
if [ -z "$ARCHIVE" ]; then
echo "::error::No extension zip found in the downloaded artifact"
exit 1
fi
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
- name: Upload and publish Microsoft Edge Add-ons extension
shell: bash
env:
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
operation_id_from_headers() {
local headers="$1"
local location
location="$(awk '
tolower($1) == "location:" {
sub(/\r$/, "")
sub(/^[^:]*:[[:space:]]*/, "")
print
exit
}
' "$headers")"
location="${location%%\?*}"
printf '%s' "${location##*/}"
}
poll_operation() {
local operation_name="$1"
local status_url="$2"
local status="InProgress"
local http_status
for attempt in $(seq 1 30); do
sleep 5
http_status="$(curl --silent --show-error \
--output "$STATUS_RESPONSE" \
--write-out '%{http_code}' \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
"$status_url")"
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
exit 1
fi
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
echo "$operation_name status check $attempt/30: $status"
case "$status" in
Succeeded)
return 0
;;
InProgress)
;;
Failed)
echo "::error::Microsoft Edge Add-ons $operation_name failed"
jq . "$STATUS_RESPONSE"
exit 1
;;
*)
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
jq . "$STATUS_RESPONSE"
exit 1
;;
esac
done
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
jq . "$STATUS_RESPONSE"
exit 1
}
HTTP_STATUS="$(curl --silent --show-error \
--output "$UPLOAD_RESPONSE" \
--dump-header "$UPLOAD_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/zip" \
--data-binary "@$ARCHIVE" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
exit 1
fi
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
if [ -z "$UPLOAD_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
cat "$UPLOAD_HEADERS"
exit 1
fi
poll_operation \
"package upload" \
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
HTTP_STATUS="$(curl --silent --show-error \
--output "$PUBLISH_RESPONSE" \
--dump-header "$PUBLISH_HEADERS" \
--write-out '%{http_code}' \
--request POST \
--header "Authorization: ApiKey $API_KEY" \
--header "X-ClientID: $CLIENT_ID" \
--header "Content-Type: application/json" \
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
"$API_ROOT/products/$PRODUCT_ID/submissions")"
if [ "$HTTP_STATUS" != "202" ]; then
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
exit 1
fi
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
if [ -z "$PUBLISH_OPERATION_ID" ]; then
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
cat "$PUBLISH_HEADERS"
exit 1
fi
poll_operation \
"submission" \
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
echo "Microsoft Edge Add-ons submission created successfully:"
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
-4
View File
@@ -11,10 +11,6 @@ dist/
# Logs
*.log
# Local browser evaluation artifacts
evals/browser/results/
evals/browser/agents.local.json
# macOS
.DS_Store
-1
View File
@@ -208,7 +208,6 @@ The repository is a Cargo + pnpm workspace:
- `apps/extension` — browser extension
- `packages/ui` and `packages/i18n` — shared extension UI support
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
## License
-1
View File
@@ -178,7 +178,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
- `apps/extension` — 浏览器扩展
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
## 许可证
@@ -247,41 +247,6 @@ describe("App", () => {
expect(copyButton.getAttribute("disabled")).not.toBeNull();
expect(screen.getByText("连接后可用")).toBeTruthy();
});
it("treats protocol drift as still connected and prompts an upgrade", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
state: "version_skew",
instanceId: "03c3e47f",
handshake: {
server: "bh",
version: mockDaemonVersion,
protocol_version: "1.0",
},
},
statusState: "version_skew",
setLabel,
setConnectionEnabled,
});
render(<App />);
expect(screen.getByText("已连接")).toBeTruthy();
expect(screen.getByText("可升级")).toBeTruthy();
expect(screen.queryByText("协议不一致")).toBeNull();
expect(screen.queryByText("Action needed")).toBeNull();
expect(screen.queryByText("兼容")).toBeNull();
const warning = screen.getByText(/协议版本不同,请及时升级/);
expect(warning.textContent).toContain("CLI 协议");
expect(warning.textContent).toContain("扩展协议");
openRecordView();
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
expect(copyButton.getAttribute("disabled")).toBeNull();
fireEvent.click(copyButton);
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
});
});
describe("control hints toggle", () => {
+3 -4
View File
@@ -79,7 +79,6 @@ export function App() {
}, [copiedTick]);
const isSkewed = statusState === "version_skew";
const connectionLive = statusState === "connected" || isSkewed;
const daemonVersion = snapshot.handshake?.version ?? "—";
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—";
@@ -91,7 +90,7 @@ export function App() {
setCopiedInstanceId(true);
};
const recordReady = connectionLive && Boolean(snapshot.instanceId);
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
const recordPurpose = purposeDraft.trim();
const recordStartUrl = startUrlDraft.trim();
const recordCommand = snapshot.instanceId
@@ -207,12 +206,12 @@ export function App() {
</div>
{isSkewed && (
<p
className="mt-2 text-xs leading-snug text-muted-foreground"
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
data-slot="popup-version-skew-warning"
>
{t("popup.versionSkewWarning", {
extensionProtocol: PROTOCOL_VERSION,
cliProtocol: daemonProtocol,
daemonProtocol,
})}
</p>
)}
@@ -1,104 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { fillEditable } from "../editable-input";
import type { CdpRunner } from "../shared";
interface FakeOptions {
initialValue?: string;
finalValue: string;
remount?: boolean;
}
function editableState(value: string, connected = true, focused = true) {
return {
result: {
value: { supported: true, connected, focused, value, kind: "input" },
},
};
}
function makeEditableCdp(options: FakeOptions) {
const sent: Array<{ method: string; params?: object }> = [];
let inserted = false;
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
sent.push({ method, params });
if (method === "Input.insertText") {
inserted = true;
return {} as T;
}
if (method === "Input.dispatchKeyEvent") {
if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
return {} as T;
}
if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
const call = params as { objectId?: string; functionDeclaration?: string };
const fn = call.functionDeclaration ?? "";
if (fn.includes("bsk-editable-selection")) {
return { result: { value: { selected: true } } } as T;
}
if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
if (fn.includes("bsk-editable-live")) {
return {
result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
} as T;
}
if (fn.includes("bsk-editable-state")) {
if (call.objectId === "replacement") return editableState(options.finalValue) as T;
if (options.remount && inserted) {
return editableState(options.finalValue, false, false) as T;
}
return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
}
throw new Error("unexpected Runtime.callFunctionOn function");
});
return { cdp: { send } as CdpRunner, sent };
}
describe("fillEditable", () => {
it("replaces text through the browser input path without synthetic events or value setters", async () => {
const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
expect(result).toMatchObject({ value: "新值", replacedNode: false });
expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
const declarations = fake.sent
.map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
.join("\n");
expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
});
it("reacquires the focused editable after a framework remount", async () => {
const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
expect(result).toMatchObject({ value: "persisted", replacedNode: true });
});
it("reports a page rollback after the bounded settlement", async () => {
const fake = makeEditableCdp({ finalValue: "" });
const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
expect(result).toMatchObject({
code: "cdp_failed",
data: {
reason: "input_not_applied",
expected_value_length: 8,
actual_value_length: 0,
phase: "readback",
},
});
});
it("clears to an empty value with a browser Backspace edit", async () => {
const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
expect(result).toMatchObject({ value: "" });
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
});
});
@@ -7,7 +7,6 @@ import {
handleHover,
handlePress,
handleSelect,
handleType,
modifiersBitfield,
parseKeySpec,
resolveKeyDescriptor,
@@ -72,21 +71,8 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
function fillRuntimeReadback(value: string) {
return (params: unknown) => {
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
if (fn.includes("bsk-editable-state")) {
return {
result: {
value: {
supported: true,
connected: true,
focused: true,
value,
kind: "input",
},
},
};
}
if (fn.includes("bsk-editable-selection")) {
return { result: { value: { selected: true } } };
if (fn.includes("bsk-input-readback")) {
return { result: { value: { supported: true, value } } };
}
return { result: { type: "undefined" } };
};
@@ -784,13 +770,13 @@ describe("handleFill", () => {
expect(res.used_ref).toBe("e1");
const insert = fake.sent.find((c) => c.method === "Input.insertText");
expect(insert?.params).toEqual({ text: "hello" });
// clear_before defaults to true, so the edit transaction prepares a
// replacement selection and performs a settled live-value readback.
// clear_before defaults to true → callFunctionOn invoked once to
// clear, once to fire input/change after typing.
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
expect(callFns.length).toBeGreaterThanOrEqual(2);
});
it("clear_before=false skips replacement selection", async () => {
it("clear_before=false skips the wipe call", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 1, { tabId: 4 });
@@ -803,20 +789,12 @@ describe("handleFill", () => {
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
"Runtime.callFunctionOn": (p) => {
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
if (fn.includes("bsk-editable-state")) {
return {
result: {
value: {
supported: true,
connected: true,
focused: true,
value: "existingx",
kind: "textarea",
},
},
};
if (fn.includes("bsk-input-readback")) {
return { result: { value: { supported: true, value: "existingx" } } };
}
expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
// No "this.value = ''" clearing on a no-clear path; only the
// post-input dispatchEvent.
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
return { result: { type: "undefined" } };
},
"Input.insertText": () => ({}),
@@ -892,23 +870,9 @@ describe("handleFill", () => {
}
if (method === "Runtime.callFunctionOn") {
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
if (fn.includes("bsk-editable-state")) {
return {
result: {
value: {
supported: true,
connected: true,
focused: true,
value: "中文输入",
kind: "input",
},
},
};
}
if (fn.includes("bsk-editable-selection")) {
return { result: { value: { selected: true } } };
}
return { result: { type: "undefined" } };
return fn.includes("bsk-input-readback")
? { result: { value: { supported: true, value: "中文输入" } } }
: { result: { type: "undefined" } };
}
throw new Error(`unexpected child call ${method}`);
}) as CdpRunner["sendToTarget"];
@@ -952,7 +916,7 @@ describe("handleFill", () => {
});
});
it("stops before Input.insertText when abort fires after preparing replacement", async () => {
it("stops before Input.insertText when abort fires after clearing", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 555, { tabId: 4 });
@@ -964,25 +928,8 @@ describe("handleFill", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
"Runtime.callFunctionOn": (params) => {
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
if (fn.includes("bsk-editable-state")) {
return {
result: {
value: {
supported: true,
connected: true,
focused: true,
value: "",
kind: "input",
},
},
};
}
if (fn.includes("bsk-editable-selection")) {
abort.abort();
return { result: { value: { selected: true } } };
}
"Runtime.callFunctionOn": () => {
abort.abort();
return { result: { type: "undefined" } };
},
"Input.insertText": () => ({}),
@@ -1025,204 +972,6 @@ describe("handleFill", () => {
});
});
describe("handleType", () => {
it("requires an explicit target mode instead of silently using focus", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const fake = makeFakeCdp({});
const missing = await handleType(
sm,
{ session_id: "aa11", text: "x" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(missing).toMatchObject({
code: "invalid_params",
message: expect.stringContaining("focused=true"),
});
const mixed = await handleType(
sm,
{ session_id: "aa11", ref: "@e1", focused: true, text: "x" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(mixed).toMatchObject({
code: "invalid_params",
message: expect.stringContaining("not both"),
});
expect(fake.sent).toHaveLength(0);
});
it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 77, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 12,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const childCalls: Array<{ method: string; params?: object }> = [];
fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
childCalls.push({ method, params });
if (method === "DOM.describeNode") {
return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
}
return {};
}) as CdpRunner["sendToTarget"];
const result = await handleType(
sm,
{ session_id: "aa11", ref: "@e1", text: "测试" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in result) throw new Error(JSON.stringify(result));
expect(result).toMatchObject({
tab_id: 4,
used_ref: "e1",
text_length: 2,
verification: "dispatched",
});
expect(childCalls.map((call) => call.method)).toEqual([
"DOM.describeNode",
"DOM.scrollIntoViewIfNeeded",
"DOM.focus",
"Input.dispatchKeyEvent",
"Input.insertText",
"Input.dispatchKeyEvent",
]);
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const fake = makeFakeCdp({
"Runtime.evaluate": () => ({
result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const childCalls: string[] = [];
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
childCalls.push(method);
if (method === "Runtime.evaluate") {
return {
result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
};
}
return {};
}) as CdpRunner["sendToTarget"];
const result = await handleType(
sm,
{ session_id: "aa11", focused: true, text: "A" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in result) throw new Error(JSON.stringify(result));
expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
expect(childCalls).toEqual([
"Runtime.evaluate",
"Input.dispatchKeyEvent",
"Input.dispatchKeyEvent",
"Input.dispatchKeyEvent",
]);
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
});
it("refuses focused mode when focus is not owned by a text-entry target", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const fake = makeFakeCdp({
"Runtime.evaluate": () => ({
result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
}),
});
const result = await handleType(
sm,
{ session_id: "aa11", focused: true, text: "x" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(result).toMatchObject({
code: "cdp_failed",
data: { reason: "focus_target_unresolved", candidate_count: 0 },
});
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
});
it("rejects explicit non-text input controls", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 77, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.describeNode": () => ({
node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
}),
});
const result = await handleType(
sm,
{ session_id: "aa11", ref: "@e1", text: "x" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(result).toMatchObject({
code: "invalid_params",
data: { reason: "target_not_fillable" },
});
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
});
it("rejects a Surface ref instead of treating it as a text target", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e1", 77, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
const fake = makeFakeCdp({});
const result = await handleType(
sm,
{ session_id: "aa11", ref: "@e1", text: "x" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(result).toMatchObject({
code: "permission_denied",
data: { reason: "ref_capability_denied", required_capability: "interact" },
});
});
});
describe("parseKeySpec", () => {
it("splits compound expressions into modifiers + base key", () => {
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
@@ -1,128 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../shared";
import { dispatchTextInput } from "../text-input";
function makeCdp(onSend?: (method: string, params?: object) => void) {
const sent: Array<{ method: string; params?: object }> = [];
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
sent.push({ method, params });
onSend?.(method, params);
return {} as T;
});
return { cdp: { send } as CdpRunner, sent };
}
describe("dispatchTextInput", () => {
it("types ASCII through complete keydown, char, and keyup sequences", async () => {
const fake = makeCdp();
const result = await dispatchTextInput(fake.cdp, 4, "Ab");
expect(result).toBeNull();
expect(
fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
).toEqual([
"Input.dispatchKeyEvent:rawKeyDown",
"Input.dispatchKeyEvent:char",
"Input.dispatchKeyEvent:keyUp",
"Input.dispatchKeyEvent:rawKeyDown",
"Input.dispatchKeyEvent:char",
"Input.dispatchKeyEvent:keyUp",
]);
expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
expect(fake.sent[1].params).toMatchObject({ text: "A" });
});
it("types non-ASCII through the committed IME insertion path", async () => {
const fake = makeCdp();
const result = await dispatchTextInput(fake.cdp, 4, "测试");
expect(result).toBeNull();
expect(fake.sent).toEqual([
{
method: "Input.dispatchKeyEvent",
params: {
type: "rawKeyDown",
key: "Process",
code: "",
windowsVirtualKeyCode: 229,
nativeVirtualKeyCode: 229,
modifiers: 0,
},
},
{ method: "Input.insertText", params: { text: "测试" } },
{
method: "Input.dispatchKeyEvent",
params: {
type: "keyUp",
key: "Process",
code: "",
windowsVirtualKeyCode: 229,
nativeVirtualKeyCode: 229,
modifiers: 0,
},
},
]);
});
it("preserves text order across keyboard and IME segments", async () => {
const fake = makeCdp();
const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
expect(result).toBeNull();
const meaningful = fake.sent.flatMap((call) => {
if (call.method === "Input.insertText") {
return [`ime:${(call.params as { text: string }).text}`];
}
if (
call.method === "Input.dispatchKeyEvent" &&
(call.params as { type?: string }).type === "char"
) {
return [`key:${(call.params as { text: string }).text}`];
}
return [];
});
expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
});
it("reports cancellation observed after an accepted IME insertion", async () => {
const abort = new AbortController();
const fake = makeCdp((method) => {
if (method === "Input.insertText") abort.abort();
});
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
expect(result).toMatchObject({ code: "cancelled" });
expect(fake.sent.map((call) => call.method)).toEqual([
"Input.dispatchKeyEvent",
"Input.insertText",
"Input.dispatchKeyEvent",
]);
expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
});
it("releases the IME process key without committing when cancellation follows keydown", async () => {
const abort = new AbortController();
const fake = makeCdp((method, params) => {
if (
method === "Input.dispatchKeyEvent" &&
(params as { type?: string }).type === "rawKeyDown"
) {
abort.abort();
}
});
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
expect(result).toMatchObject({ code: "cancelled" });
expect(fake.sent.map((call) => call.method)).toEqual([
"Input.dispatchKeyEvent",
"Input.dispatchKeyEvent",
]);
expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
});
});
+1 -21
View File
@@ -28,7 +28,6 @@ import type {
ScreenshotParams,
SelectParams,
SnapshotParams,
TypeParams,
WaitForNavigationParams,
} from "@/transport/types";
import { isRequestFrame } from "@/transport/types";
@@ -36,14 +35,7 @@ import { handleConsole } from "./console";
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import {
handleClick,
handleFill,
handleHover,
handlePress,
handleSelect,
handleType,
} from "./interaction";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
handleNavigateBack,
@@ -499,17 +491,6 @@ export class ToolDispatcher {
),
signal,
);
case "tool.type":
return this.withHoverReleaseForRequest(
req.params as TypeParams,
() =>
handleType(
this.sessions,
req.params as TypeParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.press":
return this.withHoverReleaseForRequest(
req.params as PressParams,
@@ -736,7 +717,6 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.click":
case "tool.hover":
case "tool.fill":
case "tool.type":
case "tool.press":
case "tool.select":
case "tool.evaluate":
-282
View File
@@ -1,282 +0,0 @@
// Browser-native editing transaction for input, textarea, and contenteditable
// targets. The CDP target/session is selected by interaction.ts; this module
// owns the shorter-lived editable identity inside that target.
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import type { CdpRunner } from "./shared";
interface EditableState {
supported: boolean;
connected: boolean;
focused: boolean;
value: string;
kind: "input" | "textarea" | "contenteditable" | "unsupported";
}
export interface EditableFillResult {
value: string;
replacedNode: boolean;
}
function cancelled(signal: AbortSignal | undefined): RpcError | null {
return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
}
async function inspectEditable(
cdp: CdpRunner,
tabId: number,
objectId: string,
): Promise<EditableState> {
const reply = await cdp.send<{
result?: { value?: EditableState };
}>(tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
/* bsk-editable-state */
const doc = this.ownerDocument;
const focused = !!doc && doc.activeElement === this;
if (this instanceof HTMLInputElement) {
return { supported: true, connected: this.isConnected, focused,
value: this.value, kind: 'input' };
}
if (this instanceof HTMLTextAreaElement) {
return { supported: true, connected: this.isConnected, focused,
value: this.value, kind: 'textarea' };
}
if (this.isContentEditable) {
return { supported: true, connected: this.isConnected, focused,
value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
}
return { supported: false, connected: this.isConnected === true, focused,
value: '', kind: 'unsupported' };
}`,
returnByValue: true,
});
const state = reply.result?.value;
if (!state || typeof state.value !== "string") {
throw new Error("editable state probe returned no value");
}
return state;
}
/**
* Return the active editable from the anchor's own document. Using the
* anchor's execution context also works for same-process nested frames, where
* a target-level Runtime.evaluate would otherwise see the top document.
*/
async function focusedEditableFromAnchor(
cdp: CdpRunner,
tabId: number,
anchorObjectId: string,
): Promise<string | null> {
const reply = await cdp.send<{
result?: { objectId?: string; subtype?: string };
}>(tabId, "Runtime.callFunctionOn", {
objectId: anchorObjectId,
functionDeclaration: `function() {
/* bsk-editable-live */
const active = this.ownerDocument?.activeElement;
if (!active) return null;
const editable = active instanceof HTMLInputElement ||
active instanceof HTMLTextAreaElement || active.isContentEditable;
return editable ? active : null;
}`,
returnByValue: false,
});
if (reply.result?.subtype === "null") return null;
return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
}
async function resolveLiveEditable(
cdp: CdpRunner,
tabId: number,
anchorObjectId: string,
): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
if (anchor.supported && anchor.connected && anchor.focused) {
return { objectId: anchorObjectId, state: anchor, replaced: false };
}
const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
if (!focusedObjectId) {
return rpcError(
"cdp_failed",
"input_not_applied",
"the target editable lost focus or detached before input",
{ phase: "resolve_live_editable" },
);
}
const focused = await inspectEditable(cdp, tabId, focusedObjectId);
if (!focused.supported || !focused.connected || !focused.focused) {
return rpcError(
"cdp_failed",
"input_not_applied",
"the replacement editable was not live and focused",
{ phase: "resolve_live_editable" },
);
}
return { objectId: focusedObjectId, state: focused, replaced: true };
}
async function selectEditableContents(
cdp: CdpRunner,
tabId: number,
objectId: string,
): Promise<void> {
const reply = await cdp.send<{
result?: { value?: { selected?: boolean } };
}>(tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
/* bsk-editable-selection */
if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
return { selected: false };
}
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
try {
this.setSelectionRange(0, this.value.length);
} catch {
if (typeof this.select !== 'function') return { selected: false };
this.select();
}
return { selected: true };
}
if (this.isContentEditable) {
const selection = this.ownerDocument.getSelection();
if (!selection) return { selected: false };
const range = this.ownerDocument.createRange();
range.selectNodeContents(this);
selection.removeAllRanges();
selection.addRange(range);
return { selected: true };
}
return { selected: false };
}`,
returnByValue: true,
});
if (reply.result?.value?.selected !== true) {
throw new Error("could not select the live editable contents");
}
}
async function waitForEditableSettlement(
cdp: CdpRunner,
tabId: number,
anchorObjectId: string,
): Promise<void> {
await cdp.send(tabId, "Runtime.callFunctionOn", {
objectId: anchorObjectId,
functionDeclaration: `function() {
/* bsk-editable-sync */
return new Promise((resolve) => {
let settled = false;
const finish = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve();
};
const timer = setTimeout(finish, 100);
const view = this.ownerDocument?.defaultView;
if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
});
}`,
awaitPromise: true,
returnByValue: true,
});
}
async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "rawKeyDown",
key: "Backspace",
code: "Backspace",
windowsVirtualKeyCode: 8,
});
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "keyUp",
key: "Backspace",
code: "Backspace",
windowsVirtualKeyCode: 8,
});
}
/**
* Replace or append text through Chromium's editing pipeline. No value setter
* or synthetic DOM events are used: controlled components observe the same
* beforeinput/input path as a real browser edit. The live editable is resolved
* again after settlement so a framework may remount it without producing a
* false failure or a stale-object success.
*/
export async function fillEditable(
cdp: CdpRunner,
tabId: number,
anchorObjectId: string,
value: string,
clearBefore: boolean,
signal?: AbortSignal,
): Promise<EditableFillResult | RpcError> {
try {
const abortedBefore = cancelled(signal);
if (abortedBefore) return abortedBefore;
const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
if ("code" in initial) return initial;
let replacedNode = initial.replaced;
if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
const abortedAfterSelection = cancelled(signal);
if (abortedAfterSelection) return abortedAfterSelection;
if (value.length > 0) {
await cdp.send(tabId, "Input.insertText", { text: value });
} else if (clearBefore) {
await deleteSelection(cdp, tabId);
}
const abortedAfterInput = cancelled(signal);
if (abortedAfterInput) return abortedAfterInput;
await waitForEditableSettlement(cdp, tabId, initial.objectId);
const abortedAfterSettlement = cancelled(signal);
if (abortedAfterSettlement) return abortedAfterSettlement;
let finalObjectId = initial.objectId;
let finalState = await inspectEditable(cdp, tabId, finalObjectId);
if (!finalState.connected || !finalState.focused) {
const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
if ("code" in live) return live;
finalObjectId = live.objectId;
finalState = live.state;
replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
}
const applied = clearBefore
? finalState.value === value
: value.length === 0 || finalState.value.includes(value);
if (!finalState.supported || !applied) {
return rpcError(
"cdp_failed",
"input_not_applied",
"the page did not retain the requested input value",
{
expected_value_length: value.length,
actual_value_length: finalState.value.length,
phase: "readback",
editable_replaced: replacedNode,
},
);
}
return { value: finalState.value, replacedNode };
} catch (err) {
return rpcError(
"cdp_failed",
"input_not_applied",
err instanceof Error ? err.message : String(err),
{ phase: "edit_transaction" },
);
}
}
+119 -168
View File
@@ -1,5 +1,5 @@
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
// `tool.press`, and `tool.select`.
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
// `tool.select`.
//
// All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -27,11 +27,8 @@ import type {
RpcError,
SelectParams,
SelectResult,
TypeParams,
TypeResult,
} from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { fillEditable } from "./editable-input";
import { backendNodeToObject } from "./element-geometry";
import { rpcError } from "./errors";
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
@@ -48,7 +45,6 @@ import {
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import { resolveSurfacePointerTarget } from "./surface-target";
import { dispatchTextInput } from "./text-input";
export interface InteractionDeps {
cdp: CdpRunner;
@@ -248,20 +244,18 @@ function inputContextForNode(
interface FocusProbe {
hasFocus: boolean;
activeTag: string;
textEntry: boolean;
}
/** Resolve keyboard input to the CDP target that currently owns focus. */
async function resolveFocusedInputContext(
cdp: CdpRunner,
tabId: number,
mode: "press" | "type",
): Promise<InputDispatchContext | RpcError> {
if (!cdp.getFrameGraph) {
return rpcError(
"cdp_failed",
"focus_target_unresolved",
`${mode} cannot determine the focused frame target`,
"press cannot determine the focused frame target",
);
}
@@ -290,21 +284,9 @@ async function resolveFocusedInputContext(
expression: `(() => {
/* bsk-focus-target-probe */
const active = document.activeElement;
const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
const nonTextInputTypes = new Set([
'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
'time', 'week'
]);
const role = active instanceof Element ? active.getAttribute('role') : null;
return {
hasFocus: document.hasFocus(),
activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
textEntry:
(active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
active instanceof HTMLTextAreaElement ||
(active instanceof HTMLElement && active.isContentEditable) ||
role === 'textbox' || role === 'searchbox'
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
};
})()`,
returnByValue: true,
@@ -320,17 +302,16 @@ async function resolveFocusedInputContext(
// A top document whose active element is an iframe delegates focus to
// that frame. Removing those delegating targets leaves the deepest OOPIF
// target while same-process frames continue to use the shared root target.
const owners = focused.filter(({ probe }) => {
if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
return mode !== "type" || probe.textEntry;
});
const owners = focused.filter(
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
);
if (owners.length !== 1) {
return rpcError(
"cdp_failed",
"focus_target_unresolved",
owners.length === 0
? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
: `${mode} found multiple focused frame targets`,
? "press found no unique focused frame target"
: "press found multiple focused frame targets",
{ candidate_count: owners.length },
);
}
@@ -701,47 +682,6 @@ function isFillable(node: DescribedNode): boolean {
return false;
}
/** `type` also supports custom text-entry widgets that expose an ARIA role. */
function isTypeTarget(node: DescribedNode): boolean {
const tag = (node.nodeName ?? "").toUpperCase();
const attrs = node.attributes ?? [];
const attributes = new Map<string, string>();
for (let i = 0; i + 1 < attrs.length; i += 2) {
attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
}
if (tag === "TEXTAREA") return true;
if (tag === "INPUT") {
const type = attributes.get("type") ?? "text";
return !new Set([
"button",
"checkbox",
"color",
"date",
"datetime-local",
"file",
"hidden",
"image",
"month",
"radio",
"range",
"reset",
"submit",
"time",
"week",
]).has(type);
}
const contentEditable = attributes.get("contenteditable");
if (
contentEditable === "" ||
contentEditable === "true" ||
contentEditable === "plaintext-only"
) {
return true;
}
const role = attributes.get("role");
return role === "textbox" || role === "searchbox";
}
type SelectMutationResult =
| {
ok: true;
@@ -827,114 +767,125 @@ export async function handleFill(
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
const objectId = objectIdOrErr;
const clearBefore = params.clear_before ?? true;
const fillResult = await fillEditable(
nodeCdp,
target.tabId,
objectId,
params.value,
clearBefore,
deps.signal,
);
if (isRpcError(fillResult)) return fillResult;
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: input.usedRef,
used_selector: input.usedSelector,
value_length: fillResult.value.length,
});
}
// ---------------------------------------------------------------------------
// tool.type
// ---------------------------------------------------------------------------
export async function handleType(
manager: SessionManager,
params: TypeParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<TypeResult | RpcError> {
if (!params || typeof params.text !== "string") {
return { code: "invalid_params", message: "type requires a text string" };
}
if (params.focused !== undefined && typeof params.focused !== "boolean") {
return { code: "invalid_params", message: "type focused must be a boolean" };
}
const hasExplicitTarget =
(typeof params.ref === "string" && params.ref.length > 0) ||
(typeof params.selector === "string" && params.selector.length > 0);
if (params.focused === true && hasExplicitTarget) {
return {
code: "invalid_params",
message: "type: pass a ref/selector or focused=true, not both",
};
}
if (params.focused !== true && !hasExplicitTarget) {
return {
code: "invalid_params",
message: "type requires a ref/selector or focused=true",
};
}
const ctxOrErr = lookupSession(manager, params, "type");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "type");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
let input: InputDispatchContext | RpcError;
if (hasExplicitTarget) {
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
const explicitInput = inputContextForNode(deps.cdp, node);
if (isRpcError(explicitInput)) return explicitInput;
input = explicitInput;
try {
const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
target.tabId,
"DOM.describeNode",
{ backendNodeId: explicitInput.backendNodeId },
);
if (!described.node || !isTypeTarget(described.node)) {
return rpcError(
"invalid_params",
"target_not_fillable",
`element ${described.node?.nodeName ?? "?"} not text-editable`,
);
}
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollErr = await scrollElementAndFramesIntoView(
deps.cdp,
target.tabId,
explicitInput.target,
explicitInput.backendNodeId,
explicitInput.frameId,
);
if (scrollErr) return scrollErr;
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
backendNodeId: explicitInput.backendNodeId,
try {
if (clearBefore) {
// Clear input/textarea value or wipe contenteditable innerText,
// then fire `input` so frameworks observe the empty state.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
if (this.isContentEditable) { this.textContent = ''; }
else {
const proto = this instanceof HTMLTextAreaElement
? HTMLTextAreaElement.prototype
: HTMLInputElement.prototype;
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
if (descriptor && descriptor.set) descriptor.set.call(this, '');
else this.value = '';
}
this.dispatchEvent(new Event('input', { bubbles: true }));
}`,
returnByValue: true,
});
} catch (err) {
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
}
} else {
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
if (isRpcError(input)) return input;
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// CDP `Input.insertText` handles IME / multi-byte input out of the
// box, much more reliably than per-key `dispatchKeyEvent`.
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
// Fire `input` + `change` so React / Vue controlled inputs commit.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
this.dispatchEvent(new Event('input', { bubbles: true }));
this.dispatchEvent(new Event('change', { bubbles: true }));
}`,
returnByValue: true,
});
// Give controlled inputs one bounded browser-render turn to accept or
// reject the dispatched value before reporting success.
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
/* bsk-input-sync */
return new Promise((resolve) => {
let settled = false;
const finish = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve();
};
const timer = setTimeout(finish, 100);
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
requestAnimationFrame(() => requestAnimationFrame(finish));
});
}`,
awaitPromise: true,
returnByValue: true,
});
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
}
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
if (dispatchError) return dispatchError;
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "fill aborted" };
}
let actualValue: string;
try {
const readback = await nodeCdp.send<{
result?: { value?: { supported?: boolean; value?: string } };
}>(target.tabId, "Runtime.callFunctionOn", {
objectId,
functionDeclaration: `function() {
/* bsk-input-readback */
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
return { supported: true, value: this.value };
}
if (this.isContentEditable) {
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
}
return { supported: false, value: '' };
}`,
returnByValue: true,
});
const state = readback.result?.value;
actualValue = typeof state?.value === "string" ? state.value : "";
const applied =
state?.supported === true &&
(clearBefore
? actualValue === params.value
: params.value.length === 0 || actualValue.includes(params.value));
if (!applied) {
return rpcError(
"cdp_failed",
"input_not_applied",
"the page did not retain the requested input value",
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
);
}
} catch (err) {
return rpcError(
"cdp_failed",
"input_not_applied",
err instanceof Error ? err.message : String(err),
);
}
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: input.usedRef,
used_selector: input.usedSelector,
text_length: params.text.length,
verification: "dispatched",
value_length: actualValue.length,
});
}
@@ -1139,7 +1090,7 @@ export async function handlePress(
};
}
} else {
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
if (isRpcError(input)) return input;
}
-194
View File
@@ -1,194 +0,0 @@
// Browser-native text entry for live editing sessions. Unlike fill, this
// module does not assume that the focused editable retains a DOM value: a
// Canvas editor may use an input/textarea only as a transient keyboard/IME
// sink and clear it after consuming each event.
import type { RpcError } from "@/transport/types";
import type { CdpRunner } from "./shared";
function cancelled(signal: AbortSignal | undefined): RpcError | null {
return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
}
interface KeyDescriptor {
key: string;
code: string;
windowsVirtualKeyCode: number;
nativeVirtualKeyCode?: number;
modifiers: number;
}
function asciiKeyDescriptor(character: string): KeyDescriptor {
if (/^[a-z]$/.test(character)) {
return {
key: character,
code: `Key${character.toUpperCase()}`,
windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
modifiers: 0,
};
}
if (/^[A-Z]$/.test(character)) {
return {
key: character,
code: `Key${character}`,
windowsVirtualKeyCode: character.charCodeAt(0),
modifiers: 8,
};
}
if (/^[0-9]$/.test(character)) {
return {
key: character,
code: `Digit${character}`,
windowsVirtualKeyCode: character.charCodeAt(0),
modifiers: 0,
};
}
if (character === " ") {
return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
}
return {
key: character,
code: "",
windowsVirtualKeyCode: character.charCodeAt(0),
modifiers: 0,
};
}
/**
* Run one complete browser key transaction and always release the key. The
* commit callback owns the text-producing phase: a `char` event for a
* keyboard character, or `Input.insertText` for committed IME text.
*/
async function dispatchKeyTransaction(
cdp: CdpRunner,
tabId: number,
descriptor: KeyDescriptor,
commit: () => Promise<void>,
signal?: AbortSignal,
): Promise<RpcError | null> {
let keyDown = false;
try {
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "rawKeyDown",
...descriptor,
});
keyDown = true;
const abortedAfterDown = cancelled(signal);
if (!abortedAfterDown) await commit();
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "keyUp",
...descriptor,
});
keyDown = false;
return abortedAfterDown ?? cancelled(signal);
} catch (err) {
if (keyDown) {
try {
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "keyUp",
...descriptor,
});
} catch {
// Best-effort cleanup keeps the page from observing a stuck key.
}
}
throw err;
}
}
async function typeAsciiCharacter(
cdp: CdpRunner,
tabId: number,
character: string,
signal?: AbortSignal,
): Promise<RpcError | null> {
const descriptor = asciiKeyDescriptor(character);
return dispatchKeyTransaction(
cdp,
tabId,
descriptor,
async () => {
await cdp.send(tabId, "Input.dispatchKeyEvent", {
type: "char",
...descriptor,
text: character,
unmodifiedText: character,
});
},
signal,
);
}
async function typeImeSegment(
cdp: CdpRunner,
tabId: number,
text: string,
signal?: AbortSignal,
): Promise<RpcError | null> {
const abortedBefore = cancelled(signal);
if (abortedBefore) return abortedBefore;
// A committed IME value is still part of a keyboard transaction. The
// Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
// IME owns the keystroke. It lets selection-based editors enter text mode
// before insertText commits the Unicode value, without inventing a visible
// ASCII prefix or leaving a composition transaction open.
const processKey: KeyDescriptor = {
key: "Process",
code: "",
windowsVirtualKeyCode: 229,
nativeVirtualKeyCode: 229,
modifiers: 0,
};
return dispatchKeyTransaction(
cdp,
tabId,
processKey,
() => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
signal,
);
}
/**
* Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
* delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
* IME/software-keyboard insertion path. Successful return means the browser
* accepted the input commands, not that a DOM value retained the text.
*/
export async function dispatchTextInput(
cdp: CdpRunner,
tabId: number,
text: string,
signal?: AbortSignal,
): Promise<RpcError | null> {
const abortedBefore = cancelled(signal);
if (abortedBefore) return abortedBefore;
const segments: Array<{ ime: boolean; text: string }> = [];
for (const character of text) {
const ime = character.codePointAt(0)! > 0x7f;
const previous = segments.at(-1);
if (previous?.ime === ime) previous.text += character;
else segments.push({ ime, text: character });
}
try {
for (const segment of segments) {
if (segment.ime) {
const error = await typeImeSegment(cdp, tabId, segment.text, signal);
if (error) return error;
continue;
}
for (const character of segment.text) {
const error = await typeAsciiCharacter(cdp, tabId, character, signal);
if (error) return error;
}
}
return null;
} catch (err) {
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
};
}
}
-20
View File
@@ -484,26 +484,6 @@ export interface FillResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface TypeParams {
session_id: string;
text: string;
ref?: string;
selector?: string;
/** Explicitly dispatch to the unique focused text-entry target. */
focused?: boolean;
tab_id?: number;
timeout_ms?: number;
}
export interface TypeResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
text_length: number;
verification: "dispatched";
dialogs?: JavaScriptDialogInfo[];
}
export interface PressParams {
session_id: string;
key: string;
@@ -97,25 +97,14 @@ Run with a `900x700` Agent Window.
This page exercises generic frame-aware input routing. It contains a top-level
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
frame includes an input, textarea, contenteditable editor, an input that
replaces its DOM node after accepted input, an input that rejects synthetic
DOM events, a transient keyboard/IME input proxy that consumes and clears its
DOM value, and a controlled input that resets its value on the next animation
frame.
frame includes an input, textarea, contenteditable editor, and a controlled
input that resets its value on the next animation frame.
- `fill` must route focus, text insertion, DOM events, and readback through the
ref's owning CDP target.
- `press --ref` must focus and dispatch every key event through that same target.
- Untargeted `press` must follow the actual focused target or refuse when focus
ownership cannot be determined uniquely.
- The remounting input must retain the value through its replacement node.
- The trusted input must be filled without synthetic `input` / `change` events.
- `type` must deliver ASCII key events and committed non-ASCII IME/software
keyboard insertion to the input proxy. Its DOM value remains empty while
`proxy output` retains the consumed text; the result reports
`verification=dispatched`, not a retained DOM value.
- `type` must reject an omitted target unless `focused=true` is explicit, and
focused mode must reject focus that is not owned by a text-entry target.
- The rejecting input must return `input_not_applied`, never a successful
requested-value length.
@@ -14,38 +14,9 @@
<label>Frame input <input aria-label="frame input" /></label>
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
<div contenteditable="true" aria-label="frame editor"></div>
<label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
<label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
<label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
<output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
<script>
const params = new URLSearchParams(location.search);
const remountHost = document.querySelector("#remount-host");
const armRemount = (input) => {
input.addEventListener("input", () => {
const replacement = document.createElement("input");
replacement.setAttribute("aria-label", "remounting input");
replacement.value = input.value;
armRemount(replacement);
input.replaceWith(replacement);
replacement.focus();
}, { once: true });
};
armRemount(remountHost.querySelector("input"));
const trusted = document.querySelector("#trusted");
trusted.addEventListener("input", (event) => {
if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
});
const proxy = document.querySelector("#proxy");
const proxyOutput = document.querySelector("#proxy-output");
let proxyValue = "";
proxy.addEventListener("input", (event) => {
if (event.isComposing) return;
proxyValue += event.data ?? proxy.value;
proxy.value = "";
proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
});
const rejecting = document.querySelector("#rejecting");
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
if (params.get("nested") === "1") {
-1
View File
@@ -44,7 +44,6 @@
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
"!evals/browser/results/**",
"!**/target/**",
"!**/.wxt/**",
"!**/.output/**",
+7 -30
View File
@@ -29,7 +29,7 @@ Drive the user's **real Chromium browser** (with their logins and cookies) throu
## Prerequisites
1. `bsk` on `PATH` (Rust CLI from browser-skill)
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
## Mandatory workflow
@@ -70,31 +70,10 @@ bsk navigate <url> --session <id>
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
### Choosing `fill`, `type`, and `press`
Choose by the intended postcondition, not by whether observation happens to
render `textbox [empty]`:
- `fill` sets a field's retained value. Use it when success means the target
value must equal the requested text; it replaces existing content by default.
- `type` inserts text at the active caret/selection in a live editing session.
Use it when the application must consume keyboard/IME text; it never clears
first and reports dispatch only.
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
Do not encode special keys inside `type --text`.
`type` requires an explicit targeting mode: pass a ref/selector when one is
available, or pass `--focused` only after an immediately preceding action in
the same tab deliberately established a text-entry focus. Never use `fill`
as a probe and never automatically retry a failed `fill` with `type`: an
application may already have consumed the text before clearing its proxy DOM
value. Re-observe first. After `type`, verify the application output with one
observation or screenshot; `verification=dispatched` is not business success.
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
@@ -109,7 +88,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
```
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
## Observation priority
@@ -233,9 +212,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|---------|---------|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
@@ -243,7 +220,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
| Command | Summary |
|---------|---------|
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
@@ -327,12 +304,12 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
| Code | Meaning | What to do |
|------|---------|------------|
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
+11 -18
View File
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
}
}
/// `bsk doctor` check: connected browsers should speak a protocol the
/// daemon accepts. A different protocol string is still a live
/// connection — report Ok so agents keep working. The detail is an
/// upgrade reminder, not a blocker.
/// `bsk doctor` check: every connected browser should use the same
/// protocol version as the daemon. Minor protocol drift is accepted by
/// the daemon but flagged here so the user can see who needs updating
/// (M10.4).
///
/// Review M2 (round-1 minor): when no browsers are connected, the
/// check has nothing to compare against, so it now reports
@@ -396,12 +396,13 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
})
.collect::<Vec<_>>()
.join(", ");
CheckResult::ok(
CheckResult::fail(
name,
format!(
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
"{} browser(s) have protocol minor drift from the daemon: {stale}",
status.version_skew_browsers.len()
),
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
)
}
@@ -580,7 +581,7 @@ mod m2_tests {
}
#[test]
fn browsers_check_reports_ok_when_skew_present() {
fn browsers_check_reports_fail_when_skew_present() {
let status = fake_status(
vec![BrowserStatusEntry {
instance_id: "alpha".into(),
@@ -604,17 +605,9 @@ mod m2_tests {
}],
);
let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok);
assert_eq!(check.status, CheckStatus::Fail);
assert!(check.detail.contains("alpha"));
assert!(
check
.detail
.contains("still usable — continue, and upgrade soon")
);
assert!(
check.hint.is_none(),
"a protocol-version note must not hint to stop"
);
assert!(check.hint.is_some());
}
#[test]
@@ -642,7 +635,7 @@ mod m2_tests {
}],
);
let check = check_browsers_protocol_compatible(Some(&status));
assert_eq!(check.status, CheckStatus::Ok);
assert_eq!(check.status, CheckStatus::Fail);
assert!(
check
.detail
+1 -157
View File
@@ -11,7 +11,7 @@ use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -95,27 +95,6 @@ fn split_target(
}
}
fn split_type_target(
positional: Option<String>,
explicit_ref: Option<String>,
explicit_selector: Option<String>,
focused: bool,
) -> Result<(Option<String>, Option<String>), CliError> {
if focused {
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
return Err(CliError::Local(anyhow::anyhow!(
"pass a target or --focused, not both"
)));
}
return Ok((None, None));
}
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
CliError::Local(anyhow::anyhow!(
"type requires exactly one of: <target>, --ref, --selector, or --focused"
))
})
}
// ---------------------------------------------------------------------------
// bsk click
// ---------------------------------------------------------------------------
@@ -404,95 +383,6 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk type
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
#[group(id = "type_destination", required = true, multiple = false)]
pub struct TypeDestinationArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
#[arg(value_name = "TARGET")]
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
/// Insert into the unique focused text-entry target instead of focusing a target.
#[arg(long)]
pub focused: bool,
}
#[derive(Debug, Clone, Args)]
pub struct TypeArgs {
#[command(flatten)]
pub destination: TypeDestinationArgs,
/// Text to dispatch through the live keyboard/IME editing session.
#[arg(long)]
pub text: String,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_type_target(
args.destination.target.clone(),
args.destination.ref_.clone(),
args.destination.selector.clone(),
args.destination.focused,
)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = TypeParams {
session_id: args.session.clone(),
text: args.text.clone(),
ref_,
selector,
focused: args.destination.focused,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: TypeResult = call(
info.sock_path,
Method::ToolType,
params,
"type-1",
args.timeout,
)?;
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
let target = if args.destination.focused {
"focused".into()
} else {
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
};
println!(
"type dispatched tab={} target={target} length={} verification=dispatched",
reply.tab_id, reply.text_length
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk press
// ---------------------------------------------------------------------------
@@ -769,52 +659,6 @@ mod tests {
assert_eq!(parsed.args.session, "ohli");
}
#[test]
fn type_clap_accepts_explicit_target_and_text_flag() {
use clap::Parser;
#[derive(Parser)]
#[command(name = "type")]
struct Wrapper {
#[command(flatten)]
args: TypeArgs,
}
let parsed =
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
.expect("type args should parse");
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
assert_eq!(parsed.args.text, "测试");
assert_eq!(parsed.args.session, "ohli");
}
#[test]
fn type_targeting_requires_exactly_one_explicit_mode() {
use clap::Parser;
#[derive(Parser)]
#[command(name = "type")]
struct Wrapper {
#[command(flatten)]
args: TypeArgs,
}
let parsed =
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
.expect("focused type args should parse");
assert!(parsed.args.destination.focused);
assert!(parsed.args.destination.target.is_none());
let focused = split_type_target(None, None, None, true).unwrap();
assert_eq!(focused, (None, None));
let missing = split_type_target(None, None, None, false).unwrap_err();
assert!(missing.to_string().contains("--focused"));
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
assert!(mixed.to_string().contains("not both"));
}
#[test]
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
use clap::Parser;
+1 -4
View File
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
@@ -171,9 +171,6 @@ pub enum Command {
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
/// Insert text into an explicit target or focused keyboard/IME editing session.
Type(TypeArgs),
/// Dispatch a keyboard key combo.
Press(PressArgs),
+2 -2
View File
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
let bold = "\x1b[1m";
let reset = "\x1b[0m";
eprintln!(
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
s.version_skew_browsers.len()
);
for skew in &s.version_skew_browsers {
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
skew.label.clone()
};
eprintln!(
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
skew.instance_id,
label,
display_protocol(&skew.client_protocol_version),
-1
View File
@@ -244,7 +244,6 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolClick
| Method::ToolHover
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect
| Method::ToolEvaluate
-1
View File
@@ -95,7 +95,6 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Type(args) => cli::interaction::dispatch_type(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
+3 -66
View File
@@ -1,4 +1,5 @@
//! M7 integration: drive the navigation and interaction tool RPCs
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -15,8 +16,7 @@ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
WaitUntil,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -433,51 +433,6 @@ async fn fill_round_trips_clear_before_default() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn type_round_trips_dispatched_verification() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolType);
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
assert_eq!(p.text, "测试");
assert!(!p.focused);
ResponseBody::Ok(
serde_json::to_value(TypeResult {
tab_id: 12,
used_ref: Some("e3".into()),
used_selector: None,
text_length: 2,
verification: TypeVerification::Dispatched,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: TypeResult = ipc_tool_call(
&sock,
Method::ToolType,
TypeParams {
session_id,
text: "测试".into(),
ref_: Some("@e3".into()),
selector: None,
focused: false,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("type ok");
assert_eq!(result.text_length, 2);
assert_eq!(result.verification, TypeVerification::Dispatched);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn press_round_trips_compound_key() {
let (handle, sock) = spawn_daemon().await;
@@ -580,7 +535,6 @@ async fn m7_tools_propagate_extension_errors() {
| Method::ToolReload
| Method::ToolClick
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect => ResponseBody::Err(RpcError {
code: ErrorCode::CdpFailed,
@@ -688,23 +642,6 @@ async fn m7_tools_propagate_extension_errors() {
.unwrap_err();
assert_eq!(fill.code, ErrorCode::CdpFailed);
let type_result = ipc_tool_call::<_, Value>(
&sock,
Method::ToolType,
TypeParams {
session_id: session_id.clone(),
text: "hello".into(),
ref_: Some("@e1".into()),
selector: None,
focused: false,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.unwrap_err();
assert_eq!(type_result.code, ErrorCode::CdpFailed);
let press = ipc_tool_call::<_, Value>(
&sock,
Method::ToolPress,
@@ -26,7 +26,7 @@
"format": "int64"
},
"text": {
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
"type": "string"
},
"truncated": {
@@ -1,50 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TypeParams",
"type": "object",
"required": [
"session_id",
"text"
],
"properties": {
"focused": {
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
"type": "boolean"
},
"ref": {
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"text": {
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
"type": "string"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -1,124 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TypeResult",
"type": "object",
"required": [
"tab_id",
"text_length",
"verification"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"text_length": {
"description": "UTF-16 code-unit length of text accepted for dispatch.",
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"verification": {
"$ref": "#/definitions/TypeVerification"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
},
"TypeVerification": {
"oneOf": [
{
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
"type": "string",
"enum": [
"dispatched"
]
}
]
}
}
}
@@ -81,8 +81,6 @@ fn main() {
dump!(HoverResult, "tool_hover_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(TypeParams, "tool_type_params");
dump!(TypeResult, "tool_type_result");
dump!(PressParams, "tool_press_params");
dump!(PressResult, "tool_press_result");
dump!(SelectParams, "tool_select_params");
-11
View File
@@ -74,8 +74,6 @@ pub enum Method {
ToolHover,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.type")]
ToolType,
#[serde(rename = "tool.press")]
ToolPress,
#[serde(rename = "tool.select")]
@@ -156,7 +154,6 @@ impl Method {
| Method::ToolReload
| Method::ToolClick
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect
| Method::ToolEvaluate
@@ -249,13 +246,6 @@ mod tests {
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
}
#[test]
fn type_method_round_trips() {
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
assert_eq!(method, Method::ToolType);
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
}
#[test]
fn cancel_params_and_result_round_trip() {
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
@@ -294,7 +284,6 @@ mod tests {
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolType.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
assert!(Method::ToolEvaluate.is_mutating());
+2 -100
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
//! `tool.press`, `tool.select`).
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
//! `tool.select`).
//!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector
@@ -188,60 +188,6 @@ pub struct FillResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// type
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TypeParams {
pub session_id: String,
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
/// does not require the focused DOM editable to retain this text.
pub text: String,
/// Target to focus before dispatching text. Exactly one targeting mode
/// must be selected: `ref` / `selector`, or `focused=true`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Explicitly dispatch to the unique focused text-entry target instead
/// of focusing a ref or selector.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub focused: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum TypeVerification {
/// Chromium accepted the complete keyboard/IME dispatch transaction.
/// The caller must observe the application output to verify business state.
Dispatched,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TypeResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// UTF-16 code-unit length of text accepted for dispatch.
pub text_length: u32,
pub verification: TypeVerification,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// press
// ---------------------------------------------------------------------------
@@ -405,50 +351,6 @@ mod tests {
assert_eq!(v, json!("meta"));
}
#[test]
fn type_payloads_distinguish_dispatch_from_value_verification() {
let params = TypeParams {
session_id: "abcd".into(),
text: "测试".into(),
ref_: Some("@e3".into()),
selector: None,
focused: false,
tab_id: Some(4),
timeout_ms: Some(5_000),
};
let encoded = serde_json::to_value(&params).unwrap();
assert_eq!(encoded["ref"], json!("@e3"));
assert_eq!(encoded["text"], json!("测试"));
assert!(encoded.get("focused").is_none());
let focused = TypeParams {
session_id: "abcd".into(),
text: "hello".into(),
ref_: None,
selector: None,
focused: true,
tab_id: None,
timeout_ms: None,
};
assert_eq!(
serde_json::to_value(focused).unwrap()["focused"],
json!(true)
);
let result = TypeResult {
tab_id: 4,
used_ref: Some("e3".into()),
used_selector: None,
text_length: 2,
verification: TypeVerification::Dispatched,
dialogs: vec![],
};
assert_eq!(
serde_json::to_value(result).unwrap()["verification"],
json!("dispatched")
);
}
#[test]
fn press_result_round_trips() {
let r = PressResult {
+2 -2
View File
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SnapshotResult {
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
/// `tool.type` targets.
/// agent can copy them into subsequent `tool.click` / `tool.fill`
/// selectors.
pub text: String,
/// Number of `@e<N>` refs registered for this session by this
/// snapshot. Equivalent to the size of the new ref-store map.
-259
View File
@@ -1,259 +0,0 @@
# Browser capability evaluation corpus
This directory is a deterministic, local browser evaluation environment for BrowserSkill, DSH,
and other command-line agents. Cases, fixture pages, direct `bsk` workflows, and assertions are
data-driven and discovered automatically. Adding a case does not require editing a central switch.
[中文说明](README.zh-CN.md)
## Design goals
- Agent-neutral: prompts describe observable page goals so the same cases can run through different
agents and adapters.
- Deterministic: pages run locally and use stable markers and run-scoped events.
- Extensible: manifests, fixture modules, and workflows are auto-discovered.
- Reproducible: generated cases record a seed and derive every DOM variation from it.
- Honest verification: page-observable results, response markers, and adapter evidence are reported
separately. Missing adapter evidence is `unverified`, never silently treated as passed.
- Privacy-safe: user badcases must be reduced to synthetic DOM and anonymized data before commit.
The corpus currently has six stable `core` cases and one seeded `matrix` case. The operation
inventory contains 28 browser operations; that count is capability coverage, not the number of
cases. Cases can grow independently and many cases may cover the same operation under different
page conditions.
## Quick start
These commands validate manifests and run unit/integration tests without launching a browser:
```sh
pnpm eval:browser validate
pnpm eval:browser list
pnpm eval:browser coverage
pnpm eval:browser:test
```
Build the repository CLI and exercise the real CLI, daemon, extension, Agent Window, and pages:
```sh
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
Every smoke case starts a fresh browser session and stops it in a `finally` cleanup path. Reports
and screenshots are written to the ignored `evals/browser/results/` directory.
`pnpm run eval:browser -- coverage` is also supported. The CLI intentionally accepts pnpm's leading
`--` separator.
## Suites and selection
| Suite | Intended use | Default execution |
| --- | --- | --- |
| `core` | Small stable acceptance set for everyday smoke and CI | Yes |
| `matrix` | Seeded DOM, timing, and layout variations | Explicit |
| `regression` | Minimized user badcases tied to an issue or fix | Explicit |
| `stress` | Higher volume or slower boundary testing | Explicit |
| `manual` | Human-dependent capabilities that cannot be deterministic | Explicit |
`smoke` and `run-agent` default to `core`, so adding a stress or regression case cannot silently
slow down existing CI. `list`, `coverage`, and `validate` inspect the full corpus by default.
```sh
pnpm eval:browser list --suite core
pnpm eval:browser list --tag form
pnpm eval:browser smoke --case form-controls
pnpm eval:browser smoke --case form-controls,generated-form
pnpm eval:browser smoke --case all
```
Filters from different dimensions are combined. Repeated/comma-separated suites or case ids are
alternatives; every requested tag must be present. `--task` remains a backward-compatible alias for
`--case`.
## Directory and discovery model
```text
evals/browser/
cases/
core/*.case.json
matrix/*.case.json
regression/<case-id>/
<case-id>.case.json
<case-id>.fixture.mjs
README.md
fixtures/pages/*.fixture.mjs
schemas/case.schema.json
lib/
tests/
```
- Any `cases/**/*.case.json` file is loaded as a case manifest.
- Any `fixtures/**/*.fixture.mjs` or `cases/**/*.fixture.mjs` file is registered as a fixture.
- Manifests are ordered by suite, optional numeric `order`, then id.
- `validate` rejects duplicate case ids, duplicate routes, missing fixture routes, unknown
operations/actions, invalid workflow variables, and adapter assertions with no evidence producer.
## Case manifest
A case owns all information needed by both an agent evaluation and direct CLI smoke:
```json
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "example-case",
"title": "Example browser behavior",
"order": 10,
"suite": "core",
"tags": ["form", "keyboard"],
"seed": 17,
"fixture": { "startPath": "/example" },
"prompts": { "en": "Open {url} ...", "zh-CN": "打开 {url}……" },
"coverage": ["page.navigate", "interact.press"],
"assertions": { "site": [], "response": [], "adapter": [] },
"smoke": { "steps": [{ "action": "navigate" }] }
}
```
Prompt placeholders are `{url}`, `{baseUrl}`, `{runId}`, and `{seed}`. Agent adapter arguments also
support `{prompt}`, `{caseId}` / `{taskId}`, and `{variant}`.
Assertions have three independent sources:
- `site`: events emitted by the local page, such as submitted values or a history navigation.
- `response`: stable text that must appear in the agent or CLI output.
- `adapter`: facts outside page JavaScript, such as a non-empty screenshot or confirmed session
cleanup.
The declarative smoke workflow supports navigation/history, waits, observe/snapshot/HTML,
screenshot/console/network, click/hover/fill/select/press, tabs, resize/emulation, and the manual
ownership/help operations. A step may `saveAs` its JSON result and later reference values such as
`{child.tab_id}`. A step may set `evidence` for an adapter assertion. See existing manifests for
complete examples.
## Adding a normal case
1. Add a `*.fixture.mjs` module exporting `{ id, routes, render(context) }`. It can live in shared
`fixtures/pages/` or beside a self-contained regression case.
2. Add a `*.case.json` manifest with prompts, operation coverage, assertions, and smoke steps.
3. Make all page events run-scoped using the shared `browserEval.send(...)` client helper.
4. Run `pnpm eval:browser validate` and `pnpm eval:browser:test`.
5. Run the new case against the real local build with `smoke --case <id>`.
Prefer one behavior or failure mechanism per case. Stable synthetic markers make failures easier to
diagnose than assertions against incidental prose.
## Turning a user badcase into a regression
Create a self-contained starting point:
```sh
pnpm eval:browser scaffold reported-timeout \
--title "Navigation settles after a late frame" \
--source "issue-123"
```
The command creates a manifest, fixture, and reproduction notes under
`cases/regression/reported-timeout/`. Then reduce the report to the smallest relevant mechanism:
- Keep DOM structure, timing, redirects, frames, or browser state that triggers the bug.
- Replace names, account data, URLs, tokens, screenshots, and page copy with synthetic values.
- Do not commit HAR files, cookies, credentials, production HTML, or proprietary assets.
- Record the issue/PR reference, seed when relevant, expected result, and the fixing PR.
- Confirm the new case fails on the known-bad build and passes on the fixed build when practical.
## Seeded matrix cases
`generated-form` varies label association, nesting depth, hydration delay, decoy controls, field
order, and element ids while keeping the task and oracle stable. Inspect a seed without a browser:
```sh
pnpm eval:browser generate --seed user-badcase-42
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
pnpm eval:browser smoke --case generated-form --seed 4,7,14
```
Repeated/comma-separated seeds run as independent results. Reports store every normalized seed, so
a failure can be replayed exactly and promoted into a named regression case if it represents a
distinct bug.
## Run any command-line agent
Copy `agents.example.json` to the ignored `agents.local.json` and adapt its command. Arguments are
passed directly to the executable without a shell.
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-local \
--suite core \
--repeat 3
```
An adapter may define `command`, `args`, `cwd`, `env`, `timeoutMs`, `variant`, free-form `metadata`,
and optional `metrics.errorPattern` / `metrics.toolCallPattern` regular expressions.
To compare 28 granular tools with domain tools, create two isolated adapter entries and run both
against the same selected cases and seeds:
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-granular,dsh-domain \
--suite core \
--repeat 10
```
Keep browser runs sequential to avoid session contention. Use fresh profiles and alternate adapter
order in larger experiments to reduce warm-cache bias.
## Reading results
```sh
pnpm eval:browser summarize evals/browser/results/*.json
```
The summary groups by adapter and variant and reports pass rate, fully verified rate, process
failures, matched error count, average tool calls, and average duration. The raw JSON retains case,
suite, tags, seed, prompt, command output, page event count, and individual oracle checks.
A healthy run has:
- no execution timeout or non-zero exit;
- no `executionError` in direct smoke;
- `verification.status` equal to `passed` (or intentionally `passed-with-unverified` for an agent
adapter that cannot expose external evidence);
- every session cleanup assertion verified;
- no unexpected increase in errors, tool calls, or duration versus the chosen baseline.
## Development baseline
Use these lanes as commit and PR gates:
| Change | Required checks |
| --- | --- |
| Any case, fixture, oracle, or harness change | `pnpm eval:browser:check` |
| CLI, daemon, extension, DSH plugin, or browser-operation change | The check above, `cargo build -p bsk`, then real-browser `smoke --suite core` |
| DOM discovery, form interaction, timing, or waiting change | The checks above, then `smoke --suite matrix --seed 4,7,14` |
| A bug fix with a known reproduction | Add a `regression` case and run that case on the fixed build |
Before committing this harness, the complete acceptance sequence is:
```sh
pnpm eval:browser:check
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
All commands must exit zero. The smoke summary must have 100% pass and fully verified rates, zero
execution failures/errors, and `bsk session list --json` must be `[]` afterward. Do not weaken an
oracle or replace a stable marker merely to make a regression pass; behavior changes should update
the case and expected baseline in the same reviewed change.
The direct smoke lane covers 25 of 28 operations. `tabs.borrow`, `tabs.return`, and
`assist.request-help` stay manual because they require a real user tab or human interaction and
would make the default suite destructive or non-deterministic.
-197
View File
@@ -1,197 +0,0 @@
# 浏览器能力测试语料库
这里是一套供 BrowserSkill、DSH 以及其他命令行 Agent 共用的本地确定性浏览器测试环境。
case、测试页面、`bsk` 直连流程和断言都会自动发现;新增 case 不需要修改中央 `switch`。
[English](README.md)
## 核心原则
- 与 Agent 无关:prompt 只描述可观察的页面目标,同一套 case 可以被不同 Agent 和
adapter 使用。
- 本地且确定:页面只依赖本地服务,使用稳定 marker 和按 run 隔离的事件。
- 可扩展:manifest、fixture 和 workflow 都从目录自动发现。
- 可复现:生成式页面的所有变化由 seed 决定,报告会保留 seed。
- 如实验证:页面事件、回答文本和 adapter 外部证据分别统计;拿不到的证据标为
`unverified`,不会冒充通过。
- 隐私安全:用户 badcase 必须先缩减为合成 DOM 和匿名数据,再进入仓库。
当前有 6 个稳定的 `core` case 和 1 个 seed 驱动的 `matrix` case。28 表示浏览器操作能力
清单,并不是 case 数量;同一种操作可以在不同页面结构、时序和浏览器状态下由多个 case
重复覆盖。
## 快速使用
不启动浏览器的校验:
```sh
pnpm eval:browser validate
pnpm eval:browser list
pnpm eval:browser coverage
pnpm eval:browser:test
```
使用仓库当前代码编译 `bsk`,再跑真实 CLI、daemon、扩展、小窗和页面链路:
```sh
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
每个 smoke case 都会创建独立 session,并在 `finally` 中关闭。JSON 报告和截图输出到已被
Git 忽略的 `evals/browser/results/`。
也可以写成 `pnpm run eval:browser -- coverage`;CLI 已兼容 pnpm 传入的前导 `--`。
## Suite 与筛选
| Suite | 用途 | 默认执行 |
| --- | --- | --- |
| `core` | 日常 smoke 和 CI 使用的小而稳定的验收集 | 是 |
| `matrix` | seed 驱动的 DOM、时序、布局组合 | 显式选择 |
| `regression` | 从用户 badcase 缩减并与 issue/修复关联的回归 | 显式选择 |
| `stress` | 更高次数或更慢的边界测试 | 显式选择 |
| `manual` | 必须依赖真人或真实用户标签页的能力 | 显式选择 |
`smoke` 和 `run-agent` 不带筛选时只跑 `core`,所以以后新增 stress/badcase 不会突然拖慢
已有 CI。`list`、`coverage`、`validate` 默认检查全部语料。
```sh
pnpm eval:browser list --suite core
pnpm eval:browser list --tag form
pnpm eval:browser smoke --case form-controls
pnpm eval:browser smoke --case form-controls,generated-form
pnpm eval:browser smoke --case all
```
不同维度的筛选会叠加;逗号分隔的 suite/case 是“或”,多个 tag 必须全部命中。旧参数
`--task` 仍作为 `--case` 的兼容别名。
## 自动发现结构
```text
evals/browser/
cases/
core/*.case.json
matrix/*.case.json
regression/<case-id>/
<case-id>.case.json
<case-id>.fixture.mjs
README.md
fixtures/pages/*.fixture.mjs
schemas/case.schema.json
lib/
tests/
```
- `cases/**/*.case.json` 都会作为 case manifest 加载。
- `fixtures/**/*.fixture.mjs` 和 `cases/**/*.fixture.mjs` 都会注册为页面 fixture。
- 顺序按 suite、可选的数字 `order`、id 排列。
- `validate` 会阻止重复 case id、重复路由、缺失 fixture、未知操作/action、无效 workflow
变量,以及没有任何步骤产出的 adapter evidence。
一个 manifest 同时描述:中英文 prompt、能力 coverage、页面事件/回答/adapter 三类断言,以及
`bsk` 直连 smoke 的声明式步骤。prompt 可用 `{url}`、`{baseUrl}`、`{runId}`、`{seed}`;
workflow 可以用 `saveAs` 保存 JSON 结果,并用 `{child.tab_id}` 之类的变量引用后续步骤。
完整写法可直接参考 `cases/core/` 和 `cases/matrix/`。
## 新增普通 case
1. 新增导出 `{ id, routes, render(context) }` 的 `*.fixture.mjs`,可放共享 fixture 目录,也可
和 regression case 放在一起。
2. 新增 `*.case.json`,写好 prompt、operation coverage、断言和 smoke steps。
3. 页面使用共享的 `browserEval.send(...)` 上报按 run 隔离的事件。
4. 执行 `pnpm eval:browser validate` 和 `pnpm eval:browser:test`。
5. 用 `smoke --case <id>` 验证真实本地编译链路。
建议一个 case 只表达一种行为或故障机制,并使用稳定的合成 marker,避免断言偶然文案。
## 把用户 badcase 固化成回归
先生成一个自包含模板:
```sh
pnpm eval:browser scaffold reported-timeout \
--title "Navigation settles after a late frame" \
--source "issue-123"
```
它会在 `cases/regression/reported-timeout/` 下生成 manifest、fixture 和复现说明。随后只保留
真正触发问题的 DOM 结构、时序、跳转、frame 或浏览器状态,并遵守:
- 所有姓名、账号、网址、token、截图和文案都替换为合成值。
- 不提交 HAR、cookie、凭据、生产 HTML 或私有素材。
- 记录 issue/PR 来源、相关 seed、期望行为和后续修复 PR。
- 条件允许时,证明新 case 在旧版本失败、修复版本通过。
## Seed 矩阵
`generated-form` 会确定性改变 label 关联方式、DOM 嵌套、hydration 延迟、干扰控件、字段
顺序和 element id,但任务和 oracle 保持不变:
```sh
pnpm eval:browser generate --seed user-badcase-42
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
pnpm eval:browser smoke --case generated-form --seed 4,7,14
```
逗号分隔或重复的 seed 会生成相互独立的结果。任何失败都能按报告中的规范化 seed 精确重放;
若它代表独立 bug,再提升为命名 regression case。
## 测任意 Agent 与比较 28/6 工具
把 `agents.example.json` 复制为已被 Git 忽略的 `agents.local.json`,分别配置两个隔离的 Agent
profile/build。参数以数组直接传给可执行程序,不经过 shell。
```sh
pnpm eval:browser run-agent \
--config evals/browser/agents.local.json \
--agent dsh-granular,dsh-domain \
--suite core \
--repeat 10
```
浏览器任务应串行运行,避免 session 竞争;更大实验中使用干净 profile 并轮换 adapter 顺序,
降低缓存偏差。
## 如何看结果
```sh
pnpm eval:browser summarize evals/browser/results/*.json
```
汇总按 adapter/variant 展示通过率、完全验证率、进程失败、错误数、平均工具调用数和耗时。
原始 JSON 会保留 case、suite、tag、seed、prompt、命令输出、页面事件数和每条 oracle 结果。
健康运行应满足:没有 timeout/非零退出,direct smoke 没有 `executionError`,验证状态为
`passed`(无法暴露外部证据的 Agent adapter 可明确为 `passed-with-unverified`),session cleanup
通过,并且错误数、调用数和耗时没有相对基线异常上涨。
## 开发基准线
后续开发统一按下面的门禁执行:
| 改动范围 | 必跑检查 |
| --- | --- |
| 任意 case、fixture、oracle 或 harness 改动 | `pnpm eval:browser:check` |
| CLI、daemon、扩展、DSH 插件或浏览器操作改动 | 上述检查、`cargo build -p bsk`、真实浏览器 `smoke --suite core` |
| DOM 发现、表单交互、时序或等待逻辑改动 | 上述检查,再跑 `smoke --suite matrix --seed 4,7,14` |
| 修复已有明确复现的 bug | 新增 `regression` case,并在修复版本单独运行该 case |
提交这套 harness 前,完整验收命令为:
```sh
pnpm eval:browser:check
cargo build -p bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
```
所有命令必须以 0 退出;smoke 汇总必须达到 100% pass、100% fully verified、0 execution
failure/error,结束后 `bsk session list --json` 必须为 `[]`。不能为了让回归通过而削弱 oracle
或替换稳定 marker;如果产品行为确实改变,case 与预期基线必须在同一个受审查改动中更新。
直连 smoke 自动覆盖 28 项中的 25 项。`tabs.borrow`、`tabs.return`、`assist.request-help` 需要
真实用户标签页或真人交互,因此保留在 manual lane,避免默认测试具有破坏性或不确定性。
-17
View File
@@ -1,17 +0,0 @@
{
"agents": {
"dsh-local": {
"variant": "granular-28",
"command": "dsh",
"args": ["--profile", "headless", "{prompt}"],
"cwd": "../..",
"timeoutMs": 300000,
"metadata": {
"toolCount": 28
},
"metrics": {
"toolCallPattern": "browser[_-][a-z_-]+"
}
}
}
}
-9
View File
@@ -1,9 +0,0 @@
# Browser evaluation cases
Cases are discovered recursively from `*.case.json`; colocated `*.fixture.mjs` files are discovered
as fixtures. Use `core` for small deterministic acceptance checks, `matrix` for seeded variations,
and `regression/<case-id>/` for privacy-safe minimized user badcases.
Run `pnpm eval:browser validate` after every case change. See the parent
[`README.md`](../README.md) for the manifest contract, workflow actions, scaffolding, and acceptance
criteria.
@@ -1,48 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "diagnostics",
"title": "Console and network inspection",
"order": 50,
"suite": "core",
"tags": ["console", "network", "diagnostics"],
"fixture": { "startPath": "/diagnostics" },
"prompts": {
"en": "Open {url}. Use browser diagnostics to inspect the page console and recent network activity. Report the console marker and the token returned by the ping request, then close the browser session.",
"zh-CN": "打开 {url}。使用浏览器诊断能力检查页面 console 和最近的 network 活动,报告 console marker 以及 ping 请求返回的 token,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"inspect.console",
"inspect.network"
],
"assertions": {
"site": [
{
"label": "diagnostic ping completed",
"type": "network.ping",
"minCount": 1
}
],
"response": [
{ "label": "reported console marker", "includes": "CONSOLE-61" },
{ "label": "reported network token", "includes": "NETWORK-73" }
],
"adapter": [
{ "label": "console inspection was executed", "key": "consoleRead" },
{ "label": "network inspection was executed", "key": "networkRead" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait", "duration": "300ms" },
{ "action": "console", "evidence": "consoleRead" },
{ "action": "network", "evidence": "networkRead" },
{ "action": "observe" }
]
}
}
@@ -1,82 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "form-controls",
"title": "Form, select, keyboard, and screenshot",
"order": 20,
"suite": "core",
"tags": ["form", "keyboard", "screenshot", "select"],
"fixture": { "startPath": "/form" },
"prompts": {
"en": "Open {url}. Fill Text input with \"agent-parity\" and Textarea with \"BrowserSkill works\". Select the visible option \"Two\" from Dropdown. Capture a screenshot of the form, focus Submit, and press Enter to submit. Confirm that the result says \"Received!\", then close the browser session.",
"zh-CN": "打开 {url}。把 Text input 填成“agent-parity”,Textarea 填成“BrowserSkill works”,Dropdown 选择可见选项“Two”。截取表单截图,然后聚焦 Submit 并按 Enter 提交。确认结果页出现“Received!”,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.fill",
"interact.select",
"interact.press",
"inspect.screenshot"
],
"assertions": {
"site": [
{
"label": "Enter was pressed on Submit",
"type": "form.enter_pressed",
"minCount": 1
},
{
"label": "form values were submitted",
"type": "form.submitted",
"where": {
"data.text": "agent-parity",
"data.notes": "BrowserSkill works",
"data.choice": "two"
},
"minCount": 1
},
{
"label": "result page was displayed",
"type": "page.shown",
"where": { "data.path": "/result" },
"minCount": 1
}
],
"response": [{ "label": "confirmed result", "includes": "Received!" }],
"adapter": [
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{
"action": "fill",
"selector": "#text-input",
"value": "agent-parity"
},
{
"action": "fill",
"selector": "#notes",
"value": "BrowserSkill works"
},
{ "action": "select", "selector": "#choice", "values": ["two"] },
{
"action": "screenshot",
"artifactName": "form.png",
"evidence": "screenshotCreated"
},
{ "action": "press", "key": "Enter", "selector": "#submit" },
{
"action": "wait-site-event",
"type": "page.shown",
"where": { "path": "/result" }
},
{ "action": "wait", "duration": "200ms" },
{ "action": "observe" }
]
}
}
@@ -1,59 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "hover-inspect",
"title": "Hover, local HTML, and screenshot",
"order": 30,
"suite": "core",
"tags": ["hover", "html", "screenshot"],
"fixture": { "startPath": "/hover-inspect" },
"prompts": {
"en": "Open {url}. Hover over Account, then choose Profile beta from the revealed menu. Inspect the local HTML around the Inspection card and capture a screenshot. Report both marker codes you find, then close the browser session.",
"zh-CN": "打开 {url}。把鼠标悬停在“Account”上,然后点击展开菜单中的“Profile beta”。检查 Inspection card 附近的局部 HTML,并截取页面截图。报告找到的两个标记代码,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.hover",
"interact.click",
"inspect.html",
"inspect.screenshot"
],
"assertions": {
"site": [
{
"label": "Account menu was hovered",
"type": "hover.menu_opened",
"minCount": 1
},
{
"label": "Profile beta was selected",
"type": "hover.profile_clicked",
"minCount": 1
}
],
"response": [
{ "label": "reported hover marker", "includes": "HOVER-17" },
{ "label": "reported HTML marker", "includes": "INSPECT-27" }
],
"adapter": [
{ "label": "HTML inspection was executed", "key": "htmlInspected" },
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "hover", "selector": "#account-menu" },
{ "action": "click", "selector": "#profile-link" },
{ "action": "html", "evidence": "htmlInspected" },
{
"action": "screenshot",
"artifactName": "hover-inspect.png",
"evidence": "screenshotCreated"
}
]
}
}
@@ -1,43 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "mobile-emulation",
"title": "Mobile emulation",
"order": 60,
"suite": "core",
"tags": ["emulation", "mobile", "viewport"],
"fixture": { "startPath": "/responsive" },
"prompts": {
"en": "Open {url}, emulate an iPhone 14, and report the viewport marker shown on the page. Close the browser session when finished.",
"zh-CN": "打开 {url},模拟 iPhone 14,并报告页面显示的 viewport marker。完成后关闭浏览器会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "assist.emulate"],
"assertions": {
"site": [
{
"label": "iPhone-sized viewport was observed",
"type": "viewport.sample",
"where": { "data.width": 390, "data.height": 844 },
"minCount": 1
}
],
"response": [{ "label": "reported viewport marker", "includes": "VIEWPORT-84" }],
"adapter": [
{ "label": "device emulation was executed", "key": "emulatedDevice" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "resize", "width": 900, "height": 700 },
{
"action": "emulate",
"device": "iphone-14",
"evidence": "emulatedDevice"
},
{ "action": "wait", "duration": "300ms" },
{ "action": "html" }
]
}
}
@@ -1,67 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "navigation-history",
"title": "Navigation, history, and reload",
"order": 10,
"suite": "core",
"tags": ["navigation", "history", "reload"],
"fixture": { "startPath": "/navigation/start" },
"prompts": {
"en": "Open {url}. Read the main heading, follow the Details link, and report the code shown there. Then go back, go forward, reload the page, and close the browser session. Use only the browser capability for page operations.",
"zh-CN": "打开 {url}。读取页面主标题,点击“详情”,并报告详情页上的代码。然后依次后退、前进、刷新页面,最后关闭浏览器会话。页面操作只使用浏览器能力。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"page.back",
"page.forward",
"page.reload",
"interact.click"
],
"assertions": {
"site": [
{
"label": "details link was clicked",
"type": "navigation.detail.clicked",
"minCount": 1
},
{
"label": "back returned to the start page",
"type": "page.shown",
"where": { "data.path": "/navigation/start" },
"minCount": 2
},
{
"label": "forward returned to the detail page",
"type": "page.shown",
"where": { "data.path": "/navigation/detail" },
"minCount": 2
},
{
"label": "detail page was reloaded",
"type": "page.shown",
"where": {
"data.path": "/navigation/detail",
"data.navigationType": "reload"
},
"minCount": 1
}
],
"response": [{ "label": "reported detail code", "includes": "NAV-42" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "observe" },
{ "action": "snapshot" },
{ "action": "click", "selector": "#details-link" },
{ "action": "html" },
{ "action": "back" },
{ "action": "forward" },
{ "action": "reload" }
]
}
}
-71
View File
@@ -1,71 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "tabs",
"title": "Create, select, and close a tab",
"order": 40,
"suite": "core",
"tags": ["tabs", "ownership", "lifecycle"],
"fixture": { "startPath": "/tabs" },
"prompts": {
"en": "Open {url}. Create a new browser tab at {baseUrl}/tabs/child?run={runId}, read its token, click \"Mark child visited\", close the child tab, return to the original tab, and close the browser session.",
"zh-CN": "打开 {url}。新建一个浏览器标签页并访问 {baseUrl}/tabs/child?run={runId},读取其中的 token,点击“Mark child visited”,关闭子标签页,回到原标签页,最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"interact.click",
"tabs.create",
"tabs.select",
"tabs.close"
],
"assertions": {
"site": [
{
"label": "child tab page was displayed",
"type": "page.shown",
"where": { "data.path": "/tabs/child" },
"minCount": 1
},
{
"label": "child tab action ran",
"type": "tabs.child_marked",
"minCount": 1
}
],
"response": [{ "label": "reported child token", "includes": "TAB-29" }],
"adapter": [
{ "label": "child tab was closed", "key": "childTabClosed" },
{ "label": "browser session was closed", "key": "sessionStopped" }
]
},
"smoke": {
"steps": [
{ "action": "navigate", "saveAs": "original" },
{ "action": "tab-list" },
{
"action": "tab-create",
"url": "{baseUrl}/tabs/child?run={runId}",
"saveAs": "child"
},
{
"action": "navigate",
"url": "{baseUrl}/tabs/child?run={runId}",
"tabId": "{child.tab_id}"
},
{
"action": "click",
"selector": "#mark-child",
"tabId": "{child.tab_id}"
},
{ "action": "html", "tabId": "{child.tab_id}" },
{
"action": "tab-close",
"tabId": "{child.tab_id}",
"evidence": "childTabClosed"
},
{ "action": "tab-select", "tabId": "{original.tab_id}" }
]
}
}
-5
View File
@@ -1,5 +0,0 @@
# Seeded matrix cases
Matrix cases vary page structure or timing deterministically from a recorded seed. The task,
assertions, and success marker remain stable so a failed seed can be replayed exactly. A seed that
exposes a distinct product bug should be minimized and promoted to a named regression case.
@@ -1,77 +0,0 @@
{
"$schema": "../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "generated-form",
"title": "Seeded form structure and hydration variants",
"order": 10,
"suite": "matrix",
"tags": ["dynamic-dom", "form", "hydration", "seeded"],
"seed": 20260827,
"fixture": { "startPath": "/matrix/generated-form" },
"prompts": {
"en": "Open {url}. Wait for the seeded form to finish hydrating. Fill Text input with \"seeded-agent\", select the visible option \"Two\", submit with Enter, and report the result marker and seed. Close the browser session.",
"zh-CN": "打开 {url}。等待 seeded form 完成 hydration,把 Text input 填成“seeded-agent”,选择可见选项“Two”,用 Enter 提交,并报告结果 marker 和 seed。最后关闭浏览器会话。"
},
"coverage": [
"session.start",
"session.stop",
"page.navigate",
"page.wait",
"inspect.observe",
"interact.fill",
"interact.select",
"interact.press"
],
"assertions": {
"site": [
{ "label": "generated form hydrated", "type": "matrix.ready", "minCount": 1 },
{
"label": "Enter was pressed on generated form",
"type": "matrix.enter_pressed",
"minCount": 1
},
{
"label": "generated values were submitted",
"type": "matrix.submitted",
"where": { "data.text": "seeded-agent", "data.choice": "two" },
"minCount": 1
},
{
"label": "matrix result page was displayed",
"type": "page.shown",
"where": { "data.path": "/matrix/generated-result" },
"minCount": 1
}
],
"response": [
{ "label": "reported matrix result", "includes": "MATRIX-OK" },
{ "label": "reported selected seed", "includes": "SEED-" }
],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{
"action": "wait-site-event",
"type": "matrix.ready",
"timeoutMs": 5000
},
{ "action": "observe" },
{ "action": "fill", "selector": "[name=text]", "value": "seeded-agent" },
{ "action": "select", "selector": "[name=choice]", "values": ["two"] },
{
"action": "press",
"key": "Enter",
"selector": "button[type=submit]"
},
{
"action": "wait-site-event",
"type": "page.shown",
"where": { "path": "/matrix/generated-result" }
},
{ "action": "wait", "duration": "200ms" },
{ "action": "observe" }
]
}
}
-12
View File
@@ -1,12 +0,0 @@
# Regression cases
Each user-reported badcase should live in its own directory with a manifest, synthetic fixture, and
README describing the symptom, minimal reproduction, expected behavior, source reference, and fix.
Never commit credentials, cookies, HAR files, production HTML, private screenshots, or proprietary
assets.
Create a starting point with:
```sh
pnpm eval:browser scaffold <case-id> --title "..." --source "issue-or-pr"
```
-467
View File
@@ -1,467 +0,0 @@
#!/usr/bin/env node
import { randomUUID } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { loadAgentConfig, runCommandTask } from "./lib/agent-runner.mjs";
import { runBskSmokeTask } from "./lib/bsk-runner.mjs";
import { buildOperationCoverage } from "./lib/coverage.mjs";
import { loadFixtureRegistry } from "./lib/fixture-registry.mjs";
import { verifyTask } from "./lib/oracle.mjs";
import { resultsDirectory } from "./lib/paths.mjs";
import { repositorySummary, validateRepositoryCases } from "./lib/repository-validation.mjs";
import { scaffoldCase } from "./lib/scaffold-case.mjs";
import { describeSeed, normalizeSeed } from "./lib/seeded-generator.mjs";
import { createEvalServer } from "./lib/server.mjs";
import { printSummary, summarizeReports } from "./lib/summary.mjs";
import { allTasks, getTask, listTasks, renderPrompt } from "./lib/tasks.mjs";
const booleanOptions = new Set(["help", "json"]);
function parseArguments(argv) {
const normalized = argv[0] === "--" ? argv.slice(1) : argv;
let [command, ...rest] = normalized;
const options = new Map();
const positionals = [];
if (command === "--help" || command === "-h") {
command = undefined;
options.set("help", [true]);
}
for (let index = 0; index < rest.length; index += 1) {
const argument = rest[index];
if (!argument.startsWith("--")) {
positionals.push(argument);
continue;
}
const equals = argument.indexOf("=");
const name = argument.slice(2, equals === -1 ? undefined : equals);
let value = equals === -1 ? undefined : argument.slice(equals + 1);
if (value === undefined && !booleanOptions.has(name)) {
const candidate = rest[index + 1];
if (candidate === undefined || candidate.startsWith("--")) {
throw new Error(`--${name} requires a value`);
}
value = candidate;
index += 1;
}
const values = options.get(name) ?? [];
values.push(value ?? true);
options.set(name, values);
}
return { command, options, positionals };
}
function option(parsed, name, fallback) {
return parsed.options.get(name)?.at(-1) ?? fallback;
}
function optionList(parsed, name) {
return (parsed.options.get(name) ?? [])
.flatMap((value) => String(value).split(","))
.map((value) => value.trim())
.filter(Boolean);
}
function numericOption(parsed, name, fallback) {
const value = Number(option(parsed, name, fallback));
if (!Number.isFinite(value) || value < 0) throw new Error(`--${name} must be non-negative`);
return value;
}
function seedOptions(parsed, fallback) {
const requested = optionList(parsed, "seed");
return requested.length > 0 ? unique(requested.map(normalizeSeed)) : [fallback];
}
function seedOption(parsed, fallback) {
const seeds = seedOptions(parsed, fallback);
if (seeds.length > 1) throw new Error("this command accepts only one --seed value");
return seeds[0];
}
function unique(values) {
return [...new Set(values)];
}
function selectCases(cases, parsed, { defaultSuite } = {}) {
const requestedIds = unique([...optionList(parsed, "case"), ...optionList(parsed, "task")]);
const requestedSuites = unique(optionList(parsed, "suite"));
const requestedTags = unique(optionList(parsed, "tag"));
const hasSelector =
requestedIds.length > 0 || requestedSuites.length > 0 || requestedTags.length > 0;
if (requestedIds.includes("all") && requestedIds.length > 1) {
throw new Error("--case all cannot be combined with named cases");
}
if (requestedSuites.includes("all") && requestedSuites.length > 1) {
throw new Error("--suite all cannot be combined with named suites");
}
const caseIds = requestedIds.includes("all") ? [] : requestedIds;
const suites = requestedSuites.includes("all")
? []
: requestedSuites.length > 0
? requestedSuites
: !hasSelector && defaultSuite
? [defaultSuite]
: [];
const knownIds = new Set(cases.map(({ id }) => id));
const missingIds = caseIds.filter((id) => !knownIds.has(id));
if (missingIds.length > 0) throw new Error(`unknown case(s): ${missingIds.join(", ")}`);
const knownSuites = new Set(cases.map(({ suite }) => suite));
const missingSuites = suites.filter((suite) => !knownSuites.has(suite));
if (missingSuites.length > 0) throw new Error(`unknown suite(s): ${missingSuites.join(", ")}`);
const knownTags = new Set(cases.flatMap(({ tags }) => tags));
const missingTags = requestedTags.filter((tag) => !knownTags.has(tag));
if (missingTags.length > 0) throw new Error(`unknown tag(s): ${missingTags.join(", ")}`);
const selected = cases.filter(
(testCase) =>
(caseIds.length === 0 || caseIds.includes(testCase.id)) &&
(suites.length === 0 || suites.includes(testCase.suite)) &&
requestedTags.every((tag) => testCase.tags.includes(tag)),
);
if (selected.length === 0) throw new Error("case selectors matched no cases");
return selected;
}
function reportFilename(mode) {
return `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${mode}.json`;
}
async function saveReport(report, outputDirectory) {
await mkdir(outputDirectory, { recursive: true });
const path = join(outputDirectory, reportFilename(report.mode));
await writeFile(path, `${JSON.stringify(report, null, 2)}\n`);
return path;
}
async function loadValidatedCorpus() {
const [cases, fixtureRegistry] = await Promise.all([allTasks(), loadFixtureRegistry()]);
const errors = validateRepositoryCases(cases, fixtureRegistry);
if (errors.length > 0) throw new Error(`browser eval validation failed:\n${errors.join("\n")}`);
return { cases, fixtureRegistry };
}
function selectionMetadata(cases) {
return {
cases: cases.map(({ id }) => id),
suites: unique(cases.map(({ suite }) => suite)),
tags: unique(cases.flatMap(({ tags }) => tags)).sort(),
};
}
function usage() {
console.log(`BrowserSkill local browser evaluation
Usage:
node evals/browser/cli.mjs validate [--json]
node evals/browser/cli.mjs list [--suite NAME] [--tag NAME] [--case ID] [--json]
node evals/browser/cli.mjs coverage [--suite NAME] [--tag NAME] [--case ID] [--json]
node evals/browser/cli.mjs serve [--host 127.0.0.1] [--port 4173]
node evals/browser/cli.mjs prompt <case> [--base-url URL] [--run-id ID] [--seed VALUE]
node evals/browser/cli.mjs verify <case> --base-url URL --run-id ID [--response TEXT]
node evals/browser/cli.mjs smoke [--bsk bsk] [--suite core] [--case ID] [--seed VALUE]
node evals/browser/cli.mjs run-agent --config FILE --agent NAME [--suite core] [--case ID]
node evals/browser/cli.mjs generate --seed VALUE [--json]
node evals/browser/cli.mjs scaffold <case-id> [--suite regression] [--title TEXT]
node evals/browser/cli.mjs summarize <report.json> [more reports...]
Selection:
--case ID[,ID] Select case ids; --task is a backward-compatible alias
--suite NAME[,NAME] Select suites
--tag NAME[,NAME] Require every selected tag
--case all Explicitly include every suite (run commands default to core)
--seed VALUE[,VALUE] Run one or more reproducible matrix seeds
run-agent options:
--agent NAME[,NAME] One or more entries from the config file
--variant NAME Override the report label (does not switch plugin code)
--locale en|zh-CN Prompt language (default zh-CN)
--repeat N Repeat every selected case (default 1)
--timeout MS Per-agent timeout (default 300000)
--out DIRECTORY Report/artifact directory
`);
}
async function commandValidate(parsed) {
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const summary = { ok: true, ...repositorySummary(cases, fixtureRegistry) };
if (option(parsed, "json", false)) console.log(JSON.stringify(summary, null, 2));
else {
console.log(`Validated ${summary.cases} cases and ${summary.fixtureRoutes} fixture routes.`);
console.log(
`Suites: ${Object.entries(summary.suites)
.map(([suite, count]) => `${suite}=${count}`)
.join(", ")}`,
);
}
}
async function commandList(parsed) {
const cases = selectCases(await allTasks(), parsed);
const tasks = listTasks(cases);
if (option(parsed, "json", false)) console.log(JSON.stringify(tasks, null, 2));
else {
for (const task of tasks) {
console.log(`${task.id}\t${task.suite}\t${task.tags.join(",")}\t${task.title}`);
}
}
}
async function commandCoverage(parsed) {
const cases = selectCases(await allTasks(), parsed);
const coverage = buildOperationCoverage(cases);
if (option(parsed, "json", false)) {
console.log(JSON.stringify(coverage, null, 2));
} else {
console.table(
coverage.map(({ operation, agentCases, smokeCases, directSmoke, note }) => ({
operation,
agent_cases: agentCases.join(", ") || "-",
smoke_cases: smokeCases.join(", ") || "-",
direct_smoke: directSmoke ? "yes" : "manual",
note: note ?? "",
})),
);
}
}
async function commandServe(parsed) {
const { fixtureRegistry } = await loadValidatedCorpus();
const server = createEvalServer({
host: option(parsed, "host", "127.0.0.1"),
port: numericOption(parsed, "port", 4173),
fixtureRegistry,
});
const info = await server.start();
console.log(`Browser eval server listening at ${info.baseUrl}`);
console.log("Press Ctrl-C to stop.");
await new Promise((resolveSignal) => {
process.once("SIGINT", resolveSignal);
process.once("SIGTERM", resolveSignal);
});
await server.stop();
}
async function commandPrompt(parsed) {
const cases = await allTasks();
const task = getTask(parsed.positionals[0], cases);
const prompt = renderPrompt(task, {
baseUrl: option(parsed, "base-url", "http://127.0.0.1:4173"),
runId: option(parsed, "run-id", randomUUID()),
locale: option(parsed, "locale", "zh-CN"),
seed: seedOption(parsed, task.seed),
});
console.log(prompt);
}
async function commandVerify(parsed) {
const task = getTask(parsed.positionals[0], await allTasks());
const baseUrl = option(parsed, "base-url");
const runId = option(parsed, "run-id");
if (!baseUrl || !runId) throw new Error("verify requires --base-url and --run-id");
let responseText = option(parsed, "response", "");
const responseFile = option(parsed, "response-file");
if (responseFile) responseText = await readFile(resolve(responseFile), "utf8");
const stateResponse = await fetch(`${baseUrl}/api/runs/${encodeURIComponent(runId)}`);
if (!stateResponse.ok) throw new Error(`oracle request failed with ${stateResponse.status}`);
const state = await stateResponse.json();
const verification = verifyTask(task, { events: state.events, responseText });
console.log(JSON.stringify(verification, null, 2));
if (verification.status === "failed") process.exitCode = 1;
}
async function commandRunAgent(parsed) {
const configPath = option(parsed, "config");
const agentNames = optionList(parsed, "agent");
if (!configPath || agentNames.length === 0) {
throw new Error("run-agent requires --config and at least one --agent");
}
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
const repeat = numericOption(parsed, "repeat", 1);
if (!Number.isInteger(repeat) || repeat < 1) throw new Error("--repeat must be at least 1");
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
const server = createEvalServer({ fixtureRegistry });
const serverInfo = await server.start();
const results = [];
const startedAt = new Date().toISOString();
try {
for (const agentName of agentNames) {
const { adapter, configDirectory } = await loadAgentConfig(configPath, agentName);
for (let iteration = 1; iteration <= repeat; iteration += 1) {
for (const task of selected) {
for (const seed of seedOptions(parsed, task.seed)) {
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
console.log(`[run] ${agentName} ${task.id}${seedLabel} (${iteration}/${repeat})`);
const result = await runCommandTask({
server,
serverInfo,
task,
adapter,
adapterName: agentName,
configDirectory,
iteration,
locale: option(parsed, "locale", "zh-CN"),
seed,
timeoutMs: numericOption(parsed, "timeout", 300_000),
variantOverride: option(parsed, "variant"),
});
results.push(result);
console.log(` ${result.verification.status} in ${result.execution.durationMs}ms`);
}
}
}
}
} finally {
await server.stop();
}
const report = {
schemaVersion: 1,
mode: "agent",
startedAt,
finishedAt: new Date().toISOString(),
selection: selectionMetadata(selected),
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
results,
};
const path = await saveReport(report, outputDirectory);
printSummary(summarizeReports([report]));
console.log(`Report: ${path}`);
if (
results.some(
(result) =>
result.verification.status === "failed" ||
result.execution.timedOut ||
result.execution.exitCode !== 0,
)
) {
process.exitCode = 1;
}
}
async function commandSmoke(parsed) {
const { cases, fixtureRegistry } = await loadValidatedCorpus();
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
const server = createEvalServer({ fixtureRegistry });
const serverInfo = await server.start();
const results = [];
const startedAt = new Date().toISOString();
try {
for (const task of selected) {
for (const seed of seedOptions(parsed, task.seed)) {
const runId = `bsk-${task.id}-${randomUUID().slice(0, 8)}`;
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
console.log(`[smoke] ${task.id}${seedLabel}`);
const result = await runBskSmokeTask({
bskCommand: option(parsed, "bsk", "bsk"),
outputDirectory,
server,
serverInfo,
task,
runId,
seed,
timeoutMs: numericOption(parsed, "timeout", 60_000),
});
results.push(result);
console.log(
` ${result.verification.status}${result.executionError ? `: ${result.executionError}` : ""}`,
);
}
}
} finally {
await server.stop();
}
const report = {
schemaVersion: 1,
mode: "bsk-smoke",
startedAt,
finishedAt: new Date().toISOString(),
selection: selectionMetadata(selected),
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
results,
};
const path = await saveReport(report, outputDirectory);
printSummary(summarizeReports([report]));
console.log(`Report: ${path}`);
if (results.some((result) => result.executionError || result.verification.status === "failed")) {
process.exitCode = 1;
}
}
async function commandGenerate(parsed) {
if (!parsed.options.has("seed")) throw new Error("generate requires --seed");
const dimensions = unique(optionList(parsed, "seed").map(normalizeSeed)).map(describeSeed);
const output = dimensions.length === 1 ? dimensions[0] : dimensions;
if (option(parsed, "json", false)) console.log(JSON.stringify(output, null, 2));
else console.table(dimensions);
}
async function commandScaffold(parsed) {
const id = parsed.positionals[0];
if (!id) throw new Error("scaffold requires a case id");
const result = await scaffoldCase({
id,
title: option(parsed, "title", id),
suite: option(parsed, "suite", "regression"),
source: option(parsed, "source"),
});
if (option(parsed, "json", false)) console.log(JSON.stringify(result, null, 2));
else {
console.log(`Created ${result.directory}`);
console.log(
"Edit the fixture, prompt, workflow, and assertions, then run eval:browser validate.",
);
}
}
async function commandSummarize(parsed) {
const paths = [...parsed.positionals, ...optionList(parsed, "input")];
if (paths.length === 0) throw new Error("summarize requires at least one report path");
const reports = await Promise.all(
paths.map(async (path) => JSON.parse(await readFile(resolve(path), "utf8"))),
);
const rows = summarizeReports(reports);
if (option(parsed, "json", false)) console.log(JSON.stringify(rows, null, 2));
else printSummary(rows);
}
async function main() {
const parsed = parseArguments(process.argv.slice(2));
if (!parsed.command || option(parsed, "help", false)) return usage();
switch (parsed.command) {
case "validate":
return commandValidate(parsed);
case "list":
return commandList(parsed);
case "coverage":
return commandCoverage(parsed);
case "serve":
return commandServe(parsed);
case "prompt":
return commandPrompt(parsed);
case "verify":
return commandVerify(parsed);
case "run-agent":
return commandRunAgent(parsed);
case "smoke":
return commandSmoke(parsed);
case "generate":
return commandGenerate(parsed);
case "scaffold":
return commandScaffold(parsed);
case "summarize":
return commandSummarize(parsed);
default:
throw new Error(`unknown command ${JSON.stringify(parsed.command)}`);
}
}
main().catch((error) => {
console.error(`browser eval: ${error instanceof Error ? error.message : String(error)}`);
process.exitCode = 1;
});
@@ -1,18 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "diagnostics",
routes: ["/diagnostics"],
render() {
return page({
title: "Diagnostics Lab",
body: `<section class="card"><h1>Diagnostics Lab</h1><p>Inspect console and network buffers.</p><p id="network-status" class="muted">Ping pending</p></section>`,
script: `
console.log("Browser eval console marker: CONSOLE-61");
fetch("/api/ping?run=" + encodeURIComponent(browserEval.runId))
.then((response) => response.json())
.then(({ token }) => { document.querySelector("#network-status").textContent = "Ping completed: " + token; });
`,
});
},
};
@@ -1,25 +0,0 @@
import { escapeHtml, page } from "../../lib/fixtures.mjs";
export default {
id: "form",
routes: ["/form", "/result"],
render({ pathname, runId, query, record }) {
if (pathname === "/form") {
return page({
title: "Browser Eval Form",
body: `<section class="card"><h1>Web form</h1><form action="/result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><label>Text input<input id="text-input" name="text" autocomplete="off"></label><label>Textarea<textarea id="notes" name="notes"></textarea></label><label>Dropdown<select id="choice" name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label><button id="submit" type="submit">Submit</button></form></section>`,
script: `document.querySelector("#submit").addEventListener("keydown", (event) => { if (event.key === "Enter") browserEval.send("form.enter_pressed"); });`,
});
}
const values = {
text: query.get("text") ?? "",
notes: query.get("notes") ?? "",
choice: query.get("choice") ?? "",
};
record("form.submitted", values, { path: pathname });
return page({
title: "Form Result",
body: `<section class="card"><h1>Received!</h1><p id="submitted-values" class="marker">${escapeHtml(values.text)} | ${escapeHtml(values.notes)} | ${escapeHtml(values.choice)}</p></section>`,
});
},
};
@@ -1,71 +0,0 @@
import { escapeHtml, page } from "../../lib/fixtures.mjs";
import { describeSeed } from "../../lib/seeded-generator.mjs";
function nest(content, depth) {
let nested = content;
for (let index = 0; index < depth; index += 1) {
nested = `<div data-depth="${index + 1}">${nested}</div>`;
}
return nested;
}
function textField(dimensions) {
const id = `matrix-text-${dimensions.idSuffix}`;
const input = `<input id="${id}" name="text" autocomplete="off">`;
if (dimensions.labelMode === "wrapped") return `<label>Text input${input}</label>`;
if (dimensions.labelMode === "for") return `<label for="${id}">Text input</label>${input}`;
return `<span id="${id}-label">Text input</span><input id="${id}" name="text" aria-labelledby="${id}-label" autocomplete="off">`;
}
function formMarkup(dimensions, runId) {
const text = textField(dimensions);
const select = `<label>Dropdown<select name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label>`;
const fields = dimensions.fieldOrder === "text-first" ? `${text}${select}` : `${select}${text}`;
const decoy =
dimensions.decoy === "disabled-input"
? `<label>Text input<input disabled value="decoy"></label>`
: dimensions.decoy === "duplicate-text"
? `<p class="muted">Text input is intentionally repeated in helper copy.</p>`
: "";
return nest(
`<form action="/matrix/generated-result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><input type="hidden" name="seed" value="${dimensions.seed}">${decoy}${fields}<button type="submit">Submit matrix form</button></form>`,
dimensions.nestingDepth,
);
}
export default {
id: "generated-form",
routes: ["/matrix/generated-form", "/matrix/generated-result"],
render({ pathname, runId, query, record }) {
const dimensions = describeSeed(query.get("seed") ?? 1);
if (pathname === "/matrix/generated-form") {
const markup = JSON.stringify(formMarkup(dimensions, runId));
const dimensionsJson = JSON.stringify(dimensions);
return page({
title: "Seeded Matrix Form",
body: `<section class="card"><h1>Seeded Matrix Form</h1><p class="marker">SEED-${dimensions.seed}</p><div id="matrix-root">Hydrating…</div></section>`,
script: `
const hydrate = () => {
const root = document.querySelector("#matrix-root");
root.innerHTML = ${markup};
root.querySelector("button[type=submit]").addEventListener("keydown", (event) => {
if (event.key === "Enter") browserEval.send("matrix.enter_pressed");
});
browserEval.send("matrix.ready", ${dimensionsJson});
};
setTimeout(hydrate, ${dimensions.hydrationDelayMs});
`,
});
}
const values = {
seed: dimensions.seed,
text: query.get("text") ?? "",
choice: query.get("choice") ?? "",
};
record("matrix.submitted", values, { path: pathname });
return page({
title: "Matrix Result",
body: `<section class="card"><h1>Matrix Result</h1><p class="marker">MATRIX-OK</p><p>${escapeHtml(values.text)} | ${escapeHtml(values.choice)} | seed ${values.seed}</p></section>`,
});
},
};
@@ -1,17 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "hover-inspect",
routes: ["/hover-inspect"],
render() {
return page({
title: "Hover and Inspect",
body: `<section class="card"><h1>Hover and Inspect</h1><div id="account-menu" tabindex="0"><button id="account-button" type="button">Account</button><div id="hover-panel"><button id="profile-link" type="button">Profile beta</button></div></div><p id="hover-result" class="marker" hidden>HOVER-17</p><article id="inspect-card" class="card" data-build="2026.08" data-marker="INSPECT-27"><h2>Inspection card</h2><p>Local HTML marker: <code>INSPECT-27</code></p></article></section>`,
script: `
const menu = document.querySelector("#account-menu");
menu.addEventListener("mouseenter", () => { menu.classList.add("open"); browserEval.send("hover.menu_opened"); });
document.querySelector("#profile-link").addEventListener("click", () => { document.querySelector("#hover-result").hidden = false; browserEval.send("hover.profile_clicked"); });
`,
});
},
};
@@ -1,12 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "fixture-index",
routes: ["/"],
render() {
return page({
title: "Browser Eval Fixtures",
body: `<section class="card"><h1>Browser Eval Fixtures</h1><p class="muted">Deterministic local pages for agent-neutral browser evaluation.</p></section>`,
});
},
};
@@ -1,19 +0,0 @@
import { page, withRun } from "../../lib/fixtures.mjs";
export default {
id: "navigation",
routes: ["/navigation/start", "/navigation/detail"],
render({ pathname, runId }) {
if (pathname === "/navigation/start") {
return page({
title: "Navigation Start",
body: `<section class="card"><h1>Navigation Start</h1><p>Use the link to open the deterministic detail page.</p><a id="details-link" class="button" href="${withRun("/navigation/detail", runId)}">Details</a></section>`,
script: `document.querySelector("#details-link").addEventListener("click", () => browserEval.send("navigation.detail.clicked"));`,
});
}
return page({
title: "Navigation Detail",
body: `<section class="card"><h1>Navigation Detail</h1><p>Stable detail code:</p><p id="navigation-code" class="marker">NAV-42</p></section>`,
});
},
};
@@ -1,21 +0,0 @@
import { page } from "../../lib/fixtures.mjs";
export default {
id: "responsive",
routes: ["/responsive"],
render() {
return page({
title: "Responsive Lab",
body: `<section class="card"><h1>Responsive Lab</h1><p id="viewport-marker" class="marker">VIEWPORT-84</p><p id="viewport-value"></p></section>`,
script: `
const sample = () => {
const data = { width: innerWidth, height: innerHeight, touchPoints: navigator.maxTouchPoints, userAgent: navigator.userAgent };
document.querySelector("#viewport-value").textContent = data.width + "x" + data.height;
browserEval.send("viewport.sample", data);
};
window.addEventListener("resize", sample);
sample();
`,
});
},
};
@@ -1,19 +0,0 @@
import { escapeHtml, page, withRun } from "../../lib/fixtures.mjs";
export default {
id: "tabs",
routes: ["/tabs", "/tabs/child"],
render({ pathname, runId }) {
if (pathname === "/tabs") {
return page({
title: "Tab Lab",
body: `<section class="card"><h1>Tab Lab</h1><p>Keep this original tab open while visiting the child page in a new tab.</p><p class="muted">Child URL: <code>${escapeHtml(withRun("/tabs/child", runId))}</code></p></section>`,
});
}
return page({
title: "Child Tab",
body: `<section class="card"><h1>Child Tab</h1><p id="tab-token" class="marker">TAB-29</p><button id="mark-child" type="button">Mark child visited</button><p id="child-status" hidden>Marked</p></section>`,
script: `document.querySelector("#mark-child").addEventListener("click", () => { document.querySelector("#child-status").hidden = false; browserEval.send("tabs.child_marked"); });`,
});
},
};
-99
View File
@@ -1,99 +0,0 @@
import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { countPattern, runProcess, substituteArgs } from "./process.mjs";
import { renderPrompt } from "./tasks.mjs";
function safeRunPart(value) {
return value.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 48);
}
export async function loadAgentConfig(configPath, agentName) {
const absolute = resolve(configPath);
const config = JSON.parse(await readFile(absolute, "utf8"));
const adapter = config.agents?.[agentName];
if (!adapter) {
const names = Object.keys(config.agents ?? {}).join(", ");
throw new Error(`unknown agent ${JSON.stringify(agentName)}; configured agents: ${names}`);
}
if (typeof adapter.command !== "string" || !Array.isArray(adapter.args)) {
throw new Error(`agent ${JSON.stringify(agentName)} must define command and args`);
}
return { adapter, configDirectory: dirname(absolute) };
}
export async function runCommandTask({
server,
serverInfo,
task,
adapter,
adapterName,
configDirectory,
iteration,
locale,
seed = task.seed,
timeoutMs,
variantOverride,
}) {
const variant = variantOverride ?? adapter.variant ?? "unspecified";
const runId = [
safeRunPart(adapterName),
safeRunPart(variant),
safeRunPart(task.id),
iteration,
randomUUID().slice(0, 8),
].join("-");
server.reset(runId);
const prompt = renderPrompt(task, { baseUrl: serverInfo.baseUrl, runId, locale, seed });
const values = {
prompt,
caseId: task.id,
taskId: task.id,
runId,
baseUrl: serverInfo.baseUrl,
seed: seed ?? "",
variant,
};
const args = substituteArgs(adapter.args, values);
const execution = await runProcess(adapter.command, args, {
cwd: adapter.cwd ? resolve(configDirectory, adapter.cwd) : configDirectory,
env: adapter.env,
timeoutMs: adapter.timeoutMs ?? timeoutMs,
});
await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
const responseText = execution.stdout;
const combinedOutput = `${execution.stdout}\n${execution.stderr}`;
const verification = verifyTask(task, {
events: server.snapshot(runId).events,
responseText,
});
const defaultErrorPattern = "(?:\\bError:|timed out|unfinished command|session already has)";
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
iteration,
adapter: adapterName,
variant,
adapterMetadata: adapter.metadata ?? {},
locale,
prompt,
execution,
metrics: {
errorCount: countPattern(
combinedOutput,
adapter.metrics?.errorPattern ?? defaultErrorPattern,
),
toolCallCount: countPattern(combinedOutput, adapter.metrics?.toolCallPattern),
responseChars: responseText.length,
},
eventCount: server.snapshot(runId).events.length,
verification,
};
}
-309
View File
@@ -1,309 +0,0 @@
import { mkdir, stat } from "node:fs/promises";
import { join } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { runProcess } from "./process.mjs";
import { taskUrl } from "./tasks.mjs";
function parseJsonOutput(step) {
try {
return JSON.parse(step.stdout);
} catch (error) {
throw new Error(`could not parse bsk JSON output: ${error.message}`);
}
}
function renderStepError(step) {
const raw = step.stderr.trim() || step.stdout.trim();
if (!raw) return `exit ${step.exitCode}`;
try {
const parsed = JSON.parse(raw);
return parsed.hint ? `${parsed.message} (hint: ${parsed.hint})` : parsed.message;
} catch {
return raw;
}
}
async function hasNonemptyFile(path) {
try {
return (await stat(path)).size > 0;
} catch {
return false;
}
}
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function resolveValue(value, variables) {
if (typeof value !== "string") return value;
const exact = value.match(/^\{([^{}]+)}$/);
if (exact) {
const resolved = readPath(variables, exact[1]);
if (resolved === undefined) throw new Error(`unknown workflow variable ${exact[1]}`);
return resolved;
}
return value.replaceAll(/\{([^{}]+)\}/g, (_match, path) => {
const resolved = readPath(variables, path);
if (resolved === undefined) throw new Error(`unknown workflow variable ${path}`);
return String(resolved);
});
}
function targetArgs(step, variables) {
if (step.selector) return ["--selector", String(resolveValue(step.selector, variables))];
if (step.ref) return ["--ref", String(resolveValue(step.ref, variables))];
return [];
}
function tabArgs(step, variables) {
return step.tabId === undefined ? [] : ["--tab-id", String(resolveValue(step.tabId, variables))];
}
function matchesWhere(value, where = {}) {
return Object.entries(where).every(([path, expected]) => readPath(value, path) === expected);
}
export async function runBskSmokeTask({
bskCommand,
outputDirectory,
server,
serverInfo,
task,
runId,
seed = task.seed,
timeoutMs = 60_000,
}) {
const steps = [];
const evidence = {};
const variables = {
baseUrl: serverInfo.baseUrl,
runId,
seed: seed ?? "",
url: taskUrl(task, serverInfo.baseUrl, runId, seed),
};
let sessionId;
let executionError;
await mkdir(outputDirectory, { recursive: true });
async function bsk(args, { timeout = timeoutMs } = {}) {
const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout });
steps.push(execution);
if (execution.exitCode !== 0 || execution.timedOut || execution.error) {
throw new Error(
`bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`,
);
}
return parseJsonOutput(execution);
}
async function waitForSiteEvent(step) {
const timeout = step.timeoutMs ?? 3_000;
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
if (
server
.snapshot(runId)
.events.some(
(event) => event.type === step.type && matchesWhere(event.data ?? {}, step.where ?? {}),
)
) {
return { observed: true };
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 50));
}
throw new Error(`local fixture did not report ${step.type} within ${timeout}ms`);
}
async function executeWorkflowStep(step) {
const session = ["--session", sessionId];
let result;
switch (step.action) {
case "navigate":
result = await bsk([
"navigate",
String(resolveValue(step.url ?? "{url}", variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "back":
case "forward":
result = await bsk([`navigate-${step.action}`, ...session, ...tabArgs(step, variables)]);
break;
case "reload":
result = await bsk(["reload", ...session, ...tabArgs(step, variables)]);
break;
case "wait":
result = await bsk(["wait-ms", String(resolveValue(step.duration, variables))]);
break;
case "wait-site-event":
result = await waitForSiteEvent(step);
break;
case "observe":
case "snapshot":
case "html": {
const command = step.action === "html" ? "get-html" : step.action;
result = await bsk([command, ...session, ...tabArgs(step, variables)]);
break;
}
case "screenshot": {
const name = resolveValue(step.artifactName ?? "screenshot.png", variables);
const path = join(outputDirectory, `${runId}-${name}`);
result = await bsk(["screenshot", ...session, ...tabArgs(step, variables), "--out", path]);
if (step.evidence) evidence[step.evidence] = await hasNonemptyFile(path);
break;
}
case "console":
case "network":
result = await bsk([step.action, ...session, ...tabArgs(step, variables)]);
break;
case "click":
case "hover":
result = await bsk([
step.action,
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "fill":
result = await bsk([
"fill",
...targetArgs(step, variables),
"--value",
String(resolveValue(step.value, variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "select":
result = await bsk([
"select",
...targetArgs(step, variables),
...step.values.flatMap((value) => ["--value", String(resolveValue(value, variables))]),
...session,
...tabArgs(step, variables),
]);
break;
case "press":
result = await bsk([
"press",
String(resolveValue(step.key, variables)),
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "tab-list":
result = await bsk(["tab", "list", ...session]);
break;
case "tab-create":
result = await bsk([
"tab",
"create",
...session,
...(step.url ? ["--url", String(resolveValue(step.url, variables))] : []),
]);
break;
case "tab-select":
case "tab-close":
result = await bsk([
"tab",
step.action.slice(4),
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "borrow":
case "return":
result = await bsk([
"tab",
step.action,
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "resize":
result = await bsk([
"window",
"resize",
...session,
"--width",
String(step.width),
"--height",
String(step.height),
]);
break;
case "emulate":
result = await bsk([
"emulate",
...session,
...tabArgs(step, variables),
"--device",
String(resolveValue(step.device, variables)),
]);
break;
case "request-help":
result = await bsk([
"request-help",
...session,
"--prompt",
String(resolveValue(step.prompt, variables)),
...(step.title ? ["--title", String(resolveValue(step.title, variables))] : []),
...(step.timeout ? ["--timeout", String(step.timeout)] : []),
]);
break;
default:
throw new Error(`unsupported smoke workflow action ${step.action}`);
}
if (step.saveAs) variables[step.saveAs] = result;
if (step.evidence && step.action !== "screenshot") evidence[step.evidence] = true;
}
server.reset(runId);
try {
if (task.smokeSteps.length === 0) throw new Error(`case ${task.id} has no smoke workflow`);
const started = await bsk(["session", "start", "--no-focus"], { timeout: 90_000 });
sessionId = started.session_id;
variables.sessionId = sessionId;
await bsk(["session", "list"]);
for (const step of task.smokeSteps) await executeWorkflowStep(step);
} catch (error) {
executionError = error instanceof Error ? error.message : String(error);
} finally {
if (sessionId) {
try {
const stopped = await bsk(["session", "stop", sessionId], { timeout: 90_000 });
evidence.sessionStopped =
stopped.stopped?.includes(sessionId) === true && (stopped.failed?.length ?? 0) === 0;
if (!evidence.sessionStopped) {
executionError ??= `bsk did not confirm that session ${sessionId} stopped`;
}
} catch (error) {
evidence.sessionStopped = false;
executionError ??= error instanceof Error ? error.message : String(error);
}
}
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 350));
const responseText = steps.map(({ stdout }) => stdout).join("\n");
const events = server.snapshot(runId).events;
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
adapter: "bsk-direct",
variant: "cli-direct",
executionError,
steps,
evidence,
eventCount: events.length,
verification: verifyTask(task, { events, responseText, adapterEvidence: evidence }),
};
}
-216
View File
@@ -1,216 +0,0 @@
import { readFile } from "node:fs/promises";
import { discoverFiles } from "./discovery.mjs";
import { OPERATION_NAMES, WORKFLOW_ACTIONS } from "./operations.mjs";
import { casesDirectory as defaultCasesDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const EVIDENCE_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
const TOP_LEVEL_KEYS = new Set([
"$schema",
"schemaVersion",
"id",
"title",
"order",
"suite",
"tags",
"seed",
"source",
"fixture",
"prompts",
"coverage",
"assertions",
"smoke",
]);
function isObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function checkString(errors, value, path) {
if (typeof value !== "string" || value.trim() === "") errors.push(`${path} must be a string`);
}
function validateAssertion(errors, assertion, path, kind) {
if (!isObject(assertion)) {
errors.push(`${path} must be an object`);
return;
}
checkString(errors, assertion.label, `${path}.label`);
if (kind === "site") {
checkString(errors, assertion.type, `${path}.type`);
if (!Number.isInteger(assertion.minCount) || assertion.minCount < 1) {
errors.push(`${path}.minCount must be a positive integer`);
}
if (assertion.where !== undefined && !isObject(assertion.where)) {
errors.push(`${path}.where must be an object`);
}
} else if (kind === "response") {
checkString(errors, assertion.includes, `${path}.includes`);
} else {
checkString(errors, assertion.key, `${path}.key`);
}
}
function validateWorkflowStep(errors, step, path) {
if (!isObject(step)) {
errors.push(`${path} must be an object`);
return;
}
if (!WORKFLOW_ACTIONS.has(step.action)) {
errors.push(`${path}.action is unsupported: ${JSON.stringify(step.action)}`);
}
if (step.saveAs !== undefined && !ID_PATTERN.test(step.saveAs)) {
errors.push(`${path}.saveAs must use lowercase kebab-case`);
}
if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) {
errors.push(`${path}.evidence must be an identifier`);
}
if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) {
errors.push(`${path} requires selector or ref`);
}
if (step.action === "fill") checkString(errors, step.value, `${path}.value`);
if (step.action === "select") {
if (!Array.isArray(step.values) || step.values.length === 0) {
errors.push(`${path}.values must be a non-empty array`);
} else if (step.values.some((value) => typeof value !== "string")) {
errors.push(`${path}.values must contain only strings`);
}
}
if (step.action === "press") checkString(errors, step.key, `${path}.key`);
if (step.action === "wait") checkString(errors, step.duration, `${path}.duration`);
if (step.action === "wait-site-event") checkString(errors, step.type, `${path}.type`);
if (["tab-select", "tab-close", "borrow", "return"].includes(step.action)) {
checkString(errors, step.tabId, `${path}.tabId`);
}
if (step.action === "resize") {
if (
!Number.isInteger(step.width) ||
step.width < 1 ||
!Number.isInteger(step.height) ||
step.height < 1
) {
errors.push(`${path}.width and height must be positive integers`);
}
}
if (step.action === "emulate") checkString(errors, step.device, `${path}.device`);
if (step.action === "request-help") checkString(errors, step.prompt, `${path}.prompt`);
if (step.timeoutMs !== undefined && (!Number.isInteger(step.timeoutMs) || step.timeoutMs < 1)) {
errors.push(`${path}.timeoutMs must be a positive integer`);
}
}
export function validateCaseManifest(manifest, { source = "case manifest" } = {}) {
const errors = [];
if (!isObject(manifest)) return [`${source} must contain a JSON object`];
for (const key of Object.keys(manifest)) {
if (!TOP_LEVEL_KEYS.has(key)) errors.push(`unknown top-level property ${key}`);
}
if (manifest.schemaVersion !== 1) errors.push("schemaVersion must equal 1");
if (!ID_PATTERN.test(manifest.id ?? "")) errors.push("id must use lowercase kebab-case");
checkString(errors, manifest.title, "title");
if (manifest.order !== undefined && (!Number.isInteger(manifest.order) || manifest.order < 0)) {
errors.push("order must be a non-negative integer");
}
if (!ID_PATTERN.test(manifest.suite ?? "")) errors.push("suite must use lowercase kebab-case");
if (!Array.isArray(manifest.tags) || manifest.tags.some((tag) => !ID_PATTERN.test(tag))) {
errors.push("tags must be an array of lowercase kebab-case strings");
} else if (new Set(manifest.tags).size !== manifest.tags.length) {
errors.push("tags must not contain duplicates");
}
if (!isObject(manifest.fixture)) errors.push("fixture must be an object");
else if (
typeof manifest.fixture.startPath !== "string" ||
!manifest.fixture.startPath.startsWith("/")
) {
errors.push("fixture.startPath must be an absolute URL path");
}
if (!isObject(manifest.prompts)) errors.push("prompts must be an object");
else {
checkString(errors, manifest.prompts.en, "prompts.en");
checkString(errors, manifest.prompts["zh-CN"], "prompts.zh-CN");
}
if (!Array.isArray(manifest.coverage)) errors.push("coverage must be an array");
else {
for (const operation of manifest.coverage) {
if (!OPERATION_NAMES.has(operation))
errors.push(`coverage contains unknown operation ${operation}`);
}
if (new Set(manifest.coverage).size !== manifest.coverage.length) {
errors.push("coverage must not contain duplicates");
}
}
if (!isObject(manifest.assertions)) errors.push("assertions must be an object");
else {
for (const kind of ["site", "response", "adapter"]) {
if (!Array.isArray(manifest.assertions[kind])) {
errors.push(`assertions.${kind} must be an array`);
} else {
manifest.assertions[kind].forEach((assertion, index) =>
validateAssertion(errors, assertion, `assertions.${kind}[${index}]`, kind),
);
}
}
}
if (!isObject(manifest.smoke) || !Array.isArray(manifest.smoke.steps)) {
errors.push("smoke.steps must be an array");
} else {
manifest.smoke.steps.forEach((step, index) =>
validateWorkflowStep(errors, step, `smoke.steps[${index}]`),
);
}
if (manifest.seed !== undefined && !Number.isSafeInteger(manifest.seed)) {
errors.push("seed must be a safe integer");
}
if (manifest.source !== undefined && !isObject(manifest.source)) {
errors.push("source must be an object");
}
return errors.map((error) => `${source}: ${error}`);
}
function normalizeCase(manifest, sourceFile) {
return Object.freeze({
...manifest,
startPath: manifest.fixture.startPath,
siteAssertions: manifest.assertions.site,
responseAssertions: manifest.assertions.response,
adapterAssertions: manifest.assertions.adapter,
smokeSteps: manifest.smoke.steps,
sourceFile,
});
}
export async function loadCaseManifests({ casesDirectory = defaultCasesDirectory } = {}) {
const files = await discoverFiles(casesDirectory, (path) => path.endsWith(".case.json"));
const cases = [];
const ids = new Map();
const errors = [];
for (const file of files) {
let manifest;
try {
manifest = JSON.parse(await readFile(file, "utf8"));
} catch (error) {
errors.push(`${file}: invalid JSON: ${error.message}`);
continue;
}
const manifestErrors = validateCaseManifest(manifest, { source: file });
errors.push(...manifestErrors);
if (typeof manifest?.id === "string") {
if (ids.has(manifest.id)) {
errors.push(`${file}: duplicate id also declared by ${ids.get(manifest.id)}`);
} else {
ids.set(manifest.id, file);
}
}
if (manifestErrors.length === 0) cases.push(normalizeCase(manifest, file));
}
if (files.length === 0) errors.push(`${casesDirectory}: no .case.json files found`);
if (errors.length > 0) throw new Error(`case validation failed:\n${errors.join("\n")}`);
cases.sort(
(left, right) =>
left.suite.localeCompare(right.suite) ||
(left.order ?? Number.MAX_SAFE_INTEGER) - (right.order ?? Number.MAX_SAFE_INTEGER) ||
left.id.localeCompare(right.id),
);
return Object.freeze(cases);
}
-26
View File
@@ -1,26 +0,0 @@
import { OPERATION_CATALOG, WORKFLOW_ACTION_OPERATIONS } from "./operations.mjs";
export function buildOperationCoverage(cases) {
return OPERATION_CATALOG.map(({ operation, manualNote }) => {
const agentCases = cases
.filter((testCase) => testCase.coverage.includes(operation))
.map(({ id }) => id);
let smokeCases;
if (["session.start", "session.stop", "session.list"].includes(operation)) {
smokeCases = cases.filter(({ smokeSteps }) => smokeSteps.length > 0).map(({ id }) => id);
} else {
smokeCases = cases
.filter(({ smokeSteps }) =>
smokeSteps.some((step) => WORKFLOW_ACTION_OPERATIONS[step.action] === operation),
)
.map(({ id }) => id);
}
return {
operation,
agentCases,
smokeCases,
directSmoke: smokeCases.length > 0,
note: smokeCases.length === 0 && manualNote ? `manual lane: ${manualNote}` : undefined,
};
});
}
-24
View File
@@ -1,24 +0,0 @@
import { readdir } from "node:fs/promises";
import { join } from "node:path";
export async function discoverFiles(root, predicate) {
const files = [];
async function visit(directory) {
let entries;
try {
entries = await readdir(directory, { withFileTypes: true });
} catch (error) {
if (error?.code === "ENOENT") return;
throw error;
}
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
const path = join(directory, entry.name);
if (entry.isDirectory()) await visit(path);
else if (entry.isFile() && predicate(path)) files.push(path);
}
}
await visit(root);
return files;
}
-68
View File
@@ -1,68 +0,0 @@
import { pathToFileURL } from "node:url";
import { discoverFiles } from "./discovery.mjs";
import { casesDirectory, fixturesDirectory } from "./paths.mjs";
export async function loadFixtureRegistry({ roots = [fixturesDirectory, casesDirectory] } = {}) {
const files = (
await Promise.all(
roots.map((root) => discoverFiles(root, (path) => path.endsWith(".fixture.mjs"))),
)
).flat();
const routes = new Map();
const definitions = [];
const errors = [];
for (const file of files.sort()) {
let definition;
try {
definition = (await import(pathToFileURL(file).href)).default;
} catch (error) {
errors.push(`${file}: could not import fixture: ${error.message}`);
continue;
}
if (!definition || typeof definition.id !== "string") {
errors.push(`${file}: default export must define id`);
continue;
}
if (!Array.isArray(definition.routes) || definition.routes.length === 0) {
errors.push(`${file}: default export must define non-empty routes`);
continue;
}
if (typeof definition.render !== "function") {
errors.push(`${file}: default export must define render(context)`);
continue;
}
definitions.push({ ...definition, sourceFile: file });
for (const route of definition.routes) {
if (typeof route !== "string" || !route.startsWith("/")) {
errors.push(`${file}: invalid route ${JSON.stringify(route)}`);
} else if (routes.has(route)) {
errors.push(
`${file}: route ${route} is already registered by ${routes.get(route).sourceFile}`,
);
} else {
routes.set(route, { definition, sourceFile: file });
}
}
}
if (errors.length > 0) throw new Error(`fixture validation failed:\n${errors.join("\n")}`);
return Object.freeze({
definitions: Object.freeze(definitions),
routes: Object.freeze([...routes.keys()].sort()),
hasRoute(pathname) {
return routes.has(pathname);
},
render(pathname, context) {
const entry = routes.get(pathname);
return entry ? entry.definition.render({ ...context, pathname }) : null;
},
});
}
export function validateCaseFixtureLinks(cases, fixtureRegistry) {
return cases
.filter(({ startPath }) => !fixtureRegistry.hasRoute(startPath))
.map(({ id, startPath }) => `case ${id} references unregistered fixture route ${startPath}`);
}
-78
View File
@@ -1,78 +0,0 @@
const stylesheet = `
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
body { margin: 0; background: #f4f7fb; color: #182033; }
main { width: min(760px, calc(100% - 40px)); margin: 48px auto; }
.card { background: white; border: 1px solid #d9e1ef; border-radius: 14px; padding: 24px; box-shadow: 0 8px 28px #20304a12; }
h1 { margin-top: 0; }
a, button { font: inherit; }
button, .button { display: inline-block; border: 0; border-radius: 8px; padding: 10px 16px; background: #3157d5; color: white; cursor: pointer; text-decoration: none; }
label { display: block; margin: 16px 0; font-weight: 600; }
input, textarea, select { display: block; box-sizing: border-box; width: 100%; margin-top: 6px; border: 1px solid #9caac2; border-radius: 7px; padding: 9px; font: inherit; }
textarea { min-height: 90px; }
.marker { border-left: 4px solid #3157d5; padding: 8px 12px; background: #eef2ff; font-family: ui-monospace, monospace; }
.muted { color: #5c667a; }
#account-menu { position: relative; display: inline-block; padding-bottom: 12px; }
#hover-panel { display: none; position: absolute; z-index: 2; top: 34px; left: 0; width: 190px; padding: 10px; background: white; border: 1px solid #9caac2; border-radius: 8px; box-shadow: 0 8px 24px #20304a22; }
#account-menu:hover #hover-panel, #account-menu.open #hover-panel { display: block; }
#hover-panel button { width: 100%; }
#inspect-card { margin-top: 80px; }
code { font-family: ui-monospace, monospace; }
`;
const clientScript = `
(() => {
const runId = new URLSearchParams(location.search).get("run");
const navigation = performance.getEntriesByType("navigation")[0];
const send = (type, data = {}) => {
if (!runId) return Promise.resolve();
return fetch("/api/events", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId, type, path: location.pathname, data }),
keepalive: true,
}).catch(() => undefined);
};
window.browserEval = { runId, send };
window.addEventListener("pageshow", (event) => {
send("page.shown", {
path: location.pathname,
persisted: event.persisted,
navigationType: navigation?.type ?? "unknown",
});
});
})();
`;
export function escapeHtml(value) {
return String(value)
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;");
}
export function withRun(path, runId, extra = {}) {
const url = new URL(path, "http://browser-eval.local");
url.searchParams.set("run", runId);
for (const [name, value] of Object.entries(extra)) {
if (value !== undefined) url.searchParams.set(name, String(value));
}
return `${url.pathname}${url.search}`;
}
export function page({ title, body, script = "" }) {
return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>${escapeHtml(title)}</title>
<style>${stylesheet}</style>
</head>
<body>
<main>${body}</main>
<script>${clientScript}</script>
${script ? `<script>${script}</script>` : ""}
</body>
</html>`;
}
-68
View File
@@ -1,68 +0,0 @@
export const OPERATION_CATALOG = [
{ operation: "session.start" },
{ operation: "session.stop" },
{ operation: "session.list" },
{ operation: "page.navigate" },
{ operation: "page.back" },
{ operation: "page.forward" },
{ operation: "page.reload" },
{ operation: "page.wait" },
{ operation: "inspect.observe" },
{ operation: "inspect.snapshot" },
{ operation: "inspect.html" },
{ operation: "inspect.screenshot" },
{ operation: "inspect.console" },
{ operation: "inspect.network" },
{ operation: "interact.click" },
{ operation: "interact.hover" },
{ operation: "interact.fill" },
{ operation: "interact.select" },
{ operation: "interact.press" },
{ operation: "tabs.list" },
{ operation: "tabs.create" },
{ operation: "tabs.select" },
{ operation: "tabs.close" },
{
operation: "tabs.borrow",
manualNote: "requires an existing user tab and ownership confirmation",
},
{ operation: "tabs.return", manualNote: "paired with borrowing a real user tab" },
{ operation: "assist.resize" },
{ operation: "assist.emulate" },
{ operation: "assist.request-help", manualNote: "intentionally waits for human interaction" },
];
export const WORKFLOW_ACTION_OPERATIONS = {
navigate: "page.navigate",
back: "page.back",
forward: "page.forward",
reload: "page.reload",
wait: "page.wait",
observe: "inspect.observe",
snapshot: "inspect.snapshot",
html: "inspect.html",
screenshot: "inspect.screenshot",
console: "inspect.console",
network: "inspect.network",
click: "interact.click",
hover: "interact.hover",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
"tab-list": "tabs.list",
"tab-create": "tabs.create",
"tab-select": "tabs.select",
"tab-close": "tabs.close",
borrow: "tabs.borrow",
return: "tabs.return",
resize: "assist.resize",
emulate: "assist.emulate",
"request-help": "assist.request-help",
};
export const WORKFLOW_ACTIONS = new Set([
...Object.keys(WORKFLOW_ACTION_OPERATIONS),
"wait-site-event",
]);
export const OPERATION_NAMES = new Set(OPERATION_CATALOG.map(({ operation }) => operation));
-63
View File
@@ -1,63 +0,0 @@
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function eventMatches(event, assertion) {
if (event.type !== assertion.type) return false;
return Object.entries(assertion.where ?? {}).every(
([path, expected]) => readPath(event, path) === expected,
);
}
export function verifyTask(task, { events = [], responseText = "", adapterEvidence = {} } = {}) {
const site = task.siteAssertions.map((assertion) => {
const actual = events.filter((event) => eventMatches(event, assertion)).length;
return {
kind: "site",
label: assertion.label,
status: actual >= assertion.minCount ? "passed" : "failed",
expected: `at least ${assertion.minCount}`,
actual,
};
});
const response = task.responseAssertions.map((assertion) => ({
kind: "response",
label: assertion.label,
status: responseText.includes(assertion.includes) ? "passed" : "failed",
expected: `response includes ${JSON.stringify(assertion.includes)}`,
}));
const adapter = task.adapterAssertions.map((assertion) => {
if (!Object.hasOwn(adapterEvidence, assertion.key)) {
return {
kind: "adapter",
label: assertion.label,
status: "unverified",
expected: assertion.key,
};
}
return {
kind: "adapter",
label: assertion.label,
status: adapterEvidence[assertion.key] ? "passed" : "failed",
expected: assertion.key,
actual: adapterEvidence[assertion.key],
};
});
const checks = [...site, ...response, ...adapter];
const failed = checks.filter(({ status }) => status === "failed").length;
const unverified = checks.filter(({ status }) => status === "unverified").length;
return {
taskId: task.id,
status: failed > 0 ? "failed" : unverified > 0 ? "passed-with-unverified" : "passed",
summary: {
passed: checks.length - failed - unverified,
failed,
unverified,
total: checks.length,
},
checks,
};
}
-8
View File
@@ -1,8 +0,0 @@
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export const evalDirectory = dirname(dirname(fileURLToPath(import.meta.url)));
export const casesDirectory = join(evalDirectory, "cases");
export const fixturesDirectory = join(evalDirectory, "fixtures");
export const schemasDirectory = join(evalDirectory, "schemas");
export const resultsDirectory = join(evalDirectory, "results");
-71
View File
@@ -1,71 +0,0 @@
import { spawn } from "node:child_process";
const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
function appendCaptured(current, chunk) {
if (current.length >= MAX_CAPTURE_BYTES) return current;
return `${current}${chunk}`.slice(0, MAX_CAPTURE_BYTES);
}
export function runProcess(command, args, { cwd, env, timeoutMs = 300_000 } = {}) {
return new Promise((resolve) => {
const startedAt = Date.now();
let stdout = "";
let stderr = "";
let timedOut = false;
let settled = false;
const child = spawn(command, args, {
cwd,
env: { ...process.env, ...env },
shell: false,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout.on("data", (chunk) => {
stdout = appendCaptured(stdout, chunk.toString());
});
child.stderr.on("data", (chunk) => {
stderr = appendCaptured(stderr, chunk.toString());
});
const timeout = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
}, timeoutMs);
const finish = (result) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
resolve({
command,
args,
stdout,
stderr,
timedOut,
durationMs: Date.now() - startedAt,
...result,
});
};
child.once("error", (error) => finish({ exitCode: null, signal: null, error: error.message }));
child.once("close", (exitCode, signal) => finish({ exitCode, signal }));
});
}
export function substituteArgs(args, values) {
return args.map((argument) => {
let result = argument;
for (const [key, value] of Object.entries(values)) {
result = result.replaceAll(`{${key}}`, String(value));
}
return result;
});
}
export function countPattern(text, pattern) {
if (!pattern) return null;
const matches = text.match(new RegExp(pattern, "g"));
return matches?.length ?? 0;
}
@@ -1,64 +0,0 @@
import { relative, sep } from "node:path";
import { validateCaseFixtureLinks } from "./fixture-registry.mjs";
import { casesDirectory } from "./paths.mjs";
function templateRoots(value) {
if (typeof value === "string") {
return [...value.matchAll(/\{([^{}]+)\}/g)].map((match) => match[1].split(".")[0]);
}
if (Array.isArray(value)) return value.flatMap(templateRoots);
if (value && typeof value === "object") return Object.values(value).flatMap(templateRoots);
return [];
}
export function validateRepositoryCases(cases, fixtureRegistry) {
const errors = [...validateCaseFixtureLinks(cases, fixtureRegistry)];
for (const testCase of cases) {
const relativePath = relative(casesDirectory, testCase.sourceFile);
const suiteDirectory = relativePath.split(sep)[0];
if (suiteDirectory !== testCase.suite) {
errors.push(
`case ${testCase.id} declares suite ${testCase.suite} but is stored under ${suiteDirectory}`,
);
}
const availableVariables = new Set(["baseUrl", "runId", "seed", "sessionId", "url"]);
const evidence = new Set(["sessionStopped"]);
for (const [index, step] of testCase.smokeSteps.entries()) {
for (const root of templateRoots(step)) {
if (!availableVariables.has(root)) {
errors.push(
`case ${testCase.id} smoke.steps[${index}] references unavailable variable ${root}`,
);
}
}
if (step.saveAs) availableVariables.add(step.saveAs);
if (step.evidence) evidence.add(step.evidence);
}
for (const assertion of testCase.adapterAssertions) {
if (!evidence.has(assertion.key)) {
errors.push(
`case ${testCase.id} expects adapter evidence ${assertion.key} but no smoke step produces it`,
);
}
}
}
return errors;
}
export function repositorySummary(cases, fixtureRegistry) {
const suites = Object.fromEntries(
[...new Set(cases.map(({ suite }) => suite))]
.sort()
.map((suite) => [suite, cases.filter((testCase) => testCase.suite === suite).length]),
);
const tags = [...new Set(cases.flatMap(({ tags }) => tags))].sort();
return {
cases: cases.length,
suites,
tags: tags.length,
fixtureModules: fixtureRegistry.definitions.length,
fixtureRoutes: fixtureRegistry.routes.length,
};
}
-120
View File
@@ -1,120 +0,0 @@
import { mkdir, realpath, stat, writeFile } from "node:fs/promises";
import { join, relative, sep } from "node:path";
import { casesDirectory, evalDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
async function exists(path) {
try {
await stat(path);
return true;
} catch (error) {
if (error?.code === "ENOENT") return false;
throw error;
}
}
async function moduleSpecifier(fromDirectory, target) {
const path = relative(await realpath(fromDirectory), await realpath(target))
.split(sep)
.join("/");
return path.startsWith(".") ? path : `./${path}`;
}
export async function scaffoldCase({
id,
title = id,
suite = "regression",
source,
rootCasesDirectory = casesDirectory,
}) {
if (!ID_PATTERN.test(id ?? "")) throw new Error("case id must use lowercase kebab-case");
if (!ID_PATTERN.test(suite ?? "")) throw new Error("suite must use lowercase kebab-case");
const directory = join(rootCasesDirectory, suite, id);
if (await exists(directory)) throw new Error(`case directory already exists: ${directory}`);
await mkdir(directory, { recursive: true });
const marker = `CASE-${id.toUpperCase().replaceAll("-", "_")}`;
const startPath = `/cases/${id}`;
const manifest = {
$schema: "../../../schemas/case.schema.json",
schemaVersion: 1,
id,
title,
order: 10,
suite,
tags: ["badcase", "regression"],
...(source ? { source: { type: "user-badcase", reference: source } } : {}),
fixture: { startPath },
prompts: {
en: `Open {url}, complete the minimal regression scenario, report ${marker}, and close the browser session.`,
"zh-CN": `打开 {url},完成最小回归场景,报告 ${marker},最后关闭浏览器会话。`,
},
coverage: ["session.start", "session.stop", "page.navigate", "inspect.observe"],
assertions: {
site: [
{
label: "regression fixture was displayed",
type: "page.shown",
where: { "data.path": startPath },
minCount: 1,
},
],
response: [{ label: "reported regression marker", includes: marker }],
adapter: [{ label: "browser session was closed", key: "sessionStopped" }],
},
smoke: { steps: [{ action: "navigate" }, { action: "observe" }] },
};
const fixtureHelper = await moduleSpecifier(
directory,
join(evalDirectory, "lib", "fixtures.mjs"),
);
const fixture = `import { escapeHtml, page } from ${JSON.stringify(fixtureHelper)};
const title = ${JSON.stringify(title)};
export default {
id: ${JSON.stringify(id)},
routes: [${JSON.stringify(startPath)}],
render() {
return page({
title,
body: \`<section class="card"><h1>\${escapeHtml(title)}</h1><p class="marker">${marker}</p><p>Replace this page with a privacy-safe minimal reproduction.</p></section>\`,
});
},
};
`;
const readme = `# ${title}
## Original symptom
Describe what the user observed without including private data or proprietary page content.
## Minimal reproduction
Explain the smallest DOM, timing, or browser-state condition that reproduces the failure.
## Expected behavior
Describe the observable success condition and update the case assertions accordingly.
## Regression history
- Source: ${source ?? "add issue, PR, or anonymized user report reference"}
- Fix: add the fixing PR when available
`;
const files = {
manifest: join(directory, `${id}.case.json`),
fixture: join(directory, `${id}.fixture.mjs`),
readme: join(directory, "README.md"),
};
await Promise.all([
writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }),
writeFile(files.fixture, fixture, { flag: "wx" }),
writeFile(files.readme, readme, { flag: "wx" }),
]);
return { directory, files };
}
-40
View File
@@ -1,40 +0,0 @@
function hashSeed(value) {
const text = String(value);
let hash = 2166136261;
for (const character of text) {
hash ^= character.codePointAt(0);
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
function mulberry32(seed) {
let state = seed >>> 0;
return () => {
state += 0x6d2b79f5;
let value = state;
value = Math.imul(value ^ (value >>> 15), value | 1);
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
};
}
export function normalizeSeed(value) {
const numeric = Number(value);
return Number.isSafeInteger(numeric) ? numeric : hashSeed(value);
}
export function describeSeed(value) {
const seed = normalizeSeed(value);
const random = mulberry32(seed);
const choose = (values) => values[Math.floor(random() * values.length)];
return {
seed,
labelMode: choose(["wrapped", "for", "aria"]),
nestingDepth: 1 + Math.floor(random() * 4),
hydrationDelayMs: choose([0, 40, 120, 250]),
decoy: choose(["none", "disabled-input", "duplicate-text"]),
fieldOrder: choose(["text-first", "select-first"]),
idSuffix: hashSeed(`${seed}:id`).toString(36).slice(0, 6),
};
}
-152
View File
@@ -1,152 +0,0 @@
import { createServer } from "node:http";
import { loadFixtureRegistry } from "./fixture-registry.mjs";
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]{1,120}$/;
function validRunId(value) {
return typeof value === "string" && RUN_ID_PATTERN.test(value);
}
function json(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "application/json; charset=utf-8",
});
response.end(`${JSON.stringify(value, null, 2)}\n`);
}
function html(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "text/html; charset=utf-8",
});
response.end(value);
}
async function readJson(request) {
let raw = "";
for await (const chunk of request) {
raw += chunk;
if (raw.length > 64 * 1024) {
throw new Error("request body exceeds 64 KiB");
}
}
return raw ? JSON.parse(raw) : {};
}
export function createEvalServer({ host = "127.0.0.1", port = 0, fixtureRegistry } = {}) {
const runs = new Map();
let sequence = 0;
let server;
let registry = fixtureRegistry;
function eventsFor(runId) {
if (!runs.has(runId)) runs.set(runId, []);
return runs.get(runId);
}
function record(runId, type, data = {}, metadata = {}) {
if (!validRunId(runId)) return null;
const event = {
sequence: ++sequence,
timestamp: new Date().toISOString(),
type,
data,
...metadata,
};
eventsFor(runId).push(event);
return event;
}
function snapshot(runId) {
return { runId, events: [...(runs.get(runId) ?? [])] };
}
function reset(runId) {
runs.set(runId, []);
}
async function handle(request, response) {
const origin = `http://${request.headers.host ?? `${host}:${port}`}`;
const url = new URL(request.url ?? "/", origin);
if (request.method === "GET" && url.pathname === "/healthz") {
return json(response, 200, { ok: true });
}
if (request.method === "POST" && url.pathname === "/api/events") {
const body = await readJson(request);
if (!validRunId(body.runId) || typeof body.type !== "string") {
return json(response, 400, { error: "runId and type are required" });
}
const event = record(body.runId, body.type, body.data ?? {}, {
path: typeof body.path === "string" ? body.path : undefined,
});
return json(response, 202, { accepted: true, sequence: event.sequence });
}
const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/);
if (runMatch) {
const runId = decodeURIComponent(runMatch[1]);
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
if (request.method === "GET") return json(response, 200, snapshot(runId));
if (request.method === "DELETE") {
reset(runId);
return json(response, 200, { runId, reset: true });
}
}
if (request.method === "GET" && url.pathname === "/api/ping") {
const runId = url.searchParams.get("run");
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
record(runId, "network.ping", { token: "NETWORK-73" }, { path: url.pathname });
return json(response, 200, { ok: true, token: "NETWORK-73" });
}
if (request.method !== "GET") return json(response, 405, { error: "method not allowed" });
const runId = url.searchParams.get("run") ?? "manual";
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
const fixture = registry.render(url.pathname, {
runId,
query: url.searchParams,
record: (type, data, metadata) => record(runId, type, data, metadata),
});
if (!fixture) return html(response, 404, "<!doctype html><h1>Not found</h1>");
record(runId, "page.request", { path: url.pathname }, { path: url.pathname });
return html(response, 200, fixture);
}
return {
record,
reset,
snapshot,
async start() {
if (server) throw new Error("eval server is already running");
registry ??= await loadFixtureRegistry();
server = createServer((request, response) => {
handle(request, response).catch((error) => {
json(response, 500, { error: error instanceof Error ? error.message : String(error) });
});
});
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, host, resolve);
});
const address = server.address();
if (!address || typeof address === "string") throw new Error("unexpected server address");
return { host, port: address.port, baseUrl: `http://${host}:${address.port}` };
},
async stop() {
if (!server) return;
const active = server;
server = undefined;
await new Promise((resolve, reject) => {
active.close((error) => (error ? reject(error) : resolve()));
active.closeAllConnections?.();
});
},
};
}
-73
View File
@@ -1,73 +0,0 @@
function durationFor(result) {
if (result.execution?.durationMs !== undefined) return result.execution.durationMs;
return (result.steps ?? []).reduce((sum, step) => sum + step.durationMs, 0);
}
export function summarizeReports(reports) {
const groups = new Map();
for (const report of reports) {
for (const result of report.results ?? []) {
const adapter = result.adapter ?? report.adapter ?? "unknown";
const variant = result.variant ?? report.variant ?? "unspecified";
const key = `${adapter}\u0000${variant}`;
if (!groups.has(key)) {
groups.set(key, {
adapter,
variant,
runs: 0,
passed: 0,
fullyVerified: 0,
executionFailures: 0,
errors: 0,
durationMs: 0,
toolCalls: 0,
toolCallSamples: 0,
});
}
const group = groups.get(key);
group.runs += 1;
group.durationMs += durationFor(result);
group.errors += result.metrics?.errorCount ?? (result.executionError ? 1 : 0);
if (result.metrics?.toolCallCount !== null && result.metrics?.toolCallCount !== undefined) {
group.toolCalls += result.metrics.toolCallCount;
group.toolCallSamples += 1;
}
const executionFailed =
Boolean(result.executionError) ||
result.execution?.timedOut ||
(result.execution?.exitCode !== undefined && result.execution.exitCode !== 0);
if (executionFailed) group.executionFailures += 1;
if (result.verification?.status !== "failed" && !executionFailed) group.passed += 1;
if (result.verification?.status === "passed" && !executionFailed) group.fullyVerified += 1;
}
}
return [...groups.values()].map((group) => ({
adapter: group.adapter,
variant: group.variant,
runs: group.runs,
passRate: group.runs ? group.passed / group.runs : 0,
fullyVerifiedRate: group.runs ? group.fullyVerified / group.runs : 0,
executionFailures: group.executionFailures,
errors: group.errors,
averageToolCalls: group.toolCallSamples
? Math.round((group.toolCalls / group.toolCallSamples) * 10) / 10
: null,
averageDurationMs: group.runs ? Math.round(group.durationMs / group.runs) : 0,
}));
}
export function printSummary(rows) {
const printable = rows.map((row) => ({
adapter: row.adapter,
variant: row.variant,
runs: row.runs,
pass: `${Math.round(row.passRate * 100)}%`,
verified: `${Math.round(row.fullyVerifiedRate * 100)}%`,
exec_failures: row.executionFailures,
errors: row.errors,
avg_tools: row.averageToolCalls ?? "-",
avg_ms: row.averageDurationMs,
}));
console.table(printable);
}
-48
View File
@@ -1,48 +0,0 @@
import { loadCaseManifests } from "./case-loader.mjs";
let casesPromise;
export function allTasks({ reload = false } = {}) {
if (reload || !casesPromise) casesPromise = loadCaseManifests();
return casesPromise;
}
export function listTasks(cases) {
return cases.map(({ id, title, suite, tags, seed }) => ({ id, title, suite, tags, seed }));
}
export function getTask(taskId, cases) {
const task = cases.find(({ id }) => id === taskId);
if (!task) {
const ids = cases.map(({ id }) => id).join(", ");
throw new Error(`unknown case ${JSON.stringify(taskId)}; expected one of: ${ids}`);
}
return task;
}
export function taskUrl(task, baseUrl, runId, seed = task.seed) {
const url = new URL(task.startPath, baseUrl);
url.searchParams.set("run", runId);
if (seed !== undefined) url.searchParams.set("seed", String(seed));
return url.toString();
}
function expandTemplate(template, values) {
return Object.entries(values).reduce(
(result, [name, value]) => result.replaceAll(`{${name}}`, String(value)),
template,
);
}
export function renderPrompt(task, { baseUrl, runId, locale = "zh-CN", seed = task.seed }) {
const template = task.prompts[locale];
if (!template) {
throw new Error(`unsupported locale ${JSON.stringify(locale)}; expected en or zh-CN`);
}
return expandTemplate(template, {
baseUrl,
runId: encodeURIComponent(runId),
seed: seed ?? "",
url: taskUrl(task, baseUrl, runId, seed),
});
}
-76
View File
@@ -1,76 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://browser-skill.local/schemas/browser-eval-case.schema.json",
"title": "BrowserSkill browser evaluation case",
"type": "object",
"required": [
"schemaVersion",
"id",
"title",
"suite",
"tags",
"fixture",
"prompts",
"coverage",
"assertions",
"smoke"
],
"properties": {
"$schema": { "type": "string" },
"schemaVersion": { "const": 1 },
"id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"title": { "type": "string", "minLength": 1 },
"order": { "type": "integer", "minimum": 0 },
"suite": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"tags": {
"type": "array",
"items": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" },
"uniqueItems": true
},
"fixture": {
"type": "object",
"required": ["startPath"],
"properties": { "startPath": { "type": "string", "pattern": "^/" } },
"additionalProperties": false
},
"prompts": {
"type": "object",
"required": ["en", "zh-CN"],
"properties": {
"en": { "type": "string", "minLength": 1 },
"zh-CN": { "type": "string", "minLength": 1 }
},
"additionalProperties": false
},
"coverage": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
"assertions": {
"type": "object",
"required": ["site", "response", "adapter"],
"properties": {
"site": { "type": "array", "items": { "type": "object" } },
"response": { "type": "array", "items": { "type": "object" } },
"adapter": { "type": "array", "items": { "type": "object" } }
},
"additionalProperties": false
},
"smoke": {
"type": "object",
"required": ["steps"],
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"required": ["action"],
"properties": { "action": { "type": "string" } },
"additionalProperties": true
}
}
},
"additionalProperties": false
},
"seed": { "type": "integer" },
"source": { "type": "object" }
},
"additionalProperties": false
}
-328
View File
@@ -1,328 +0,0 @@
import assert from "node:assert/strict";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import test from "node:test";
import { runCommandTask } from "../lib/agent-runner.mjs";
import { loadCaseManifests, validateCaseManifest } from "../lib/case-loader.mjs";
import { buildOperationCoverage } from "../lib/coverage.mjs";
import { loadFixtureRegistry } from "../lib/fixture-registry.mjs";
import { verifyTask } from "../lib/oracle.mjs";
import { runProcess } from "../lib/process.mjs";
import { repositorySummary, validateRepositoryCases } from "../lib/repository-validation.mjs";
import { scaffoldCase } from "../lib/scaffold-case.mjs";
import { describeSeed } from "../lib/seeded-generator.mjs";
import { createEvalServer } from "../lib/server.mjs";
import { summarizeReports } from "../lib/summary.mjs";
import { allTasks, getTask, renderPrompt } from "../lib/tasks.mjs";
const cases = await allTasks();
const fixtureRegistry = await loadFixtureRegistry();
async function withServer(run) {
const server = createEvalServer({ fixtureRegistry });
const info = await server.start();
try {
await run(server, info);
} finally {
await server.stop();
}
}
test("case manifests are discovered, ordered, and grouped into suites", () => {
assert.deepEqual(
cases.map(({ id }) => id),
[
"navigation-history",
"form-controls",
"hover-inspect",
"tabs",
"diagnostics",
"mobile-emulation",
"generated-form",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
const invalid = {
schemaVersion: 1,
id: "invalid-case",
title: "Invalid",
suite: "regression",
tags: ["regression"],
fixture: { startPath: "/invalid" },
prompts: { en: "Open {url}", "zh-CN": "打开 {url}" },
coverage: ["page.teleport"],
assertions: { site: [], response: [], adapter: [] },
smoke: { steps: [{ action: "select", selector: "#choice", values: [] }] },
typo: true,
};
const errors = validateCaseManifest(invalid);
assert.ok(errors.some((error) => error.includes("unknown top-level property typo")));
assert.ok(errors.some((error) => error.includes("unknown operation page.teleport")));
assert.ok(errors.some((error) => error.includes("values must be a non-empty array")));
});
test("fixture server exposes deterministic task and seeded matrix markers", async () => {
await withServer(async (_server, { baseUrl }) => {
const fixtures = [
["/navigation/detail?run=test-markers", "NAV-42"],
["/form?run=test-markers", '<option value="two">Two</option>'],
["/hover-inspect?run=test-markers", "INSPECT-27"],
["/tabs/child?run=test-markers", "TAB-29"],
["/diagnostics?run=test-markers", "CONSOLE-61"],
["/responsive?run=test-markers", "VIEWPORT-84"],
["/matrix/generated-form?run=test-markers&seed=17", "SEED-17"],
];
for (const [path, marker] of fixtures) {
const response = await fetch(`${baseUrl}${path}`);
assert.equal(response.status, 200);
assert.match(await response.text(), new RegExp(marker));
}
});
});
test("server records and resets run-scoped events", async () => {
await withServer(async (server, { baseUrl }) => {
const response = await fetch(`${baseUrl}/api/events`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId: "event-test", type: "example", data: { value: 7 } }),
});
assert.equal(response.status, 202);
assert.equal(server.snapshot("event-test").events[0].data.value, 7);
const resetResponse = await fetch(`${baseUrl}/api/runs/event-test`, { method: "DELETE" });
assert.equal(resetResponse.status, 200);
assert.deepEqual(server.snapshot("event-test").events, []);
});
});
test("form case oracle checks submitted values and separates adapter evidence", () => {
const task = getTask("form-controls", cases);
const events = [
{ type: "form.enter_pressed", data: {} },
{
type: "form.submitted",
data: { text: "agent-parity", notes: "BrowserSkill works", choice: "two" },
},
{ type: "page.shown", data: { path: "/result" } },
];
const partial = verifyTask(task, { events, responseText: "The page says Received!" });
assert.equal(partial.status, "passed-with-unverified");
assert.equal(partial.summary.unverified, 2);
const complete = verifyTask(task, {
events,
responseText: "The page says Received!",
adapterEvidence: { screenshotCreated: true, sessionStopped: true },
});
assert.equal(complete.status, "passed");
});
test("navigation case fails when reload is missing", () => {
const task = getTask("navigation-history", cases);
const verification = verifyTask(task, {
events: [
{ type: "navigation.detail.clicked", data: {} },
{ type: "page.shown", data: { path: "/navigation/start" } },
{ type: "page.shown", data: { path: "/navigation/start" } },
{ type: "page.shown", data: { path: "/navigation/detail" } },
{ type: "page.shown", data: { path: "/navigation/detail" } },
],
responseText: "NAV-42",
});
assert.equal(verification.status, "failed");
assert.equal(
verification.checks.find(({ label }) => label === "detail page was reloaded").status,
"failed",
);
});
test("prompt rendering is run-scoped, localized, and seed-aware", () => {
const task = getTask("generated-form", cases);
const input = { baseUrl: "http://127.0.0.1:4173", runId: "prompt-test", seed: 17 };
assert.match(renderPrompt(task, { ...input, locale: "en" }), /seed=17/);
assert.match(renderPrompt(task, { ...input, locale: "zh-CN" }), /seeded form/);
});
test("command adapter carries a run from process launch through oracle verification", async () => {
await withServer(async (server, serverInfo) => {
const script = `
const [baseUrl, runId] = process.argv.slice(1);
const send = (type, data) => fetch(baseUrl + "/api/events", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId, type, data }),
});
await send("navigation.detail.clicked", {});
await send("page.shown", { path: "/navigation/start" });
await send("page.shown", { path: "/navigation/start" });
await send("page.shown", { path: "/navigation/detail" });
await send("page.shown", { path: "/navigation/detail" });
await send("page.shown", { path: "/navigation/detail", navigationType: "reload" });
console.log("NAV-42");
`;
const result = await runCommandTask({
server,
serverInfo,
task: getTask("navigation-history", cases),
adapter: {
command: process.execPath,
args: ["--input-type=module", "-e", script, "{baseUrl}", "{runId}"],
variant: "fake-test",
},
adapterName: "fake-agent",
configDirectory: process.cwd(),
iteration: 1,
locale: "en",
timeoutMs: 5_000,
});
assert.equal(result.execution.exitCode, 0);
assert.equal(result.verification.status, "passed-with-unverified");
assert.equal(result.eventCount, 6);
});
});
test("report summary keeps partial verification distinct from full verification", () => {
const rows = summarizeReports([
{
results: [
{
adapter: "agent-a",
variant: "granular-28",
execution: { exitCode: 0, durationMs: 100 },
metrics: { errorCount: 0, toolCallCount: 4 },
verification: { status: "passed-with-unverified" },
},
{
adapter: "agent-a",
variant: "granular-28",
execution: { exitCode: 0, durationMs: 300 },
metrics: { errorCount: 1, toolCallCount: 6 },
verification: { status: "passed" },
},
],
},
]);
assert.equal(rows[0].passRate, 1);
assert.equal(rows[0].fullyVerifiedRate, 0.5);
assert.equal(rows[0].averageDurationMs, 200);
assert.equal(rows[0].averageToolCalls, 5);
assert.equal(rows[0].errors, 1);
});
test("coverage inventory contains all 28 operations and three manual lanes", () => {
const coverage = buildOperationCoverage(cases);
const operations = coverage.map(({ operation }) => operation);
assert.equal(operations.length, 28);
assert.equal(new Set(operations).size, 28);
assert.deepEqual(
coverage.filter(({ directSmoke }) => !directSmoke).map(({ operation }) => operation),
["tabs.borrow", "tabs.return", "assist.request-help"],
);
assert.deepEqual(coverage.find(({ operation }) => operation === "interact.fill").smokeCases, [
"form-controls",
"generated-form",
]);
});
test("seeded dimensions are reproducible and vary across seeds", () => {
assert.deepEqual(describeSeed("badcase-17"), describeSeed("badcase-17"));
assert.notDeepEqual(describeSeed("badcase-17"), describeSeed("badcase-18"));
assert.equal(describeSeed(17).seed, 17);
});
test("badcase scaffold creates a loadable, escaped, self-contained case", async () => {
const temporaryRoot = await mkdtemp(join(tmpdir(), "browser-eval-scaffold-"));
try {
const title = "Quoted `title` ${notCode} <unsafe>";
const result = await scaffoldCase({
id: "reported-timeout",
title,
suite: "regression",
source: "issue-123",
rootCasesDirectory: temporaryRoot,
});
const loaded = await loadCaseManifests({ casesDirectory: temporaryRoot });
assert.equal(loaded[0].id, "reported-timeout");
assert.equal(loaded[0].source.reference, "issue-123");
const generatedRegistry = await loadFixtureRegistry({ roots: [temporaryRoot] });
const html = generatedRegistry.render("/cases/reported-timeout", {
runId: "scaffold-test",
query: new URLSearchParams(),
record() {},
});
assert.match(html, /Quoted `title` \$\{notCode\} &lt;unsafe&gt;/);
assert.match(await readFile(result.files.readme, "utf8"), /issue-123/);
} finally {
await rm(temporaryRoot, { recursive: true, force: true });
}
});
test("CLI accepts pnpm separator and supports suite and tag filters", async () => {
const cli = resolve("evals/browser/cli.mjs");
const coverageResult = await runProcess(process.execPath, [cli, "--", "coverage", "--json"], {
timeoutMs: 5_000,
});
assert.equal(coverageResult.exitCode, 0, coverageResult.stderr);
assert.equal(JSON.parse(coverageResult.stdout).length, 28);
const coreResult = await runProcess(
process.execPath,
[cli, "list", "--suite", "core", "--json"],
{
timeoutMs: 5_000,
},
);
assert.equal(coreResult.exitCode, 0, coreResult.stderr);
assert.equal(JSON.parse(coreResult.stdout).length, 6);
const formResult = await runProcess(process.execPath, [cli, "list", "--tag", "form", "--json"], {
timeoutMs: 5_000,
});
assert.equal(formResult.exitCode, 0, formResult.stderr);
assert.deepEqual(
JSON.parse(formResult.stdout).map(({ id }) => id),
["form-controls", "generated-form"],
);
});
test("CLI validation and seed generation are machine-readable", async () => {
const cli = resolve("evals/browser/cli.mjs");
const validation = await runProcess(process.execPath, [cli, "validate", "--json"], {
timeoutMs: 5_000,
});
assert.equal(validation.exitCode, 0, validation.stderr);
assert.equal(JSON.parse(validation.stdout).ok, true);
const generated = await runProcess(
process.execPath,
[cli, "generate", "--seed", "known-badcase", "--json"],
{ timeoutMs: 5_000 },
);
assert.equal(generated.exitCode, 0, generated.stderr);
assert.equal(typeof JSON.parse(generated.stdout).hydrationDelayMs, "number");
const generatedSet = await runProcess(
process.execPath,
[cli, "generate", "--seed", "4,7,14", "--json"],
{ timeoutMs: 5_000 },
);
assert.equal(generatedSet.exitCode, 0, generatedSet.stderr);
assert.deepEqual(
JSON.parse(generatedSet.stdout).map(({ seed }) => seed),
[4, 7, 14],
);
});
-3
View File
@@ -4,9 +4,6 @@
"packageManager": "pnpm@10.17.0",
"scripts": {
"cli:build": "cargo run -p bsk-protocol --bin dump-schema --locked && pnpm -r build",
"eval:browser": "node evals/browser/cli.mjs",
"eval:browser:check": "node evals/browser/cli.mjs validate && node --test evals/browser/tests/*.test.mjs",
"eval:browser:test": "node --test evals/browser/tests/*.test.mjs",
"ext:build": "pnpm --filter @browser-skill/extension build",
"ext:build:zip": "pnpm --filter @browser-skill/extension zip",
"ext:dev": "pnpm --filter @browser-skill/extension dev",
+41 -21
View File
@@ -5,20 +5,41 @@ npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wx
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
## Tools
| Tool | Actions | Purpose |
| Tool | bsk command | Purpose |
| --- | --- | --- |
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
| `browser_session_start` | `bsk session start` | Open an Agent Window session; optional initial URL, window size, and device emulation preset. Returns the session id and makes it the current session. |
| `browser_session_stop` | `bsk session stop` | Stop a session (the current one by default) and close its Agent Window. Only plugin-created sessions can be stopped. |
| `browser_session_list` | — (registry only) | List the sessions this plugin created, marking the current one. Foreign daemon sessions are never shown. |
| `browser_navigate` | `bsk navigate` | Navigate the active tab, with `waitUntil` / timeout control. |
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
@@ -42,7 +63,7 @@ seam degrade silently.
One agent conversation can drive several browser sessions at once:
- `browser_session` with `action: start` returns the session id and makes it the **current session**.
- `browser_session_start` returns the session id and makes it the **current session**.
- Every operation tool accepts an optional `session` argument. When omitted, the call acts on the
current session (the one most recently started or used); when given, that session becomes current.
- Every tool result echoes the session it actually acted on, so the model never has to guess.
@@ -51,9 +72,9 @@ One agent conversation can drive several browser sessions at once:
**Ownership boundary**: the bsk daemon may be shared with other agents, terminals, or dsh
instances. The plugin therefore only ever sees and operates on sessions it created itself —
an explicit `session` argument naming a foreign or unknown id is rejected, the `list` action on
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
can never touch a session owned by another program.
an explicit `session` argument naming a foreign or unknown id is rejected, `browser_session_list`
shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup can never touch
a session owned by another program.
## Installation
@@ -88,7 +109,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
# lazyTools: true # reveal browser_* tools only after the skill is invoked
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
@@ -110,7 +131,7 @@ like the BrowserSkill extension without leaking a single selector into the host
shell's theme cannot bleed back in.
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
`browser_session_start`; disappears when all sessions stop (or the plugin unloads).
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
stage while the next one loads, and is kept on errors so the card does not flash),
@@ -152,11 +173,11 @@ shell's theme cannot bleed back in.
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
an image block, resolves the durable attachment through the client session's authorized
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
custom view keeps the terminal block (command + output) and, when the settled result carries an
image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
The other five tools keep the stock terminal card. The bundle follows the dsh client
Every other `browser_*` tool keeps the stock terminal card. The bundle follows the dsh client
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
lightningcss.
@@ -177,9 +198,8 @@ pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
## Publishing
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id, client
bundle registration, and `cordis.patch.yml` name are the same specifier so
dsh can import and materialize the plugin without a ModuleLoader id mismatch.
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
`dsh-plugin-browserskill`.
Trigger it by pushing a tag that matches `package.json`'s `version`:
@@ -50,10 +50,6 @@
"@deepseek-ai/dsh-client-runtime",
"@deepseek-ai/dsh-client-ui-tool",
"@deepseek-ai/dsh-client-ui-layout"
],
"external": [
"@deepseek-ai/dsh-client-ui-attachment",
"@deepseek-ai/dsh-client-ui-primitives"
]
}
},
@@ -18,15 +18,37 @@ function stripFrontmatter(markdown) {
}
const SUPPORTED_BROWSER_TOOLS = [
"browser_session",
"browser_page",
"browser_inspect",
"browser_interact",
"browser_tabs",
"browser_assist",
"browser_session_start",
"browser_session_stop",
"browser_session_list",
"browser_navigate",
"browser_navigate_back",
"browser_navigate_forward",
"browser_reload",
"browser_wait_for_navigation",
"browser_snapshot",
"browser_observe",
"browser_get_html",
"browser_screenshot",
"browser_console",
"browser_network",
"browser_click",
"browser_hover",
"browser_fill",
"browser_select",
"browser_press",
"browser_tab_list",
"browser_tab_create",
"browser_tab_select",
"browser_tab_close",
"browser_tab_borrow",
"browser_tab_return",
"browser_request_help",
"browser_emulate",
"browser_window_resize",
];
function validateModelFacingSkill(markdown, supportedTools) {
function validateModelFacingSkill(markdown) {
if (/\bbsk\b/i.test(markdown)) {
throw new Error("DSH skill must not expose the internal CLI name");
}
@@ -35,9 +57,9 @@ function validateModelFacingSkill(markdown, supportedTools) {
}
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
const supported = new Set(supportedTools);
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
const unknown = [...mentioned].filter((name) => !supported.has(name));
const missing = supportedTools.filter((name) => !mentioned.has(name));
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
if (unknown.length > 0) {
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
}
@@ -53,11 +75,11 @@ const DESCRIPTION =
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
const content = `${stripFrontmatter(source).trim()}\n`;
validateModelFacingSkill(content, SUPPORTED_BROWSER_TOOLS);
validateModelFacingSkill(content);
const banner =
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
"// Source: skill/SKILL.md (DeepSeek Harness six-tool instructions).\n";
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
writeFileSync(
join(pkg, "src", "skill-content.generated.ts"),
+218 -93
View File
@@ -1,146 +1,271 @@
---
name: browser-skill
description: Browser automation through six injected domain tools.
description: |
Use the injected browser_* tools to automate the user's logged-in Chromium
browser in a managed Agent Window.
---
# browser-skill for DeepSeek Harness
Drive the user's real Chromium browser through the six structured domain tools provided by this
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
Agent Window. Every call must include the tool's `action` field; other fields depend on that action.
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
fields, not command-line flags.
## Tool availability
Loading this skill reveals the browser tools for the rest of the conversation. All browser work
must use the injected tools directly. Do not invoke another process to control the browser. Direct
tool calls preserve session ownership, the live observation window, cancellation, attachments, and
cleanup.
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
The six tools are:
All browser operations must use the injected tools directly. Do not translate the examples in this
document into terminal commands or invoke another process to control the browser. Direct tool calls
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
- `browser_session`: start, stop, or list plugin-owned sessions.
- `browser_page`: navigate, back, forward, reload, or wait for navigation.
- `browser_inspect`: observe, snapshot, html, screenshot, console, or network.
- `browser_interact`: click, hover, fill, select, or press.
- `browser_tabs`: list, create, select, close, borrow, or return tabs.
- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user.
## When to use
- Visit and read pages that require the user's existing browser session.
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
- Work with a user tab after explicitly borrowing it into the Agent Window.
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
personal tab longer than the immediate task requires.
## Mandatory workflow
Every browser task follows this lifecycle:
1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`.
2. Use that session for every later call. Pass `session` explicitly when more than one exists.
3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task.
Treat cleanup like a finally path unless the user explicitly asks to keep the session open.
1. Call `browser_session_start` and retain its returned `sessionId`.
2. Use that session for every browser operation. Pass `session` explicitly when more than one
session exists.
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
Conceptual sequence:
Conceptual tool-call sequence:
```text
browser_session({ action: "start", url: "https://example.com" })
browser_inspect({ action: "observe", session: sessionId })
browser_interact({ action: "click", session: sessionId, target: "@e4" })
browser_inspect({ action: "observe", session: sessionId })
browser_session({ action: "stop", session: sessionId })
browser_session_start({ url: "https://example.com" })
browser_observe({ session: sessionId })
browser_click({ session: sessionId, target: "@e4" })
browser_observe({ session: sessionId })
browser_session_stop({ session: sessionId })
```
The plugin owns only sessions it starts. Never guess or reuse a session id from another program.
When `session` is omitted, most actions use the current owned session, but explicit ids are safer.
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
only those owned sessions. Never guess or reuse a session id from another program. When `session`
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
for multi-session work.
## Stop when the goal is met
Browser work is bounded. Define an observable success condition, take the shortest path, and stop
as soon as success is visible. Do not refresh or click after success. If an action fails twice
without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh
observation and then ask the user.
Browser work is a bounded task, not open-ended browsing.
## Observe and interact
1. Define one observable success condition from the user's request.
2. Take the shortest path: observe, act, then perform at most one relevant check.
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
verification after success.
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
progress, pause instead of brute-forcing.
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
user rather than continuing to click.
Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles,
states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful.
Prefer a fresh ref over a raw CSS selector for interactions.
## Core interaction loop
Refs invalidate after navigation and may become stale after large DOM changes. Observe again after
navigation or a meaningful DOM update before using another ref. If an observation identifies a
hover-triggered surface, hover the trigger, observe again, then interact with the revealed control.
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
before interacting:
Use `browser_interact` as follows:
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
including hints about conditional hover or focus surfaces.
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
insufficient or a stricter tree is useful.
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
4. Observe again after navigation or a meaningful DOM change before using another ref.
- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`.
- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`.
- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing.
- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value
attributes, not necessarily the visible labels.
- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`.
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
trigger, observe again, and then act on the newly visible ref.
## Navigate and wait
Use interaction tools according to their schemas:
Use `browser_page` with these actions:
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
intentional.
- `browser_select`: pass `values` as an array, including for a single option.
- `browser_press`: send a key or combo, optionally focusing `target` first.
- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`.
- `back` and `forward` move one history entry and optionally accept lifecycle wait fields.
- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary.
- `wait` waits for an expected navigation after an interaction that did not already wait.
Actions may trigger a navigation after the action itself returns. Use
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
new page.
Avoid speculative waits when no navigation is expected. After any page change, discard old refs and
observe again.
## Reading priority
## Observation and page reading priority
Escalate page reading only as needed:
1. `browser_inspect` with `action: "observe"` for normal understanding and refs.
2. The same tool with `action: "snapshot"` for a stricter static tree.
3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and
use only a fresh ref when scoping a subtree.
4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref`
crops to an element; omit it for the visible tab.
1. `browser_observe` for normal semantic understanding and interaction refs.
2. `browser_snapshot` for a stricter static accessibility tree.
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
Do not start with raw HTML or a screenshot merely to discover ordinary controls.
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
return an inline image or a saved path depending on the deployment.
## Tabs and the Agent Window boundary
Use `browser_tabs` with these actions:
Write actions normally affect only Agent Window tabs.
- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing.
- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`.
- `select` focuses an Agent Window tab and requires a returned `tabId`.
- `close` closes an Agent Window tab and requires a returned `tabId`.
- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`.
- `return` restores a borrowed tab and requires that borrowed `tabId`.
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
`scope: "user"` to locate a user tab before borrowing it.
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
for a background tab.
- `browser_tab_select` focuses an Agent Window tab.
- `browser_tab_close` closes an Agent Window tab.
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback
cleanup. Never invent a tab id.
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
from page position or title.
## Human help and display controls
## Navigation and lifecycle
Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation,
consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or
selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text
conditions. After the user continues or criteria complete, observe again before reasoning about the
new state.
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
wait. Avoid speculative waits when no navigation is expected.
Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use
`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires
dimensions. Use `off: true` alone to clear emulation. Emulation is per tab.
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
`browser_snapshot` again.
## Read-only diagnostics
## Ask the user for browser help
Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and
`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and
`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries.
Console additionally accepts `includeStack`.
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
adds clarity.
Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not
invent tool names or route around that limitation.
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
```text
browser_request_help({
session: sessionId,
title: "Sign in required",
prompt: "Complete sign-in, then return control after the account page appears.",
targets: ["@e7"],
timeoutMs: 300000,
completionCriteria: {
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
stableForMs: 1000
}
})
```
Interpret outcomes deliberately:
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
again before reasoning about the new state or using refs.
- `cancelled`: stop the requested flow.
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
## Debugging without page scripting
Use the read-only debugging tools only when relevant to the user's task:
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
Set `includeStack: true` when stack frames are useful.
- `browser_network` reads buffered response and failure metadata. It does not return request or
response headers or bodies and does not modify traffic.
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
strictly requires either capability, explain that it is not available in this phase.
## Window and device controls
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
must be from 100 through 7680 CSS pixels.
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
inherit it.
## Tool reference
### Sessions
| Tool | Purpose |
| --- | --- |
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
| `browser_session_list` | List owned sessions and identify the current one. |
### Navigation and reading
| Tool | Purpose |
| --- | --- |
| `browser_navigate` | Navigate the active tab to a URL. |
| `browser_navigate_back` | Go back one history entry. |
| `browser_navigate_forward` | Go forward one history entry. |
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
| `browser_observe` | Read the primary semantic page representation with refs. |
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
### Interaction
| Tool | Purpose |
| --- | --- |
| `browser_click` | Click a ref or CSS selector. |
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
| `browser_fill` | Fill a text-capable element. |
| `browser_select` | Select one or more option values. |
| `browser_press` | Send a keyboard key or combo. |
### Tabs
| Tool | Purpose |
| --- | --- |
| `browser_tab_list` | List visible user and Agent Window tabs. |
| `browser_tab_create` | Create an Agent Window tab. |
| `browser_tab_select` | Focus an Agent Window tab. |
| `browser_tab_close` | Close an Agent Window tab. |
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
| `browser_tab_return` | Return a borrowed tab to its original window. |
### Assistance, diagnostics, and display
| Tool | Purpose |
| --- | --- |
| `browser_request_help` | Pause for an explicit user browser step. |
| `browser_console` | Read buffered console and exception entries. |
| `browser_network` | Read buffered response and failure metadata. |
| `browser_window_resize` | Resize the Agent Window. |
| `browser_emulate` | Apply or clear per-tab device emulation. |
## Error handling and final checklist
On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On
an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide
whether one longer purposeful wait is justified, and never immediately repeat a failed history
action without first checking page state.
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
actions with unchanged state.
Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are
returned when practical, the owned session is stopped, and no extra page action occurs after
success.
Before finishing a browser task, verify that:
- The user's observable goal is met, or the blocker is clearly reported.
- Any borrowed tab has been returned when practical.
- The owned session has been stopped unless the user explicitly asked to keep it open.
- No further page action is taken after success.
@@ -7,7 +7,7 @@
* are hidden from every surface, so their Agent Windows would otherwise
* linger with no way back to them.
*
* Ownership lineage: browser_session action=start records the calling agent's
* Ownership lineage: browser_session_start records the calling agent's
* session id PLUS every ancestor along `header.parentSession` (a subagent's
* browsers are reaped when any ancestor is archived, the root conversation
* included). Lineage is resolved through the host session store; an
@@ -1,334 +0,0 @@
/**
* The six model-facing BrowserSkill tools. Each action dispatches to a private
* operation handler, so the model receives only six schemas while every action
* retains the existing ownership, cancellation, queuing, observation,
* screenshot, and presentation behavior.
*/
import { defineTool, type ParameterSchemaSpec, type ToolDefinition } from "@deepseek-ai/dsh-tools";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
import { createBrowserOperationDefinitions, type ToolDeps } from "./tools";
const DEVICE_PRESETS = [
"iphone-14",
"iphone-14-pro-max",
"iphone-se",
"pixel-7",
"galaxy-s23",
"ipad-mini",
"galaxy-tab-s8",
] as const;
const TARGET_PARAM = {
type: "string",
description: "Snapshot ref such as @e3, or a CSS selector.",
} as const;
const DOMAIN_RESULT = { type: "json" } as const;
interface BrowserToolSpec {
name: string;
description: string;
actions: Record<string, string>;
parameters: ParameterSchemaSpec;
}
function operationArgs(args: unknown): Record<string, unknown> {
const { action: _action, ...rest } = args as Record<string, unknown>;
return rest;
}
function definitionFor(
definitions: ReadonlyMap<string, ToolDefinition>,
actions: Record<string, string>,
action: unknown,
): ToolDefinition {
const operationName = typeof action === "string" ? actions[action] : undefined;
const definition = operationName === undefined ? undefined : definitions.get(operationName);
if (definition === undefined) throw new Error(`unsupported browser action: ${String(action)}`);
return definition;
}
function defineBrowserTool(
spec: BrowserToolSpec,
definitions: ReadonlyMap<string, ToolDefinition>,
): ToolDefinition {
const actionNames = Object.keys(spec.actions);
return defineTool({
name: spec.name,
description: spec.description,
parameters: {
action: {
type: "string",
required: true,
enum: actionNames,
description:
"Operation to perform. Other arguments are action-dependent and are validated by that operation.",
},
...spec.parameters,
},
output: {
schema: DOMAIN_RESULT,
render(args, value) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.output.render(operationArgs(args), value);
},
},
isConcurrencySafe(args) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.isConcurrencySafe?.(operationArgs(args)) === true;
},
async execute(args, exec) {
const definition = definitionFor(definitions, spec.actions, args.action);
return (await definition.execute(operationArgs(args), exec)) as never;
},
presentCall(args) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.presentCall?.(operationArgs(args));
},
presentResult(args, result) {
const definition = definitionFor(definitions, spec.actions, args.action);
return definition.presentResult?.(operationArgs(args), result);
},
});
}
const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{
name: "browser_session",
description:
"Manage plugin-owned browser sessions. Actions: start opens an Agent Window; stop closes an " +
"owned session; list returns owned sessions. For start, url/device/width/height/noFocus/browser " +
"are optional. For stop, session is optional and defaults to the current owned session.",
actions: {
start: "session.start",
stop: "session.stop",
list: "session.list",
},
parameters: {
session: SESSION_PARAM,
url: { type: "string", description: "Initial URL for start." },
width: { type: "integer", description: "Agent Window width; start requires height too." },
height: { type: "integer", description: "Agent Window height; start requires width too." },
noFocus: { type: "boolean", description: "Start the Agent Window in the background." },
browser: { type: "string", description: "Browser instance id for start." },
device: { type: "string", enum: DEVICE_PRESETS, description: "Device preset for start." },
},
},
{
name: "browser_page",
description:
"Navigate and wait on the active Agent Window tab. Actions: navigate, back, forward, reload, " +
"wait. navigate requires url; reload optionally accepts hard; navigation actions accept " +
"waitUntil/timeoutMs. Observe again after a meaningful page change before reusing refs.",
actions: {
navigate: "page.navigate",
back: "page.back",
forward: "page.forward",
reload: "page.reload",
wait: "page.wait",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
url: { type: "string", description: "Destination URL; required for navigate." },
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
hard: { type: "boolean", description: "Bypass cache for reload." },
},
},
{
name: "browser_inspect",
description:
"Read page state without arbitrary script execution. Actions: observe, snapshot, html, " +
"screenshot, console, network. Prefer observe, then snapshot, then bounded html; use screenshot " +
"for visual evidence. console/network support cursor fields since/limit/maxTextChars.",
actions: {
observe: "inspect.observe",
snapshot: "inspect.snapshot",
html: "inspect.html",
screenshot: "inspect.screenshot",
console: "inspect.console",
network: "inspect.network",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
maxDepth: { type: "integer", description: "Tree depth cap for observe/snapshot." },
maxTokens: { type: "integer", description: "Token cap for observe/snapshot." },
ref: { type: "string", description: "Fresh ref for scoped html or cropped screenshot." },
maxBytes: { type: "integer", description: "HTML byte cap." },
since: { type: "integer", description: "Console/network sequence cursor." },
limit: { type: "integer", description: "Console/network entry cap." },
maxTextChars: { type: "integer", description: "Console/network per-entry text cap." },
includeStack: { type: "boolean", description: "Include console stack frames." },
},
},
{
name: "browser_interact",
description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.",
actions: {
click: "interact.click",
hover: "interact.hover",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
target: TARGET_PARAM,
button: { type: "string", enum: ["left", "middle", "right"], description: "Click button." },
clickCount: { type: "integer", description: "Click count." },
value: { type: "string", description: "Text for fill." },
noClear: { type: "boolean", description: "Append instead of clearing for fill." },
modifiers: {
type: "array",
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
description: "Modifiers held during hover.",
},
settleMs: { type: "integer", description: "Hover settle delay." },
timeoutMs: TIMEOUT_MS_PARAM,
values: {
type: "array",
items: { type: "string" },
description: "Option values for select; at least one is required.",
},
key: { type: "string", description: "Key or combo for press." },
holdMs: { type: "integer", description: "Key hold duration for press." },
},
},
{
name: "browser_tabs",
description:
"Manage tabs visible to an owned session. Actions: list, create, select, close, borrow, return. " +
"select/close/borrow/return require tabId from list/create. Borrow moves a user tab into the " +
"Agent Window; return it as soon as the task finishes.",
actions: {
list: "tabs.list",
create: "tabs.create",
select: "tabs.select",
close: "tabs.close",
borrow: "tabs.borrow",
return: "tabs.return",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
scope: {
type: "string",
enum: ["user", "agent", "all"],
description: "Tab scope for list.",
},
url: { type: "string", description: "Initial URL for create." },
active: { type: "boolean", description: "Focus the created tab (default true)." },
index: { type: "integer", description: "Insertion index for create." },
},
},
{
name: "browser_assist",
description:
"Display and human-assistance operations. Actions: resize, emulate, request-help. resize " +
"requires width/height; emulate accepts device or width/height/mobile, or off alone; " +
"request-help requires prompt and can wait for explicit completion criteria.",
actions: {
resize: "assist.resize",
emulate: "assist.emulate",
"request-help": "assist.request-help",
},
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
width: { type: "integer", description: "Window/viewport width." },
height: { type: "integer", description: "Window/viewport height." },
device: { type: "string", enum: DEVICE_PRESETS, description: "Emulation device preset." },
mobile: { type: "boolean", description: "Enable a mobile viewport with width/height." },
off: { type: "boolean", description: "Clear emulation; use alone." },
prompt: { type: "string", description: "Instructions shown to the user for request-help." },
title: { type: "string", description: "Optional request-help title." },
targets: {
type: "array",
items: { type: "string" },
description: "Refs/selectors highlighted for request-help.",
},
timeoutMs: TIMEOUT_MS_PARAM,
completionCriteria: {
type: "object",
additionalProperties: false,
description: "Automatic completion detector for request-help.",
properties: {
any: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
},
},
all: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
},
},
stableForMs: { type: "integer" },
},
},
},
},
];
function indexOperations(definitions: ToolDefinition[]): ReadonlyMap<string, ToolDefinition> {
const indexed = new Map<string, ToolDefinition>();
for (const definition of definitions) {
if (indexed.has(definition.name)) {
throw new Error(`duplicate browser operation definition: ${definition.name}`);
}
indexed.set(definition.name, definition);
}
const routed = new Set(BROWSER_TOOL_SPECS.flatMap((spec) => Object.values(spec.actions)));
const missing = [...routed].filter((name) => !indexed.has(name));
const unreachable = [...indexed.keys()].filter((name) => !routed.has(name));
if (missing.length > 0 || unreachable.length > 0) {
throw new Error(
[
missing.length > 0 ? `missing operations: ${missing.join(", ")}` : undefined,
unreachable.length > 0 ? `unreachable operations: ${unreachable.join(", ")}` : undefined,
]
.filter(Boolean)
.join("; "),
);
}
return indexed;
}
/** Register the complete six-tool browser suite; returns its combined disposer. */
export function registerBrowserTools(deps: ToolDeps): () => void {
const definitions = indexOperations(createBrowserOperationDefinitions(deps));
const disposers = BROWSER_TOOL_SPECS.map((spec) =>
deps.ctx.tools.register(defineBrowserTool(spec, definitions)),
).filter((dispose): dispose is () => void => typeof dispose === "function");
return () => {
for (const dispose of disposers.splice(0)) dispose();
};
}
@@ -1,4 +1,4 @@
/* browser_inspect screenshot toolview: disclosure row over a terminal block plus the
/* browser_screenshot toolview: disclosure row over a terminal block plus the
screenshot image. Semantic tokens only (--dsw-alias-*), no literal colors. */
.card {
@@ -1,6 +1,8 @@
// browser_inspect keyed toolview: all inspect actions retain a terminal card;
// screenshot results additionally render the image attachment. Pure function
// of the frozen call/result slice, so live and replay paths stay identical.
// browser_screenshot keyed toolview: the terminal card's command line and
// output stay exactly as the host renders them, and a result carrying an
// image block additionally renders the screenshot itself (thumbnail with the
// shared click-to-open lightbox). Pure function of the frozen call/result
// slice — live and replay paths render identically.
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
import {
@@ -17,13 +19,13 @@ import {
} from "@deepseek-ai/dsh-client-ui-primitives";
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
import { useState } from "react";
import css from "./BrowserInspectToolView.module.css";
import css from "./ScreenshotToolView.module.css";
/** Resolved loader bound to the owning session at registration time. */
export type BrowserInspectImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
export type ScreenshotImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
export type BrowserInspectToolViewProps = ToolCallViewProps & {
loadImage: BrowserInspectImageLoader;
export type ScreenshotToolViewProps = ToolCallViewProps & {
loadImage: ScreenshotImageLoader;
};
type ViewState = "running" | "ok" | "error";
@@ -34,7 +36,6 @@ interface ViewModel {
readonly output: string | null;
readonly image: ImageAttachmentRef | null;
readonly summary: string;
readonly title: string;
}
const TERMINAL_LABELS: Partial<TerminalBlockLabels> = {
@@ -65,71 +66,18 @@ function firstLine(text: string): string {
}
/** Rebuild the command line from the logged arguments (mirrors the host presenter). */
const INSPECT_COMMANDS: Record<string, string> = {
observe: "observe",
snapshot: "snapshot",
html: "get-html",
screenshot: "screenshot",
console: "console",
network: "network",
};
function titleOf(action: string): string {
return action === "html" ? "HTML" : `${action.slice(0, 1).toUpperCase()}${action.slice(1)}`;
}
interface InspectArgs {
action?: unknown;
session?: unknown;
tabId?: unknown;
maxDepth?: unknown;
maxTokens?: unknown;
ref?: unknown;
maxBytes?: unknown;
since?: unknown;
limit?: unknown;
maxTextChars?: unknown;
includeStack?: unknown;
}
function appendNumber(parts: string[], flag: string, value: unknown): void {
if (typeof value === "number" && Number.isFinite(value)) parts.push(flag, String(value));
}
function commandOf(argsRaw: string, callId: string): { command: string; title: string } {
function commandOf(argsRaw: string, callId: string): string {
try {
const args = JSON.parse(argsRaw) as InspectArgs;
const action = typeof args.action === "string" ? args.action : "inspect";
const command = INSPECT_COMMANDS[action] ?? action;
const parts = ["bsk", command, "--session"];
const args = JSON.parse(argsRaw) as { session?: unknown; ref?: unknown };
const parts = ["bsk", "screenshot", "--session"];
parts.push(
typeof args.session === "string" && args.session !== "" ? args.session : "(current)",
);
if (action === "observe" || action === "snapshot") {
appendNumber(parts, "--max-depth", args.maxDepth);
appendNumber(parts, "--max-tokens", args.maxTokens);
}
if (action === "html" || action === "console" || action === "network") {
appendNumber(parts, "--tab-id", args.tabId);
}
if ((action === "html" || action === "screenshot") && typeof args.ref === "string") {
if (args.ref !== "") parts.push("--ref", args.ref);
}
if (action === "html") appendNumber(parts, "--max-bytes", args.maxBytes);
if (action === "console" || action === "network") {
appendNumber(parts, "--since", args.since);
appendNumber(parts, "--limit", args.limit);
appendNumber(parts, "--max-text-chars", args.maxTextChars);
}
if (action === "console" && args.includeStack === true) parts.push("--include-stack");
return { command: parts.join(" "), title: titleOf(action) };
if (typeof args.ref === "string" && args.ref !== "") parts.push("--ref", args.ref);
return parts.join(" ");
} catch {
// Streaming can expose a truncated JSON prefix; it is still the best label.
return {
command:
argsRaw === "" ? `browser_inspect (${callId})` : `browser_inspect ${firstLine(argsRaw)}`,
title: "Inspect",
};
return argsRaw === "" ? `bsk screenshot (${callId})` : `bsk screenshot ${firstLine(argsRaw)}`;
}
}
@@ -137,9 +85,9 @@ function commandOf(argsRaw: string, callId: string): { command: string; title: s
export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
const settled = "kind" in block;
const argsRaw = (settled ? block.call?.argsRaw : block.argsRaw) ?? "";
const { command, title } = commandOf(argsRaw, block.callId);
const command = commandOf(argsRaw, block.callId);
if (!settled) {
return { state: "running", command, output: null, image: null, summary: command, title };
return { state: "running", command, output: null, image: null, summary: command };
}
const textBlock = block.content.find((item) => item.type === "text");
const imageBlock = block.content.find((item) => item.type === "image");
@@ -148,7 +96,7 @@ export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
imageBlock !== undefined && imageBlock.type === "image" ? imageBlock.attachment : null;
const state: ViewState = block.isError ? "error" : "ok";
const summary = output !== null ? firstLine(output) : command;
return { state, command, output, image, summary, title };
return { state, command, output, image, summary };
}
function dotState(state: ViewState): StateDotState {
@@ -163,24 +111,19 @@ function dotState(state: ViewState): StateDotState {
}
/**
* Render one browser_inspect call: a disclosure row over a terminal block,
* plus the screenshot image when that action returns an image attachment.
* Render one browser_screenshot call: a disclosure row over a terminal block
* (command + output) plus the screenshot image when the result carries one.
*/
export function BrowserInspectToolView({
block,
cwd,
inspect,
loadImage,
}: BrowserInspectToolViewProps) {
export function ScreenshotToolView({ block, cwd, inspect, loadImage }: ScreenshotToolViewProps) {
const model = viewModelOf(block);
const [expanded, setExpanded] = useState(false);
const expandable = model.state === "running" || model.output !== null || model.image !== null;
const open = expanded && expandable;
return (
<div className={css.card} data-tool="browser_inspect" data-state={model.state}>
<div className={css.card} data-tool="browser_screenshot" data-state={model.state}>
<DisclosureRow
icon={<StateDot state={dotState(model.state)} />}
title={model.title}
title="Screenshot"
open={open}
expandable={expandable}
onToggle={() => setExpanded((value) => !value)}
@@ -1,6 +1,6 @@
/**
* dsh-plugin-browserskill browser half: the `browser_inspect` keyed toolview
* plus the observation overlay (live thumbnails + interrupt). The
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
* toolview plus the observation overlay (live thumbnails + interrupt). The
* default carrier is a floating card on the `shell.overlay` seat; when the
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
* moves into a sidebar tab instead (see observation-sidebar.tsx).
@@ -16,10 +16,10 @@ import { createElement } from "react";
// <style> tags; selectors stay unhashed so `cn(..., "bsk-obs")` roots match).
import "./bsk-tokens.nomodule.css";
import "./bsk-ui.nomodule.css";
import { BrowserInspectToolView } from "./BrowserInspectToolView";
import { ObservationOverlay } from "./ObservationOverlay";
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
import { ScreenshotToolView } from "./ScreenshotToolView";
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
export const inject = ["slots", "sessions"];
@@ -65,9 +65,9 @@ export function apply(ctx: ClientContext): void {
const sessions = ctx.get("sessions") as unknown as ISessions;
ctx.slots.inject("tool.call.toolview", () =>
ctx.slots.register(
{ name: "tool.call.toolview", key: "browser_inspect" },
{ name: "tool.call.toolview", key: "browser_screenshot" },
(props: ToolCallViewProps) =>
createElement(BrowserInspectToolView, {
createElement(ScreenshotToolView, {
...props,
loadImage: (attachment: ImageAttachmentRef) =>
loadSessionImage(sessions, props.sessionId, attachment),
+9 -65
View File
@@ -1,11 +1,10 @@
/**
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
* file path plus pixel metadata); when the host mounts a durable attachment
* store AND the calling route declares image input, the capture bytes are
* additionally committed through `ctx.attachments` so the render step can
* attach the image itself to the tool result. Any uncertainty (no store,
* unknown route, text-only model, unrecognized bytes) falls back to the
* path-only form.
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
* path plus pixel metadata); when the host mounts a durable attachment store
* AND the calling route declares image input, the PNG bytes are additionally
* committed through `ctx.attachments` so the render step can attach the image
* itself to the tool result. Any uncertainty (no store, unknown route,
* text-only model) falls back to the path-only form.
*/
import type { Context } from "@deepseek-ai/cordis";
@@ -34,54 +33,6 @@ interface LlmLike {
): Promise<{ inputModalities?: readonly string[] }>;
}
/**
* Sniff the image media type from magic bytes. The capture path can hand back
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
* returning JPEG regardless of the requested format), so the declared type can
* never be trusted from the file extension alone.
* @param data - first bytes of the capture.
* @returns the detected media type, or undefined when unrecognized.
*/
function sniffImageMediaType(data: Uint8Array): string | undefined {
if (data.length >= 4) {
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
return "image/png";
}
// JPEG SOI: FF D8 FF
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
return "image/jpeg";
}
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
if (
data[0] === 0x52 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x46 &&
data.length >= 12 &&
data[8] === 0x57 &&
data[9] === 0x45 &&
data[10] === 0x42 &&
data[11] === 0x50
) {
return "image/webp";
}
// GIF: "GIF87a" or "GIF89a"
if (
data[0] === 0x47 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x38 &&
data.length >= 6 &&
(data[4] === 0x37 || data[4] === 0x39) &&
data[5] === 0x61
) {
return "image/gif";
}
}
return undefined;
}
/**
* Commit screenshot bytes to the host attachment store when the composition
* supports durable images on the current model route.
@@ -95,14 +46,7 @@ export async function trySaveScreenshot(
): Promise<ImageAttachmentRef | undefined> {
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
if (attachments === undefined) return undefined;
// Declare the real media type from the bytes, never a hard-coded "image/png".
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
// silently degrades to path-only; sniffing first lets JPEG captures (returned
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
// inline correctly.
const mediaType = sniffImageMediaType(data);
if (mediaType === undefined) return undefined;
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
if (
data.byteLength >
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
@@ -111,9 +55,9 @@ export async function trySaveScreenshot(
}
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
try {
return await attachments.saveImage({ data, mediaType, name });
return await attachments.saveImage({ data, mediaType: "image/png", name });
} catch {
// A store hiccup must not fail the tool call; the path-only form still works.
// A store hiccup must not fail the tool call; the PNG path still works.
return undefined;
}
}
@@ -13,7 +13,6 @@
import type { Context } from "@deepseek-ai/cordis";
import Schema from "@deepseek-ai/schemastery";
import { armArchiveCleanup } from "./archive-cleanup";
import { registerBrowserTools } from "./browser-tools";
import { armLazyTools } from "./lazy-tools";
import { ObservationService } from "./observation";
import { registerObservationRoutes } from "./observation-http";
@@ -21,9 +20,9 @@ import { KeyedExecutor } from "./queue";
import { type BskRunner, createBskRunner } from "./runner";
import { SessionRegistry } from "./sessions";
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
import type { PluginConfig } from "./tools";
import { type PluginConfig, registerTools } from "./tools";
export const name = "@wxg-prc-cpg/browser-skill-dsh-plugin";
export const name = "dsh-plugin-browserskill";
export const inject = ["tools"];
/** Runtime configuration schema (validated and defaulted by Cordis). */
@@ -101,9 +100,11 @@ export function apply(
// exact agent context at startup so DSH always loads the browser_* protocol
// instructions; the shared CLI skill remains untouched for other agents.
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
const registerSuite = () =>
registerBrowserTools({ ctx, runner, registry, config: resolved, observation, queue });
const removeSuite = resolved.lazyTools ? armLazyTools(ctx, registerSuite) : registerSuite();
const removeSuite = resolved.lazyTools
? armLazyTools(ctx, () =>
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
)
: registerTools({ ctx, runner, registry, config: resolved, observation, queue });
// Route registration rides ctx.inject: the webServer service may be provided
// AFTER this plugin loads, and in headless compositions it never appears (the
// callback simply never runs, leaving the rest of the plugin unaffected).
@@ -132,7 +133,7 @@ export function apply(
);
// Unload cleanup: kill in-flight children, then stop every session this
// plugin OWNS (created via browser_session action=start). Referenced or unknown
// plugin OWNS (created via browser_session_start). Referenced or unknown
// sessions belonging to other programs on the shared daemon are never
// touched; per-stop failures (already stopped externally, daemon restart)
// are swallowed so one stale handle cannot abort the rest.
@@ -153,7 +154,6 @@ export function apply(
}
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
export { registerBrowserTools } from "./browser-tools";
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
export { ObservationService } from "./observation";
export { registerObservationRoutes } from "./observation-http";
@@ -162,3 +162,4 @@ export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
export { BskError, createBskRunner } from "./runner";
export { SessionRegistry } from "./sessions";
export type { PluginConfig, ToolDeps } from "./tools";
export { registerTools } from "./tools";
@@ -1,6 +1,6 @@
/**
* Lazy tool-schema injection ("progressive disclosure", final stage): with
* `lazyTools` on (the default), the six browser tool schemas stay OUT of the
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
* system prompt until the `browser-skill` skill has actually been invoked —
* the skill catalog entry is the only advertisement. One successful
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
/**
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
* the reveal already happened — the tool suite itself.
* @param registerSuite - registers the six browser tools and returns their disposer.
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
*/
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
let suiteDisposer: (() => void) | undefined;
@@ -161,7 +161,7 @@ export class ObservationService {
this.emit({ type: "upsert", session: { ...entry } });
}
/** Register a fresh owned session (called from browser_session action=start). */
/** Register a fresh owned session (called from browser_session_start). */
addSession(sessionId: string, url?: string): void {
if (!this.deps.options.enabled || this.disposed) return;
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
@@ -176,7 +176,7 @@ export class ObservationService {
this.scheduleCapture(sessionId, 0);
}
/** Drop a session (called from browser_session action=stop). */
/** Drop a session (called from browser_session_stop). */
removeSession(sessionId: string): void {
this.cancelCapture(sessionId);
this.foregroundDepth.delete(sessionId);
@@ -276,7 +276,7 @@ export class ObservationService {
/**
* Stop one owned session and close its Agent Window (the overlay's stop
* button — same end state as `browser_session` action=stop). Never waits behind a
* button — same end state as `browser_session_stop`). Never waits behind a
* hung in-flight command: tool children are killed first so the session's
* keyed queue drains immediately, and no further captures queue up. A
* session the daemon already forgot stops idempotently — the goal state
@@ -1,7 +1,7 @@
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
import type { ToolDeps } from "./tools";
export type ToolRegistrar = (definition: ToolDefinition) => void;
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
@@ -23,10 +23,10 @@ export function registerPhaseOneInteractionTools(
register(
defineTool({
name: "interact.hover",
name: "browser_hover",
description:
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
"Run browser_inspect action=observe or snapshot afterwards to discover newly visible refs.",
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
parameters: {
target: {
type: "string",
@@ -68,7 +68,7 @@ export function registerPhaseOneInteractionTools(
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
requirePositive(args.settleMs, "settleMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=hover)");
const sessionId = registry.resolve(args.session, "browser_hover");
const cmdArgs = ["hover", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.modifiers !== undefined && args.modifiers.length > 0) {
@@ -102,7 +102,7 @@ export function registerPhaseOneInteractionTools(
register(
defineTool({
name: "interact.select",
name: "browser_select",
description:
"Set a select element's option values. Pass one value for a normal select or multiple " +
"values for a multi-select; values replace the current selection.",
@@ -155,7 +155,7 @@ export function registerPhaseOneInteractionTools(
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
if (args.values.length === 0) throw new Error("values must contain at least one option");
const sessionId = registry.resolve(args.session, "browser_interact(action=select)");
const sessionId = registry.resolve(args.session, "browser_select");
const cmdArgs = ["select", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
for (const value of args.values) cmdArgs.push("--value", value);
@@ -51,7 +51,7 @@ export function registerPhaseOneNavigationTools(
const { registry } = deps;
const registerHistory = (direction: "back" | "forward") => {
const toolName = `page.${direction}` as const;
const toolName = `browser_navigate_${direction}` as const;
const command = `navigate-${direction}`;
register(
defineTool({
@@ -106,7 +106,7 @@ export function registerPhaseOneNavigationTools(
register(
defineTool({
name: "page.reload",
name: "browser_reload",
description:
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
"bypass the HTTP cache.",
@@ -132,7 +132,7 @@ export function registerPhaseOneNavigationTools(
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_page(action=reload)");
const sessionId = registry.resolve(args.session, "browser_reload");
const cmdArgs = ["reload", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
@@ -163,7 +163,7 @@ export function registerPhaseOneNavigationTools(
register(
defineTool({
name: "page.wait",
name: "browser_wait_for_navigation",
description:
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
"that may navigate when the action result itself does not wait for navigation.",
@@ -195,7 +195,7 @@ export function registerPhaseOneNavigationTools(
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_page(action=wait)");
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
const timeoutMs = args.timeoutMs ?? 30_000;
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
@@ -143,7 +143,7 @@ export function registerPhaseOneSupportTools(
register(
defineTool({
name: "assist.request-help",
name: "browser_request_help",
description:
"Pause browser automation and ask the user to complete an in-page step such as login, " +
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
@@ -233,7 +233,7 @@ export function registerPhaseOneSupportTools(
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
throw new Error("completionCriteria.stableForMs must be zero or greater");
}
const sessionId = registry.resolve(args.session, "browser_assist(action=request-help)");
const sessionId = registry.resolve(args.session, "browser_request_help");
const timeoutMs = args.timeoutMs ?? 300_000;
const cmdArgs = [
"request-help",
@@ -285,10 +285,10 @@ export function registerPhaseOneSupportTools(
register(
defineTool({
name: "inspect.html",
name: "browser_get_html",
description:
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
"snapshot ref. Use only when browser_inspect observe/snapshot cannot answer the question.",
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
@@ -326,7 +326,7 @@ export function registerPhaseOneSupportTools(
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
}
requirePositive(args.maxBytes, "maxBytes");
const sessionId = registry.resolve(args.session, "browser_inspect(action=html)");
const sessionId = registry.resolve(args.session, "browser_get_html");
const cmdArgs = ["get-html", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
@@ -361,7 +361,7 @@ export function registerPhaseOneSupportTools(
register(
defineTool({
name: "inspect.console",
name: "browser_console",
description:
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
"This is read-only and does not evaluate JavaScript.",
@@ -448,7 +448,7 @@ export function registerPhaseOneSupportTools(
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_inspect(action=console)");
const sessionId = registry.resolve(args.session, "browser_console");
const cmdArgs = ["console", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
if (args.includeStack === true) cmdArgs.push("--include-stack");
@@ -493,7 +493,7 @@ export function registerPhaseOneSupportTools(
register(
defineTool({
name: "inspect.network",
name: "browser_network",
description:
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
"and response headers and bodies are not captured.",
@@ -552,7 +552,7 @@ export function registerPhaseOneSupportTools(
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_inspect(action=network)");
const sessionId = registry.resolve(args.session, "browser_network");
const cmdArgs = ["network", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
@@ -592,7 +592,7 @@ export function registerPhaseOneSupportTools(
register(
defineTool({
name: "assist.resize",
name: "browser_window_resize",
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
parameters: {
width: {
@@ -629,7 +629,7 @@ export function registerPhaseOneSupportTools(
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
throw new Error("width and height must each be in the range 100..=7680");
}
const sessionId = registry.resolve(args.session, "browser_assist(action=resize)");
const sessionId = registry.resolve(args.session, "browser_window_resize");
const reply = (await runtime.run(
exec,
[
@@ -7,7 +7,7 @@ import type { ToolDeps } from "./tools";
const TAB_ID_REQUIRED = {
type: "integer",
required: true,
description: "Chrome tab id returned by browser_tabs list or create.",
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
} as const;
interface RawTabInfo {
@@ -29,7 +29,7 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.list",
name: "browser_tab_list",
description:
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
"hidden; use scope=user to find a user tab before borrowing it.",
@@ -83,7 +83,7 @@ export function registerPhaseOneTabTools(
},
isConcurrencySafe: () => true,
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tabs(action=list)");
const sessionId = registry.resolve(args.session, "browser_tab_list");
const cmdArgs = ["tab", "list", "--session", sessionId];
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
@@ -118,7 +118,7 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.create",
name: "browser_tab_create",
description:
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
"set active=false to open it in the background.",
@@ -148,7 +148,7 @@ export function registerPhaseOneTabTools(
},
async execute(args, exec) {
if (args.url !== undefined) requireNonEmpty(args.url, "url");
const sessionId = registry.resolve(args.session, "browser_tabs(action=create)");
const sessionId = registry.resolve(args.session, "browser_tab_create");
const cmdArgs = ["tab", "create", "--session", sessionId];
if (args.url !== undefined) cmdArgs.push("--url", args.url);
if (args.active === false) cmdArgs.push("--no-active");
@@ -185,7 +185,7 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.close",
name: "browser_tab_close",
description: "Close a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
@@ -202,7 +202,7 @@ export function registerPhaseOneTabTools(
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tabs(action=close)");
const sessionId = registry.resolve(args.session, "browser_tab_close");
const reply = (await runtime.run(
exec,
["tab", "close", String(args.tabId), "--session", sessionId],
@@ -228,7 +228,7 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.select",
name: "browser_tab_select",
description: "Focus a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
@@ -249,7 +249,7 @@ export function registerPhaseOneTabTools(
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tabs(action=select)");
const sessionId = registry.resolve(args.session, "browser_tab_select");
const reply = (await runtime.run(
exec,
["tab", "select", String(args.tabId), "--session", sessionId],
@@ -275,10 +275,10 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.borrow",
name: "browser_tab_borrow",
description:
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
"Return it with browser_tabs action=return; stopping the session also auto-returns it.",
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
@@ -302,7 +302,7 @@ export function registerPhaseOneTabTools(
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tabs(action=borrow)");
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
const reply = (await runtime.run(
exec,
["tab", "borrow", String(args.tabId), "--session", sessionId],
@@ -339,7 +339,7 @@ export function registerPhaseOneTabTools(
register(
defineTool({
name: "tabs.return",
name: "browser_tab_return",
description: "Return a borrowed tab to its original user window and tab-strip position.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
@@ -365,7 +365,7 @@ export function registerPhaseOneTabTools(
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tabs(action=return)");
const sessionId = registry.resolve(args.session, "browser_tab_return");
const reply = (await runtime.run(
exec,
["tab", "return", String(args.tabId), "--session", sessionId],
@@ -7,7 +7,7 @@
*
* Strict ownership boundary (the bsk daemon may be shared with other agents,
* terminals, or dsh instances): the registry ONLY ever holds sessions created
* by this plugin's browser_session start action. Tools cannot see or act on any
* by this plugin's browser_session_start. Tools cannot see or act on any
* other session — an explicit `session` argument naming a foreign id is an
* error, the list tool shows owned sessions only, and stop/unload cleanup
* can never touch a session this plugin did not create.
@@ -48,7 +48,7 @@ export class SessionRegistry {
if (this.sessions.size + this.pendingStarts >= this.maxSessions) {
throw new Error(
`session limit reached (${this.maxSessions} concurrent sessions); ` +
"stop one with browser_session action=stop before starting another",
"stop one with browser_session_stop before starting another",
);
}
this.pendingStarts += 1;
@@ -69,7 +69,7 @@ export class SessionRegistry {
if (!this.sessions.has(session.sessionId) && this.sessions.size >= this.maxSessions) {
throw new Error(
`session limit reached (${this.maxSessions} concurrent sessions); ` +
"stop one with browser_session action=stop before starting another",
"stop one with browser_session_stop before starting another",
);
}
this.sessions.set(session.sessionId, { ...session, owned: true });
@@ -149,7 +149,7 @@ export class SessionRegistry {
private foreignError(sessionId: string, toolName: string): Error {
return new Error(
`${toolName}: session "${sessionId}" does not belong to this plugin — only sessions ` +
"created by browser_session action=start are visible and operable here",
"created by browser_session_start are visible and operable here",
);
}
@@ -168,7 +168,7 @@ export class SessionRegistry {
const current = this.current();
if (current === undefined) {
throw new Error(
`${toolName} needs a session but none is active — use browser_session action=start`,
`${toolName} needs a session but none is active — start one with browser_session_start`,
);
}
this.touch(current);
@@ -184,11 +184,10 @@ export class SessionRegistry {
explicit !== undefined && explicit.trim().length > 0 ? explicit : this.current();
if (candidate === undefined) {
throw new Error(
"browser_session action=stop needs a session but none is active — use action=start first",
"browser_session_stop needs a session but none is active — start one with browser_session_start",
);
}
if (!this.isOwned(candidate))
throw this.foreignError(candidate, "browser_session(action=stop)");
if (!this.isOwned(candidate)) throw this.foreignError(candidate, "browser_session_stop");
return candidate;
}
}
@@ -3,7 +3,7 @@
export const SESSION_PARAM = {
type: "string",
description:
"bsk session id to act on; must be one created by browser_session with action=start. " +
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
"Omit to use the current session (the one most recently started or used).",
} as const;
+47 -54
View File
@@ -1,8 +1,9 @@
/**
* Internal browser operation definitions shared by the six model-facing tools.
* These definitions are never registered with `ctx.tools`: they provide the
* action-level validation, execution, rendering, and UI presentation reused by
* the domain tools without exposing one schema per operation to the model.
* Model-facing tool definitions. Each tool maps to one `bsk <cmd> --json`
* invocation: spawn via the runner, parse the JSON envelope, and return a
* canonical value whose `output.render` produces the model-facing text.
* UI cards reuse the terminal style: the pending card is the command line,
* the completed card carries the output.
*/
import { readFile, unlink } from "node:fs/promises";
@@ -157,15 +158,22 @@ function abortError(): Error {
return error;
}
type DefinitionRegistrar = (definition: ToolDefinition) => void;
/** Define the private action handlers through an injected collector. */
function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): void {
const { registry } = deps;
/** Register the full browser tool suite; returns the combined unregister disposer. */
export function registerTools(deps: ToolDeps): () => void {
const { ctx, registry } = deps;
// Track every registration without touching the cordis context itself
// (contexts are fiber-owned; the lazy path registers from a listener fiber).
const disposers: (() => void)[] = [];
const register = (definition: ToolDefinition): (() => void) => {
const dispose = ctx.tools.register(definition);
// Test doubles occasionally return the registry map instead of a disposer.
if (typeof dispose === "function") disposers.push(dispose);
return dispose;
};
register(
defineTool({
name: "session.start",
name: "browser_session_start",
description:
"Start a new browser session: opens an Agent Window in the connected browser and returns " +
"its session id. The new session becomes the current session for subsequent browser_* calls. " +
@@ -306,10 +314,10 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "session.stop",
name: "browser_session_stop",
description:
"Stop a browser session and close its Agent Window. Stops the given session, or the " +
"current session when `session` is omitted. Only plugin-created sessions " +
"current session when `session` is omitted. Only sessions created by browser_session_start " +
"can be stopped — sessions owned by other programs sharing the bsk daemon are refused.",
parameters: { session: SESSION_PARAM },
output: {
@@ -346,7 +354,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "session.list",
name: "browser_session_list",
description:
"List the browser sessions created by this plugin. Sessions owned by other programs on the " +
"shared bsk daemon are not visible here. The session marked `current` is the one browser_* " +
@@ -411,7 +419,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "page.navigate",
name: "browser_navigate",
description:
"Navigate the session's active tab to a URL and wait for a page lifecycle phase " +
"(default: load). Returns the final URL after redirects.",
@@ -453,7 +461,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
async execute(args, exec) {
if (args.url.trim().length === 0) throw new Error("url must be a non-empty string");
const sessionId = registry.resolve(args.session, "browser_page(action=navigate)");
const sessionId = registry.resolve(args.session, "browser_navigate");
const cmdArgs = ["navigate", "--session", sessionId];
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
@@ -484,14 +492,14 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
const registerObservationTool = (kind: "snapshot" | "observe") => {
const isSnapshot = kind === "snapshot";
const name = isSnapshot ? "inspect.snapshot" : "inspect.observe";
const name = isSnapshot ? "browser_snapshot" : "browser_observe";
const description = isSnapshot
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
"probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
"understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
"alongside them. Read-only: never submits input.";
"probes) with @eN refs. Visual-only surfaces require image understanding and their refs are " +
"screenshot-only; keep using any DOM/AX semantics rendered alongside them. " +
"Read-only: never submits input.";
register(
defineTool({
name,
@@ -560,11 +568,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "interact.click",
name: "browser_click",
description:
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
"last browser_inspect observation, or a CSS selector. A visual Surface requires the " +
"captureId and image coordinates returned by browser_inspect action=screenshot.",
"last browser_snapshot/browser_observe, or a CSS selector. A visual Surface requires the " +
"captureId and image coordinates returned by browser_screenshot.",
parameters: {
target: {
type: "string",
@@ -588,8 +596,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
captureId: {
type: "string",
description:
"Short-lived Surface capture id returned by browser_inspect action=screenshot.",
description: "Short-lived Surface capture id returned by browser_screenshot.",
},
imageX: {
type: "number",
@@ -628,7 +635,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
if (pointArgCount !== 0 && pointArgCount !== 3) {
throw new Error("captureId, imageX, and imageY must be given together");
}
const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
const sessionId = registry.resolve(args.session, "browser_click");
const cmdArgs = ["click", "--session", sessionId];
if (args.button !== undefined) cmdArgs.push("--button", args.button);
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
@@ -671,7 +678,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "interact.fill",
name: "browser_fill",
description:
"Fill an input / textarea / contenteditable element, clearing it first by default. " +
"Target is a snapshot ref (@e3) or a CSS selector.",
@@ -707,7 +714,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
async execute(args, exec) {
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
const sessionId = registry.resolve(args.session, "browser_interact(action=fill)");
const sessionId = registry.resolve(args.session, "browser_fill");
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
if (args.noClear === true) cmdArgs.push("--no-clear");
cmdArgs.push(args.target);
@@ -728,7 +735,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "interact.press",
name: "browser_press",
description:
"Dispatch a keyboard key or combo (Enter, Escape, ArrowDown, Ctrl+A, …) to the session's " +
"active tab, optionally focusing a target element first.",
@@ -767,7 +774,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
async execute(args, exec) {
if (args.key.trim().length === 0) throw new Error("key must be a non-empty string");
const sessionId = registry.resolve(args.session, "browser_interact(action=press)");
const sessionId = registry.resolve(args.session, "browser_press");
const cmdArgs = ["press", "--session", sessionId];
if (args.target !== undefined) {
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
@@ -798,10 +805,10 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "inspect.screenshot",
name: "browser_screenshot",
description:
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
"This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
"This is the supported operation for visual surface refs emitted by browser_observe. Only use " +
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
"not guess content or coordinates. " +
@@ -832,11 +839,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
additionalProperties: false,
properties: {
attachmentId: { type: "string", required: true },
mediaType: {
type: "string",
required: true,
enum: ["image/png", "image/jpeg", "image/webp", "image/gif"],
},
mediaType: { type: "string", required: true, const: "image/png" },
bytes: { type: "integer", required: true },
width: { type: "integer", required: true },
height: { type: "integer", required: true },
@@ -874,7 +877,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
isConcurrencySafe: () => true,
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_inspect(action=screenshot)");
const sessionId = registry.resolve(args.session, "browser_screenshot");
const outPath = join(
tmpdir(),
`bsk-screenshot-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`,
@@ -914,7 +917,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
? {
image: {
attachmentId: String(ref.attachmentId),
mediaType: ref.mediaType,
mediaType: "image/png" as const,
bytes: ref.bytes,
width: ref.width,
height: ref.height,
@@ -943,7 +946,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
register(
defineTool({
name: "assist.emulate",
name: "browser_emulate",
description:
"Apply (or clear) mobile device emulation on the session's active tab: viewport metrics, " +
"user agent, and touch. Overrides are per-tab and not inherited by new tabs.",
@@ -984,7 +987,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_assist(action=emulate)");
const sessionId = registry.resolve(args.session, "browser_emulate");
if (args.off === true) {
if (args.device !== undefined || args.width !== undefined || args.height !== undefined) {
throw new Error("off is mutually exclusive with device/width/height");
@@ -1033,17 +1036,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
commandLine: (args) => cmdline(deps, args),
presentTerminalResult,
});
}
/**
* Build the private action handlers without publishing them to the model. The
* six browser tools dispatch through these handlers, preserving validation,
* rendering, cancellation, ownership, UI presentation, and attachments.
*/
export function createBrowserOperationDefinitions(deps: ToolDeps): ToolDefinition[] {
const definitions: ToolDefinition[] = [];
defineBrowserOperations(deps, (definition) => {
definitions.push(definition);
});
return definitions;
return () => {
for (const dispose of disposers.splice(0)) dispose();
};
}
@@ -1,17 +1,17 @@
// @vitest-environment happy-dom
// browser_inspect keyed toolview: view-model derivation from frozen blocks,
// browser_screenshot keyed toolview: view-model derivation from frozen blocks,
// image/path-only/running/error rendering, the registration key, and the
// session-bound attachment loader.
import type { RunningToolCall, ToolResultNode } from "@deepseek-ai/dsh-client-runtime/client";
import { cleanup, render, screen, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
BrowserInspectToolView,
type BrowserInspectToolViewProps,
viewModelOf,
} from "../../src/client/BrowserInspectToolView";
import { apply } from "../../src/client/index";
import {
ScreenshotToolView,
type ScreenshotToolViewProps,
viewModelOf,
} from "../../src/client/ScreenshotToolView";
const ATTACHMENT = {
attachmentId: "sha256:abc123",
@@ -25,7 +25,7 @@ const ATTACHMENT = {
function runningBlock(argsRaw: string): RunningToolCall {
return {
callId: "c1",
name: "browser_inspect",
name: "browser_screenshot",
argsRaw,
turn: 1,
step: 1,
@@ -41,7 +41,7 @@ function settledBlock(argsRaw: string, content: unknown[], isError = false): Too
seq: 1,
time: 0,
callId: "c1",
call: { name: "browser_inspect", argsRaw },
call: { name: "browser_screenshot", argsRaw },
callTime: 0,
content,
isError,
@@ -58,14 +58,14 @@ afterEach(cleanup);
describe("viewModelOf", () => {
it("derives the running model from the call frame", () => {
const model = viewModelOf(runningBlock('{"action":"screenshot","session":"s1"}'));
const model = viewModelOf(runningBlock('{"session":"s1"}'));
expect(model.state).toBe("running");
expect(model.command).toBe("bsk screenshot --session s1");
expect(model.image).toBeNull();
});
it("marks settled results carrying an image block", () => {
const block = settledBlock('{"action":"screenshot"}', [
const block = settledBlock("{}", [
{ type: "text", text: IMAGE_TEXT },
{ type: "image", attachment: ATTACHMENT },
]);
@@ -77,56 +77,30 @@ describe("viewModelOf", () => {
});
it("keeps the path-only form image-free", () => {
const model = viewModelOf(
settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]),
);
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: PATH_TEXT }]));
expect(model.state).toBe("ok");
expect(model.image).toBeNull();
expect(model.summary).toBe(PATH_TEXT);
});
it("marks error results", () => {
const model = viewModelOf(
settledBlock('{"action":"screenshot"}', [{ type: "text", text: "Error: boom" }], true),
);
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: "Error: boom" }], true));
expect(model.state).toBe("error");
});
it("renders non-screenshot inspect actions as ordinary terminal calls", () => {
const model = viewModelOf(
settledBlock('{"action":"observe","session":"s1"}', [
{ type: "text", text: "page observation" },
]),
);
expect(model.title).toBe("Observe");
expect(model.command).toBe("bsk observe --session s1");
expect(model.image).toBeNull();
});
it("preserves action-specific arguments in diagnostic command cards", () => {
const model = viewModelOf(
runningBlock(
'{"action":"console","session":"s1","tabId":7,"since":4,"limit":20,"maxTextChars":500,"includeStack":true}',
),
);
expect(model.command).toBe(
"bsk console --session s1 --tab-id 7 --since 4 --limit 20 --max-text-chars 500 --include-stack",
);
});
});
describe("BrowserInspectToolView", () => {
const baseProps = { callId: "c1", toolName: "browser_inspect", openFile: () => {} };
describe("ScreenshotToolView", () => {
const baseProps = { callId: "c1", toolName: "browser_screenshot", openFile: () => {} };
const renderView = (
block: BrowserInspectToolViewProps["block"],
loadImage: BrowserInspectToolViewProps["loadImage"],
block: ScreenshotToolViewProps["block"],
loadImage: ScreenshotToolViewProps["loadImage"],
) => {
const props = { ...baseProps, block, loadImage } as unknown as BrowserInspectToolViewProps;
return render(<BrowserInspectToolView {...props} />);
const props = { ...baseProps, block, loadImage } as unknown as ScreenshotToolViewProps;
return render(<ScreenshotToolView {...props} />);
};
it("renders the screenshot image through the loader", async () => {
const block = settledBlock('{"action":"screenshot"}', [
const block = settledBlock("{}", [
{ type: "text", text: IMAGE_TEXT },
{ type: "image", attachment: ATTACHMENT },
]);
@@ -141,7 +115,7 @@ describe("BrowserInspectToolView", () => {
});
it("renders the path-only form without touching the loader", async () => {
const block = settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]);
const block = settledBlock("{}", [{ type: "text", text: PATH_TEXT }]);
const loadImage = vi.fn(async () => "blob:unused");
renderView(block, loadImage);
screen.getByRole("button", { name: /screenshot/i }).click();
@@ -151,14 +125,14 @@ describe("BrowserInspectToolView", () => {
it("renders the running form without output", () => {
const loadImage = vi.fn();
renderView(runningBlock('{"action":"screenshot"}'), loadImage);
renderView(runningBlock("{}"), loadImage);
expect(screen.getByText("bsk screenshot --session (current)")).toBeTruthy();
expect(loadImage).not.toHaveBeenCalled();
});
});
describe("client plugin registration", () => {
it("registers the browser_inspect toolview and the shell observation overlay", () => {
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
const registrations: { name: string; key?: string; id?: string }[] = [];
const sessions = { binding: () => undefined };
const ctx = {
@@ -176,7 +150,7 @@ describe("client plugin registration", () => {
};
apply(ctx as never);
expect(registrations).toEqual([
{ name: "tool.call.toolview", key: "browser_inspect" },
{ name: "tool.call.toolview", key: "browser_screenshot" },
{ name: "shell.overlay", id: "bsk-observation" },
]);
});
@@ -73,15 +73,15 @@ describe("dispose cleanup ownership", () => {
};
apply(ctx as never, { maxSessions: 5, lazyTools: false }, { runnerFactory: () => runner });
const session = tools.get("browser_session");
const inspect = tools.get("browser_inspect");
if (session === undefined || inspect === undefined) throw new Error("tools not registered");
await session.execute({ action: "start" }, makeExec());
await session.execute({ action: "start" }, makeExec());
const start = tools.get("browser_session_start");
const snapshot = tools.get("browser_snapshot");
if (start === undefined || snapshot === undefined) throw new Error("tools not registered");
await start.execute({}, makeExec());
await start.execute({}, makeExec());
// A foreign session cannot even be referenced any more.
await expect(
inspect.execute({ action: "snapshot", session: "ext9" }, makeExec()),
).rejects.toThrow(/does not belong to this plugin/);
await expect(snapshot.execute({ session: "ext9" }, makeExec())).rejects.toThrow(
/does not belong to this plugin/,
);
await Promise.all(disposers.map((dispose) => dispose()));
@@ -211,25 +211,19 @@ describe("lazyTools wiring in apply()", () => {
it("lazyTools: false registers the suite at apply time", () => {
const { tools } = applyHarness({ lazyTools: false });
expect([...tools.keys()].sort()).toEqual([
"browser_assist",
"browser_inspect",
"browser_interact",
"browser_page",
"browser_session",
"browser_tabs",
]);
expect(tools.has("browser_session_start")).toBe(true);
expect(tools.has("browser_emulate")).toBe(true);
});
it("lazyTools: true hides the suite until the skill fires", () => {
const { tools, listeners } = applyHarness({ lazyTools: true });
expect(tools.has("browser_session")).toBe(false);
expect(tools.has("browser_session_start")).toBe(false);
callListeners(
listeners,
"tools/result",
{ name: "skill", arguments: { name: "browser-skill" } },
{ isError: false },
);
expect(tools.has("browser_session")).toBe(true);
expect(tools.has("browser_session_start")).toBe(true);
});
});
@@ -1,32 +0,0 @@
// dsh 0.1 keys the client graph and Node import by `export const name`.
// That string must stay identical to the published package name (and therefore
// the client.cjs __ModuleLoader__ id, which tsdown reads from package.json).
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { name } from "../src/index";
const pkg = JSON.parse(
readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8"),
) as {
name: string;
dsh: { client: { external?: string[] } };
};
describe("plugin identity", () => {
it("uses the published package name as the Cordis plugin id", () => {
expect(name).toBe("@wxg-prc-cpg/browser-skill-dsh-plugin");
expect(name).toBe(pkg.name);
});
it("declares client require()s so dsh arrives them before materialize", () => {
expect(pkg.dsh.client.external).toEqual(
expect.arrayContaining([
"@deepseek-ai/dsh-client-ui-attachment",
"@deepseek-ai/dsh-client-ui-primitives",
]),
);
});
});
@@ -71,9 +71,7 @@ describe("SessionRegistry", () => {
it("resolve throws actionable guidance when no session exists", () => {
const registry = new SessionRegistry(5);
expect(() => registry.resolve(undefined, "browser_interact(action=click)")).toThrow(
/browser_session action=start/,
);
expect(() => registry.resolve(undefined, "browser_click")).toThrow(/browser_session_start/);
});
it("recency order survives remove of the current session", () => {
Loaded 100 of 105 files, more files were not shown because too many files have changed in this diff. Show more