test: add extensible browser evaluation harness

This commit is contained in:
drakezhang committed 2026-08-28 12:16:39 +08:00
1 parent 7aaa5a2d01
commit 9ad33801c7
46 files changed
+3554

No files matched your search

+99
View File
@@ -0,0 +1,99 @@
import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { countPattern, runProcess, substituteArgs } from "./process.mjs";
import { renderPrompt } from "./tasks.mjs";
function safeRunPart(value) {
return value.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 48);
}
export async function loadAgentConfig(configPath, agentName) {
const absolute = resolve(configPath);
const config = JSON.parse(await readFile(absolute, "utf8"));
const adapter = config.agents?.[agentName];
if (!adapter) {
const names = Object.keys(config.agents ?? {}).join(", ");
throw new Error(`unknown agent ${JSON.stringify(agentName)}; configured agents: ${names}`);
}
if (typeof adapter.command !== "string" || !Array.isArray(adapter.args)) {
throw new Error(`agent ${JSON.stringify(agentName)} must define command and args`);
}
return { adapter, configDirectory: dirname(absolute) };
}
export async function runCommandTask({
server,
serverInfo,
task,
adapter,
adapterName,
configDirectory,
iteration,
locale,
seed = task.seed,
timeoutMs,
variantOverride,
}) {
const variant = variantOverride ?? adapter.variant ?? "unspecified";
const runId = [
safeRunPart(adapterName),
safeRunPart(variant),
safeRunPart(task.id),
iteration,
randomUUID().slice(0, 8),
].join("-");
server.reset(runId);
const prompt = renderPrompt(task, { baseUrl: serverInfo.baseUrl, runId, locale, seed });
const values = {
prompt,
caseId: task.id,
taskId: task.id,
runId,
baseUrl: serverInfo.baseUrl,
seed: seed ?? "",
variant,
};
const args = substituteArgs(adapter.args, values);
const execution = await runProcess(adapter.command, args, {
cwd: adapter.cwd ? resolve(configDirectory, adapter.cwd) : configDirectory,
env: adapter.env,
timeoutMs: adapter.timeoutMs ?? timeoutMs,
});
await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
const responseText = execution.stdout;
const combinedOutput = `${execution.stdout}\n${execution.stderr}`;
const verification = verifyTask(task, {
events: server.snapshot(runId).events,
responseText,
});
const defaultErrorPattern = "(?:\\bError:|timed out|unfinished command|session already has)";
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
iteration,
adapter: adapterName,
variant,
adapterMetadata: adapter.metadata ?? {},
locale,
prompt,
execution,
metrics: {
errorCount: countPattern(
combinedOutput,
adapter.metrics?.errorPattern ?? defaultErrorPattern,
),
toolCallCount: countPattern(combinedOutput, adapter.metrics?.toolCallPattern),
responseChars: responseText.length,
},
eventCount: server.snapshot(runId).events.length,
verification,
};
}
+309
View File
@@ -0,0 +1,309 @@
import { mkdir, stat } from "node:fs/promises";
import { join } from "node:path";
import { verifyTask } from "./oracle.mjs";
import { runProcess } from "./process.mjs";
import { taskUrl } from "./tasks.mjs";
function parseJsonOutput(step) {
try {
return JSON.parse(step.stdout);
} catch (error) {
throw new Error(`could not parse bsk JSON output: ${error.message}`);
}
}
function renderStepError(step) {
const raw = step.stderr.trim() || step.stdout.trim();
if (!raw) return `exit ${step.exitCode}`;
try {
const parsed = JSON.parse(raw);
return parsed.hint ? `${parsed.message} (hint: ${parsed.hint})` : parsed.message;
} catch {
return raw;
}
}
async function hasNonemptyFile(path) {
try {
return (await stat(path)).size > 0;
} catch {
return false;
}
}
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function resolveValue(value, variables) {
if (typeof value !== "string") return value;
const exact = value.match(/^\{([^{}]+)}$/);
if (exact) {
const resolved = readPath(variables, exact[1]);
if (resolved === undefined) throw new Error(`unknown workflow variable ${exact[1]}`);
return resolved;
}
return value.replaceAll(/\{([^{}]+)\}/g, (_match, path) => {
const resolved = readPath(variables, path);
if (resolved === undefined) throw new Error(`unknown workflow variable ${path}`);
return String(resolved);
});
}
function targetArgs(step, variables) {
if (step.selector) return ["--selector", String(resolveValue(step.selector, variables))];
if (step.ref) return ["--ref", String(resolveValue(step.ref, variables))];
return [];
}
function tabArgs(step, variables) {
return step.tabId === undefined ? [] : ["--tab-id", String(resolveValue(step.tabId, variables))];
}
function matchesWhere(value, where = {}) {
return Object.entries(where).every(([path, expected]) => readPath(value, path) === expected);
}
export async function runBskSmokeTask({
bskCommand,
outputDirectory,
server,
serverInfo,
task,
runId,
seed = task.seed,
timeoutMs = 60_000,
}) {
const steps = [];
const evidence = {};
const variables = {
baseUrl: serverInfo.baseUrl,
runId,
seed: seed ?? "",
url: taskUrl(task, serverInfo.baseUrl, runId, seed),
};
let sessionId;
let executionError;
await mkdir(outputDirectory, { recursive: true });
async function bsk(args, { timeout = timeoutMs } = {}) {
const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout });
steps.push(execution);
if (execution.exitCode !== 0 || execution.timedOut || execution.error) {
throw new Error(
`bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`,
);
}
return parseJsonOutput(execution);
}
async function waitForSiteEvent(step) {
const timeout = step.timeoutMs ?? 3_000;
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
if (
server
.snapshot(runId)
.events.some(
(event) => event.type === step.type && matchesWhere(event.data ?? {}, step.where ?? {}),
)
) {
return { observed: true };
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 50));
}
throw new Error(`local fixture did not report ${step.type} within ${timeout}ms`);
}
async function executeWorkflowStep(step) {
const session = ["--session", sessionId];
let result;
switch (step.action) {
case "navigate":
result = await bsk([
"navigate",
String(resolveValue(step.url ?? "{url}", variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "back":
case "forward":
result = await bsk([`navigate-${step.action}`, ...session, ...tabArgs(step, variables)]);
break;
case "reload":
result = await bsk(["reload", ...session, ...tabArgs(step, variables)]);
break;
case "wait":
result = await bsk(["wait-ms", String(resolveValue(step.duration, variables))]);
break;
case "wait-site-event":
result = await waitForSiteEvent(step);
break;
case "observe":
case "snapshot":
case "html": {
const command = step.action === "html" ? "get-html" : step.action;
result = await bsk([command, ...session, ...tabArgs(step, variables)]);
break;
}
case "screenshot": {
const name = resolveValue(step.artifactName ?? "screenshot.png", variables);
const path = join(outputDirectory, `${runId}-${name}`);
result = await bsk(["screenshot", ...session, ...tabArgs(step, variables), "--out", path]);
if (step.evidence) evidence[step.evidence] = await hasNonemptyFile(path);
break;
}
case "console":
case "network":
result = await bsk([step.action, ...session, ...tabArgs(step, variables)]);
break;
case "click":
case "hover":
result = await bsk([
step.action,
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "fill":
result = await bsk([
"fill",
...targetArgs(step, variables),
"--value",
String(resolveValue(step.value, variables)),
...session,
...tabArgs(step, variables),
]);
break;
case "select":
result = await bsk([
"select",
...targetArgs(step, variables),
...step.values.flatMap((value) => ["--value", String(resolveValue(value, variables))]),
...session,
...tabArgs(step, variables),
]);
break;
case "press":
result = await bsk([
"press",
String(resolveValue(step.key, variables)),
...targetArgs(step, variables),
...session,
...tabArgs(step, variables),
]);
break;
case "tab-list":
result = await bsk(["tab", "list", ...session]);
break;
case "tab-create":
result = await bsk([
"tab",
"create",
...session,
...(step.url ? ["--url", String(resolveValue(step.url, variables))] : []),
]);
break;
case "tab-select":
case "tab-close":
result = await bsk([
"tab",
step.action.slice(4),
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "borrow":
case "return":
result = await bsk([
"tab",
step.action,
String(resolveValue(step.tabId, variables)),
...session,
]);
break;
case "resize":
result = await bsk([
"window",
"resize",
...session,
"--width",
String(step.width),
"--height",
String(step.height),
]);
break;
case "emulate":
result = await bsk([
"emulate",
...session,
...tabArgs(step, variables),
"--device",
String(resolveValue(step.device, variables)),
]);
break;
case "request-help":
result = await bsk([
"request-help",
...session,
"--prompt",
String(resolveValue(step.prompt, variables)),
...(step.title ? ["--title", String(resolveValue(step.title, variables))] : []),
...(step.timeout ? ["--timeout", String(step.timeout)] : []),
]);
break;
default:
throw new Error(`unsupported smoke workflow action ${step.action}`);
}
if (step.saveAs) variables[step.saveAs] = result;
if (step.evidence && step.action !== "screenshot") evidence[step.evidence] = true;
}
server.reset(runId);
try {
if (task.smokeSteps.length === 0) throw new Error(`case ${task.id} has no smoke workflow`);
const started = await bsk(["session", "start", "--no-focus"], { timeout: 90_000 });
sessionId = started.session_id;
variables.sessionId = sessionId;
await bsk(["session", "list"]);
for (const step of task.smokeSteps) await executeWorkflowStep(step);
} catch (error) {
executionError = error instanceof Error ? error.message : String(error);
} finally {
if (sessionId) {
try {
const stopped = await bsk(["session", "stop", sessionId], { timeout: 90_000 });
evidence.sessionStopped =
stopped.stopped?.includes(sessionId) === true && (stopped.failed?.length ?? 0) === 0;
if (!evidence.sessionStopped) {
executionError ??= `bsk did not confirm that session ${sessionId} stopped`;
}
} catch (error) {
evidence.sessionStopped = false;
executionError ??= error instanceof Error ? error.message : String(error);
}
}
}
await new Promise((resolveDelay) => setTimeout(resolveDelay, 350));
const responseText = steps.map(({ stdout }) => stdout).join("\n");
const events = server.snapshot(runId).events;
return {
runId,
caseId: task.id,
taskId: task.id,
suite: task.suite,
tags: task.tags,
seed,
adapter: "bsk-direct",
variant: "cli-direct",
executionError,
steps,
evidence,
eventCount: events.length,
verification: verifyTask(task, { events, responseText, adapterEvidence: evidence }),
};
}
+216
View File
@@ -0,0 +1,216 @@
import { readFile } from "node:fs/promises";
import { discoverFiles } from "./discovery.mjs";
import { OPERATION_NAMES, WORKFLOW_ACTIONS } from "./operations.mjs";
import { casesDirectory as defaultCasesDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const EVIDENCE_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
const TOP_LEVEL_KEYS = new Set([
"$schema",
"schemaVersion",
"id",
"title",
"order",
"suite",
"tags",
"seed",
"source",
"fixture",
"prompts",
"coverage",
"assertions",
"smoke",
]);
function isObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function checkString(errors, value, path) {
if (typeof value !== "string" || value.trim() === "") errors.push(`${path} must be a string`);
}
function validateAssertion(errors, assertion, path, kind) {
if (!isObject(assertion)) {
errors.push(`${path} must be an object`);
return;
}
checkString(errors, assertion.label, `${path}.label`);
if (kind === "site") {
checkString(errors, assertion.type, `${path}.type`);
if (!Number.isInteger(assertion.minCount) || assertion.minCount < 1) {
errors.push(`${path}.minCount must be a positive integer`);
}
if (assertion.where !== undefined && !isObject(assertion.where)) {
errors.push(`${path}.where must be an object`);
}
} else if (kind === "response") {
checkString(errors, assertion.includes, `${path}.includes`);
} else {
checkString(errors, assertion.key, `${path}.key`);
}
}
function validateWorkflowStep(errors, step, path) {
if (!isObject(step)) {
errors.push(`${path} must be an object`);
return;
}
if (!WORKFLOW_ACTIONS.has(step.action)) {
errors.push(`${path}.action is unsupported: ${JSON.stringify(step.action)}`);
}
if (step.saveAs !== undefined && !ID_PATTERN.test(step.saveAs)) {
errors.push(`${path}.saveAs must use lowercase kebab-case`);
}
if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) {
errors.push(`${path}.evidence must be an identifier`);
}
if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) {
errors.push(`${path} requires selector or ref`);
}
if (step.action === "fill") checkString(errors, step.value, `${path}.value`);
if (step.action === "select") {
if (!Array.isArray(step.values) || step.values.length === 0) {
errors.push(`${path}.values must be a non-empty array`);
} else if (step.values.some((value) => typeof value !== "string")) {
errors.push(`${path}.values must contain only strings`);
}
}
if (step.action === "press") checkString(errors, step.key, `${path}.key`);
if (step.action === "wait") checkString(errors, step.duration, `${path}.duration`);
if (step.action === "wait-site-event") checkString(errors, step.type, `${path}.type`);
if (["tab-select", "tab-close", "borrow", "return"].includes(step.action)) {
checkString(errors, step.tabId, `${path}.tabId`);
}
if (step.action === "resize") {
if (
!Number.isInteger(step.width) ||
step.width < 1 ||
!Number.isInteger(step.height) ||
step.height < 1
) {
errors.push(`${path}.width and height must be positive integers`);
}
}
if (step.action === "emulate") checkString(errors, step.device, `${path}.device`);
if (step.action === "request-help") checkString(errors, step.prompt, `${path}.prompt`);
if (step.timeoutMs !== undefined && (!Number.isInteger(step.timeoutMs) || step.timeoutMs < 1)) {
errors.push(`${path}.timeoutMs must be a positive integer`);
}
}
export function validateCaseManifest(manifest, { source = "case manifest" } = {}) {
const errors = [];
if (!isObject(manifest)) return [`${source} must contain a JSON object`];
for (const key of Object.keys(manifest)) {
if (!TOP_LEVEL_KEYS.has(key)) errors.push(`unknown top-level property ${key}`);
}
if (manifest.schemaVersion !== 1) errors.push("schemaVersion must equal 1");
if (!ID_PATTERN.test(manifest.id ?? "")) errors.push("id must use lowercase kebab-case");
checkString(errors, manifest.title, "title");
if (manifest.order !== undefined && (!Number.isInteger(manifest.order) || manifest.order < 0)) {
errors.push("order must be a non-negative integer");
}
if (!ID_PATTERN.test(manifest.suite ?? "")) errors.push("suite must use lowercase kebab-case");
if (!Array.isArray(manifest.tags) || manifest.tags.some((tag) => !ID_PATTERN.test(tag))) {
errors.push("tags must be an array of lowercase kebab-case strings");
} else if (new Set(manifest.tags).size !== manifest.tags.length) {
errors.push("tags must not contain duplicates");
}
if (!isObject(manifest.fixture)) errors.push("fixture must be an object");
else if (
typeof manifest.fixture.startPath !== "string" ||
!manifest.fixture.startPath.startsWith("/")
) {
errors.push("fixture.startPath must be an absolute URL path");
}
if (!isObject(manifest.prompts)) errors.push("prompts must be an object");
else {
checkString(errors, manifest.prompts.en, "prompts.en");
checkString(errors, manifest.prompts["zh-CN"], "prompts.zh-CN");
}
if (!Array.isArray(manifest.coverage)) errors.push("coverage must be an array");
else {
for (const operation of manifest.coverage) {
if (!OPERATION_NAMES.has(operation))
errors.push(`coverage contains unknown operation ${operation}`);
}
if (new Set(manifest.coverage).size !== manifest.coverage.length) {
errors.push("coverage must not contain duplicates");
}
}
if (!isObject(manifest.assertions)) errors.push("assertions must be an object");
else {
for (const kind of ["site", "response", "adapter"]) {
if (!Array.isArray(manifest.assertions[kind])) {
errors.push(`assertions.${kind} must be an array`);
} else {
manifest.assertions[kind].forEach((assertion, index) =>
validateAssertion(errors, assertion, `assertions.${kind}[${index}]`, kind),
);
}
}
}
if (!isObject(manifest.smoke) || !Array.isArray(manifest.smoke.steps)) {
errors.push("smoke.steps must be an array");
} else {
manifest.smoke.steps.forEach((step, index) =>
validateWorkflowStep(errors, step, `smoke.steps[${index}]`),
);
}
if (manifest.seed !== undefined && !Number.isSafeInteger(manifest.seed)) {
errors.push("seed must be a safe integer");
}
if (manifest.source !== undefined && !isObject(manifest.source)) {
errors.push("source must be an object");
}
return errors.map((error) => `${source}: ${error}`);
}
function normalizeCase(manifest, sourceFile) {
return Object.freeze({
...manifest,
startPath: manifest.fixture.startPath,
siteAssertions: manifest.assertions.site,
responseAssertions: manifest.assertions.response,
adapterAssertions: manifest.assertions.adapter,
smokeSteps: manifest.smoke.steps,
sourceFile,
});
}
export async function loadCaseManifests({ casesDirectory = defaultCasesDirectory } = {}) {
const files = await discoverFiles(casesDirectory, (path) => path.endsWith(".case.json"));
const cases = [];
const ids = new Map();
const errors = [];
for (const file of files) {
let manifest;
try {
manifest = JSON.parse(await readFile(file, "utf8"));
} catch (error) {
errors.push(`${file}: invalid JSON: ${error.message}`);
continue;
}
const manifestErrors = validateCaseManifest(manifest, { source: file });
errors.push(...manifestErrors);
if (typeof manifest?.id === "string") {
if (ids.has(manifest.id)) {
errors.push(`${file}: duplicate id also declared by ${ids.get(manifest.id)}`);
} else {
ids.set(manifest.id, file);
}
}
if (manifestErrors.length === 0) cases.push(normalizeCase(manifest, file));
}
if (files.length === 0) errors.push(`${casesDirectory}: no .case.json files found`);
if (errors.length > 0) throw new Error(`case validation failed:\n${errors.join("\n")}`);
cases.sort(
(left, right) =>
left.suite.localeCompare(right.suite) ||
(left.order ?? Number.MAX_SAFE_INTEGER) - (right.order ?? Number.MAX_SAFE_INTEGER) ||
left.id.localeCompare(right.id),
);
return Object.freeze(cases);
}
+26
View File
@@ -0,0 +1,26 @@
import { OPERATION_CATALOG, WORKFLOW_ACTION_OPERATIONS } from "./operations.mjs";
export function buildOperationCoverage(cases) {
return OPERATION_CATALOG.map(({ operation, manualNote }) => {
const agentCases = cases
.filter((testCase) => testCase.coverage.includes(operation))
.map(({ id }) => id);
let smokeCases;
if (["session.start", "session.stop", "session.list"].includes(operation)) {
smokeCases = cases.filter(({ smokeSteps }) => smokeSteps.length > 0).map(({ id }) => id);
} else {
smokeCases = cases
.filter(({ smokeSteps }) =>
smokeSteps.some((step) => WORKFLOW_ACTION_OPERATIONS[step.action] === operation),
)
.map(({ id }) => id);
}
return {
operation,
agentCases,
smokeCases,
directSmoke: smokeCases.length > 0,
note: smokeCases.length === 0 && manualNote ? `manual lane: ${manualNote}` : undefined,
};
});
}
+24
View File
@@ -0,0 +1,24 @@
import { readdir } from "node:fs/promises";
import { join } from "node:path";
export async function discoverFiles(root, predicate) {
const files = [];
async function visit(directory) {
let entries;
try {
entries = await readdir(directory, { withFileTypes: true });
} catch (error) {
if (error?.code === "ENOENT") return;
throw error;
}
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
const path = join(directory, entry.name);
if (entry.isDirectory()) await visit(path);
else if (entry.isFile() && predicate(path)) files.push(path);
}
}
await visit(root);
return files;
}
+68
View File
@@ -0,0 +1,68 @@
import { pathToFileURL } from "node:url";
import { discoverFiles } from "./discovery.mjs";
import { casesDirectory, fixturesDirectory } from "./paths.mjs";
export async function loadFixtureRegistry({ roots = [fixturesDirectory, casesDirectory] } = {}) {
const files = (
await Promise.all(
roots.map((root) => discoverFiles(root, (path) => path.endsWith(".fixture.mjs"))),
)
).flat();
const routes = new Map();
const definitions = [];
const errors = [];
for (const file of files.sort()) {
let definition;
try {
definition = (await import(pathToFileURL(file).href)).default;
} catch (error) {
errors.push(`${file}: could not import fixture: ${error.message}`);
continue;
}
if (!definition || typeof definition.id !== "string") {
errors.push(`${file}: default export must define id`);
continue;
}
if (!Array.isArray(definition.routes) || definition.routes.length === 0) {
errors.push(`${file}: default export must define non-empty routes`);
continue;
}
if (typeof definition.render !== "function") {
errors.push(`${file}: default export must define render(context)`);
continue;
}
definitions.push({ ...definition, sourceFile: file });
for (const route of definition.routes) {
if (typeof route !== "string" || !route.startsWith("/")) {
errors.push(`${file}: invalid route ${JSON.stringify(route)}`);
} else if (routes.has(route)) {
errors.push(
`${file}: route ${route} is already registered by ${routes.get(route).sourceFile}`,
);
} else {
routes.set(route, { definition, sourceFile: file });
}
}
}
if (errors.length > 0) throw new Error(`fixture validation failed:\n${errors.join("\n")}`);
return Object.freeze({
definitions: Object.freeze(definitions),
routes: Object.freeze([...routes.keys()].sort()),
hasRoute(pathname) {
return routes.has(pathname);
},
render(pathname, context) {
const entry = routes.get(pathname);
return entry ? entry.definition.render({ ...context, pathname }) : null;
},
});
}
export function validateCaseFixtureLinks(cases, fixtureRegistry) {
return cases
.filter(({ startPath }) => !fixtureRegistry.hasRoute(startPath))
.map(({ id, startPath }) => `case ${id} references unregistered fixture route ${startPath}`);
}
+78
View File
@@ -0,0 +1,78 @@
const stylesheet = `
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
body { margin: 0; background: #f4f7fb; color: #182033; }
main { width: min(760px, calc(100% - 40px)); margin: 48px auto; }
.card { background: white; border: 1px solid #d9e1ef; border-radius: 14px; padding: 24px; box-shadow: 0 8px 28px #20304a12; }
h1 { margin-top: 0; }
a, button { font: inherit; }
button, .button { display: inline-block; border: 0; border-radius: 8px; padding: 10px 16px; background: #3157d5; color: white; cursor: pointer; text-decoration: none; }
label { display: block; margin: 16px 0; font-weight: 600; }
input, textarea, select { display: block; box-sizing: border-box; width: 100%; margin-top: 6px; border: 1px solid #9caac2; border-radius: 7px; padding: 9px; font: inherit; }
textarea { min-height: 90px; }
.marker { border-left: 4px solid #3157d5; padding: 8px 12px; background: #eef2ff; font-family: ui-monospace, monospace; }
.muted { color: #5c667a; }
#account-menu { position: relative; display: inline-block; padding-bottom: 12px; }
#hover-panel { display: none; position: absolute; z-index: 2; top: 34px; left: 0; width: 190px; padding: 10px; background: white; border: 1px solid #9caac2; border-radius: 8px; box-shadow: 0 8px 24px #20304a22; }
#account-menu:hover #hover-panel, #account-menu.open #hover-panel { display: block; }
#hover-panel button { width: 100%; }
#inspect-card { margin-top: 80px; }
code { font-family: ui-monospace, monospace; }
`;
const clientScript = `
(() => {
const runId = new URLSearchParams(location.search).get("run");
const navigation = performance.getEntriesByType("navigation")[0];
const send = (type, data = {}) => {
if (!runId) return Promise.resolve();
return fetch("/api/events", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ runId, type, path: location.pathname, data }),
keepalive: true,
}).catch(() => undefined);
};
window.browserEval = { runId, send };
window.addEventListener("pageshow", (event) => {
send("page.shown", {
path: location.pathname,
persisted: event.persisted,
navigationType: navigation?.type ?? "unknown",
});
});
})();
`;
export function escapeHtml(value) {
return String(value)
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;");
}
export function withRun(path, runId, extra = {}) {
const url = new URL(path, "http://browser-eval.local");
url.searchParams.set("run", runId);
for (const [name, value] of Object.entries(extra)) {
if (value !== undefined) url.searchParams.set(name, String(value));
}
return `${url.pathname}${url.search}`;
}
export function page({ title, body, script = "" }) {
return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>${escapeHtml(title)}</title>
<style>${stylesheet}</style>
</head>
<body>
<main>${body}</main>
<script>${clientScript}</script>
${script ? `<script>${script}</script>` : ""}
</body>
</html>`;
}
+68
View File
@@ -0,0 +1,68 @@
export const OPERATION_CATALOG = [
{ operation: "session.start" },
{ operation: "session.stop" },
{ operation: "session.list" },
{ operation: "page.navigate" },
{ operation: "page.back" },
{ operation: "page.forward" },
{ operation: "page.reload" },
{ operation: "page.wait" },
{ operation: "inspect.observe" },
{ operation: "inspect.snapshot" },
{ operation: "inspect.html" },
{ operation: "inspect.screenshot" },
{ operation: "inspect.console" },
{ operation: "inspect.network" },
{ operation: "interact.click" },
{ operation: "interact.hover" },
{ operation: "interact.fill" },
{ operation: "interact.select" },
{ operation: "interact.press" },
{ operation: "tabs.list" },
{ operation: "tabs.create" },
{ operation: "tabs.select" },
{ operation: "tabs.close" },
{
operation: "tabs.borrow",
manualNote: "requires an existing user tab and ownership confirmation",
},
{ operation: "tabs.return", manualNote: "paired with borrowing a real user tab" },
{ operation: "assist.resize" },
{ operation: "assist.emulate" },
{ operation: "assist.request-help", manualNote: "intentionally waits for human interaction" },
];
export const WORKFLOW_ACTION_OPERATIONS = {
navigate: "page.navigate",
back: "page.back",
forward: "page.forward",
reload: "page.reload",
wait: "page.wait",
observe: "inspect.observe",
snapshot: "inspect.snapshot",
html: "inspect.html",
screenshot: "inspect.screenshot",
console: "inspect.console",
network: "inspect.network",
click: "interact.click",
hover: "interact.hover",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
"tab-list": "tabs.list",
"tab-create": "tabs.create",
"tab-select": "tabs.select",
"tab-close": "tabs.close",
borrow: "tabs.borrow",
return: "tabs.return",
resize: "assist.resize",
emulate: "assist.emulate",
"request-help": "assist.request-help",
};
export const WORKFLOW_ACTIONS = new Set([
...Object.keys(WORKFLOW_ACTION_OPERATIONS),
"wait-site-event",
]);
export const OPERATION_NAMES = new Set(OPERATION_CATALOG.map(({ operation }) => operation));
+63
View File
@@ -0,0 +1,63 @@
function readPath(value, path) {
return path.split(".").reduce((current, part) => current?.[part], value);
}
function eventMatches(event, assertion) {
if (event.type !== assertion.type) return false;
return Object.entries(assertion.where ?? {}).every(
([path, expected]) => readPath(event, path) === expected,
);
}
export function verifyTask(task, { events = [], responseText = "", adapterEvidence = {} } = {}) {
const site = task.siteAssertions.map((assertion) => {
const actual = events.filter((event) => eventMatches(event, assertion)).length;
return {
kind: "site",
label: assertion.label,
status: actual >= assertion.minCount ? "passed" : "failed",
expected: `at least ${assertion.minCount}`,
actual,
};
});
const response = task.responseAssertions.map((assertion) => ({
kind: "response",
label: assertion.label,
status: responseText.includes(assertion.includes) ? "passed" : "failed",
expected: `response includes ${JSON.stringify(assertion.includes)}`,
}));
const adapter = task.adapterAssertions.map((assertion) => {
if (!Object.hasOwn(adapterEvidence, assertion.key)) {
return {
kind: "adapter",
label: assertion.label,
status: "unverified",
expected: assertion.key,
};
}
return {
kind: "adapter",
label: assertion.label,
status: adapterEvidence[assertion.key] ? "passed" : "failed",
expected: assertion.key,
actual: adapterEvidence[assertion.key],
};
});
const checks = [...site, ...response, ...adapter];
const failed = checks.filter(({ status }) => status === "failed").length;
const unverified = checks.filter(({ status }) => status === "unverified").length;
return {
taskId: task.id,
status: failed > 0 ? "failed" : unverified > 0 ? "passed-with-unverified" : "passed",
summary: {
passed: checks.length - failed - unverified,
failed,
unverified,
total: checks.length,
},
checks,
};
}
+8
View File
@@ -0,0 +1,8 @@
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export const evalDirectory = dirname(dirname(fileURLToPath(import.meta.url)));
export const casesDirectory = join(evalDirectory, "cases");
export const fixturesDirectory = join(evalDirectory, "fixtures");
export const schemasDirectory = join(evalDirectory, "schemas");
export const resultsDirectory = join(evalDirectory, "results");
+71
View File
@@ -0,0 +1,71 @@
import { spawn } from "node:child_process";
const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
function appendCaptured(current, chunk) {
if (current.length >= MAX_CAPTURE_BYTES) return current;
return `${current}${chunk}`.slice(0, MAX_CAPTURE_BYTES);
}
export function runProcess(command, args, { cwd, env, timeoutMs = 300_000 } = {}) {
return new Promise((resolve) => {
const startedAt = Date.now();
let stdout = "";
let stderr = "";
let timedOut = false;
let settled = false;
const child = spawn(command, args, {
cwd,
env: { ...process.env, ...env },
shell: false,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout.on("data", (chunk) => {
stdout = appendCaptured(stdout, chunk.toString());
});
child.stderr.on("data", (chunk) => {
stderr = appendCaptured(stderr, chunk.toString());
});
const timeout = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
}, timeoutMs);
const finish = (result) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
resolve({
command,
args,
stdout,
stderr,
timedOut,
durationMs: Date.now() - startedAt,
...result,
});
};
child.once("error", (error) => finish({ exitCode: null, signal: null, error: error.message }));
child.once("close", (exitCode, signal) => finish({ exitCode, signal }));
});
}
export function substituteArgs(args, values) {
return args.map((argument) => {
let result = argument;
for (const [key, value] of Object.entries(values)) {
result = result.replaceAll(`{${key}}`, String(value));
}
return result;
});
}
export function countPattern(text, pattern) {
if (!pattern) return null;
const matches = text.match(new RegExp(pattern, "g"));
return matches?.length ?? 0;
}
@@ -0,0 +1,64 @@
import { relative, sep } from "node:path";
import { validateCaseFixtureLinks } from "./fixture-registry.mjs";
import { casesDirectory } from "./paths.mjs";
function templateRoots(value) {
if (typeof value === "string") {
return [...value.matchAll(/\{([^{}]+)\}/g)].map((match) => match[1].split(".")[0]);
}
if (Array.isArray(value)) return value.flatMap(templateRoots);
if (value && typeof value === "object") return Object.values(value).flatMap(templateRoots);
return [];
}
export function validateRepositoryCases(cases, fixtureRegistry) {
const errors = [...validateCaseFixtureLinks(cases, fixtureRegistry)];
for (const testCase of cases) {
const relativePath = relative(casesDirectory, testCase.sourceFile);
const suiteDirectory = relativePath.split(sep)[0];
if (suiteDirectory !== testCase.suite) {
errors.push(
`case ${testCase.id} declares suite ${testCase.suite} but is stored under ${suiteDirectory}`,
);
}
const availableVariables = new Set(["baseUrl", "runId", "seed", "sessionId", "url"]);
const evidence = new Set(["sessionStopped"]);
for (const [index, step] of testCase.smokeSteps.entries()) {
for (const root of templateRoots(step)) {
if (!availableVariables.has(root)) {
errors.push(
`case ${testCase.id} smoke.steps[${index}] references unavailable variable ${root}`,
);
}
}
if (step.saveAs) availableVariables.add(step.saveAs);
if (step.evidence) evidence.add(step.evidence);
}
for (const assertion of testCase.adapterAssertions) {
if (!evidence.has(assertion.key)) {
errors.push(
`case ${testCase.id} expects adapter evidence ${assertion.key} but no smoke step produces it`,
);
}
}
}
return errors;
}
export function repositorySummary(cases, fixtureRegistry) {
const suites = Object.fromEntries(
[...new Set(cases.map(({ suite }) => suite))]
.sort()
.map((suite) => [suite, cases.filter((testCase) => testCase.suite === suite).length]),
);
const tags = [...new Set(cases.flatMap(({ tags }) => tags))].sort();
return {
cases: cases.length,
suites,
tags: tags.length,
fixtureModules: fixtureRegistry.definitions.length,
fixtureRoutes: fixtureRegistry.routes.length,
};
}
+120
View File
@@ -0,0 +1,120 @@
import { mkdir, realpath, stat, writeFile } from "node:fs/promises";
import { join, relative, sep } from "node:path";
import { casesDirectory, evalDirectory } from "./paths.mjs";
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
async function exists(path) {
try {
await stat(path);
return true;
} catch (error) {
if (error?.code === "ENOENT") return false;
throw error;
}
}
async function moduleSpecifier(fromDirectory, target) {
const path = relative(await realpath(fromDirectory), await realpath(target))
.split(sep)
.join("/");
return path.startsWith(".") ? path : `./${path}`;
}
export async function scaffoldCase({
id,
title = id,
suite = "regression",
source,
rootCasesDirectory = casesDirectory,
}) {
if (!ID_PATTERN.test(id ?? "")) throw new Error("case id must use lowercase kebab-case");
if (!ID_PATTERN.test(suite ?? "")) throw new Error("suite must use lowercase kebab-case");
const directory = join(rootCasesDirectory, suite, id);
if (await exists(directory)) throw new Error(`case directory already exists: ${directory}`);
await mkdir(directory, { recursive: true });
const marker = `CASE-${id.toUpperCase().replaceAll("-", "_")}`;
const startPath = `/cases/${id}`;
const manifest = {
$schema: "../../../schemas/case.schema.json",
schemaVersion: 1,
id,
title,
order: 10,
suite,
tags: ["badcase", "regression"],
...(source ? { source: { type: "user-badcase", reference: source } } : {}),
fixture: { startPath },
prompts: {
en: `Open {url}, complete the minimal regression scenario, report ${marker}, and close the browser session.`,
"zh-CN": `打开 {url},完成最小回归场景,报告 ${marker},最后关闭浏览器会话。`,
},
coverage: ["session.start", "session.stop", "page.navigate", "inspect.observe"],
assertions: {
site: [
{
label: "regression fixture was displayed",
type: "page.shown",
where: { "data.path": startPath },
minCount: 1,
},
],
response: [{ label: "reported regression marker", includes: marker }],
adapter: [{ label: "browser session was closed", key: "sessionStopped" }],
},
smoke: { steps: [{ action: "navigate" }, { action: "observe" }] },
};
const fixtureHelper = await moduleSpecifier(
directory,
join(evalDirectory, "lib", "fixtures.mjs"),
);
const fixture = `import { escapeHtml, page } from ${JSON.stringify(fixtureHelper)};
const title = ${JSON.stringify(title)};
export default {
id: ${JSON.stringify(id)},
routes: [${JSON.stringify(startPath)}],
render() {
return page({
title,
body: \`<section class="card"><h1>\${escapeHtml(title)}</h1><p class="marker">${marker}</p><p>Replace this page with a privacy-safe minimal reproduction.</p></section>\`,
});
},
};
`;
const readme = `# ${title}
## Original symptom
Describe what the user observed without including private data or proprietary page content.
## Minimal reproduction
Explain the smallest DOM, timing, or browser-state condition that reproduces the failure.
## Expected behavior
Describe the observable success condition and update the case assertions accordingly.
## Regression history
- Source: ${source ?? "add issue, PR, or anonymized user report reference"}
- Fix: add the fixing PR when available
`;
const files = {
manifest: join(directory, `${id}.case.json`),
fixture: join(directory, `${id}.fixture.mjs`),
readme: join(directory, "README.md"),
};
await Promise.all([
writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }),
writeFile(files.fixture, fixture, { flag: "wx" }),
writeFile(files.readme, readme, { flag: "wx" }),
]);
return { directory, files };
}
+40
View File
@@ -0,0 +1,40 @@
function hashSeed(value) {
const text = String(value);
let hash = 2166136261;
for (const character of text) {
hash ^= character.codePointAt(0);
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
function mulberry32(seed) {
let state = seed >>> 0;
return () => {
state += 0x6d2b79f5;
let value = state;
value = Math.imul(value ^ (value >>> 15), value | 1);
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
};
}
export function normalizeSeed(value) {
const numeric = Number(value);
return Number.isSafeInteger(numeric) ? numeric : hashSeed(value);
}
export function describeSeed(value) {
const seed = normalizeSeed(value);
const random = mulberry32(seed);
const choose = (values) => values[Math.floor(random() * values.length)];
return {
seed,
labelMode: choose(["wrapped", "for", "aria"]),
nestingDepth: 1 + Math.floor(random() * 4),
hydrationDelayMs: choose([0, 40, 120, 250]),
decoy: choose(["none", "disabled-input", "duplicate-text"]),
fieldOrder: choose(["text-first", "select-first"]),
idSuffix: hashSeed(`${seed}:id`).toString(36).slice(0, 6),
};
}
+152
View File
@@ -0,0 +1,152 @@
import { createServer } from "node:http";
import { loadFixtureRegistry } from "./fixture-registry.mjs";
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]{1,120}$/;
function validRunId(value) {
return typeof value === "string" && RUN_ID_PATTERN.test(value);
}
function json(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "application/json; charset=utf-8",
});
response.end(`${JSON.stringify(value, null, 2)}\n`);
}
function html(response, status, value) {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "text/html; charset=utf-8",
});
response.end(value);
}
async function readJson(request) {
let raw = "";
for await (const chunk of request) {
raw += chunk;
if (raw.length > 64 * 1024) {
throw new Error("request body exceeds 64 KiB");
}
}
return raw ? JSON.parse(raw) : {};
}
export function createEvalServer({ host = "127.0.0.1", port = 0, fixtureRegistry } = {}) {
const runs = new Map();
let sequence = 0;
let server;
let registry = fixtureRegistry;
function eventsFor(runId) {
if (!runs.has(runId)) runs.set(runId, []);
return runs.get(runId);
}
function record(runId, type, data = {}, metadata = {}) {
if (!validRunId(runId)) return null;
const event = {
sequence: ++sequence,
timestamp: new Date().toISOString(),
type,
data,
...metadata,
};
eventsFor(runId).push(event);
return event;
}
function snapshot(runId) {
return { runId, events: [...(runs.get(runId) ?? [])] };
}
function reset(runId) {
runs.set(runId, []);
}
async function handle(request, response) {
const origin = `http://${request.headers.host ?? `${host}:${port}`}`;
const url = new URL(request.url ?? "/", origin);
if (request.method === "GET" && url.pathname === "/healthz") {
return json(response, 200, { ok: true });
}
if (request.method === "POST" && url.pathname === "/api/events") {
const body = await readJson(request);
if (!validRunId(body.runId) || typeof body.type !== "string") {
return json(response, 400, { error: "runId and type are required" });
}
const event = record(body.runId, body.type, body.data ?? {}, {
path: typeof body.path === "string" ? body.path : undefined,
});
return json(response, 202, { accepted: true, sequence: event.sequence });
}
const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/);
if (runMatch) {
const runId = decodeURIComponent(runMatch[1]);
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
if (request.method === "GET") return json(response, 200, snapshot(runId));
if (request.method === "DELETE") {
reset(runId);
return json(response, 200, { runId, reset: true });
}
}
if (request.method === "GET" && url.pathname === "/api/ping") {
const runId = url.searchParams.get("run");
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
record(runId, "network.ping", { token: "NETWORK-73" }, { path: url.pathname });
return json(response, 200, { ok: true, token: "NETWORK-73" });
}
if (request.method !== "GET") return json(response, 405, { error: "method not allowed" });
const runId = url.searchParams.get("run") ?? "manual";
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
const fixture = registry.render(url.pathname, {
runId,
query: url.searchParams,
record: (type, data, metadata) => record(runId, type, data, metadata),
});
if (!fixture) return html(response, 404, "<!doctype html><h1>Not found</h1>");
record(runId, "page.request", { path: url.pathname }, { path: url.pathname });
return html(response, 200, fixture);
}
return {
record,
reset,
snapshot,
async start() {
if (server) throw new Error("eval server is already running");
registry ??= await loadFixtureRegistry();
server = createServer((request, response) => {
handle(request, response).catch((error) => {
json(response, 500, { error: error instanceof Error ? error.message : String(error) });
});
});
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, host, resolve);
});
const address = server.address();
if (!address || typeof address === "string") throw new Error("unexpected server address");
return { host, port: address.port, baseUrl: `http://${host}:${address.port}` };
},
async stop() {
if (!server) return;
const active = server;
server = undefined;
await new Promise((resolve, reject) => {
active.close((error) => (error ? reject(error) : resolve()));
active.closeAllConnections?.();
});
},
};
}
+73
View File
@@ -0,0 +1,73 @@
function durationFor(result) {
if (result.execution?.durationMs !== undefined) return result.execution.durationMs;
return (result.steps ?? []).reduce((sum, step) => sum + step.durationMs, 0);
}
export function summarizeReports(reports) {
const groups = new Map();
for (const report of reports) {
for (const result of report.results ?? []) {
const adapter = result.adapter ?? report.adapter ?? "unknown";
const variant = result.variant ?? report.variant ?? "unspecified";
const key = `${adapter}\u0000${variant}`;
if (!groups.has(key)) {
groups.set(key, {
adapter,
variant,
runs: 0,
passed: 0,
fullyVerified: 0,
executionFailures: 0,
errors: 0,
durationMs: 0,
toolCalls: 0,
toolCallSamples: 0,
});
}
const group = groups.get(key);
group.runs += 1;
group.durationMs += durationFor(result);
group.errors += result.metrics?.errorCount ?? (result.executionError ? 1 : 0);
if (result.metrics?.toolCallCount !== null && result.metrics?.toolCallCount !== undefined) {
group.toolCalls += result.metrics.toolCallCount;
group.toolCallSamples += 1;
}
const executionFailed =
Boolean(result.executionError) ||
result.execution?.timedOut ||
(result.execution?.exitCode !== undefined && result.execution.exitCode !== 0);
if (executionFailed) group.executionFailures += 1;
if (result.verification?.status !== "failed" && !executionFailed) group.passed += 1;
if (result.verification?.status === "passed" && !executionFailed) group.fullyVerified += 1;
}
}
return [...groups.values()].map((group) => ({
adapter: group.adapter,
variant: group.variant,
runs: group.runs,
passRate: group.runs ? group.passed / group.runs : 0,
fullyVerifiedRate: group.runs ? group.fullyVerified / group.runs : 0,
executionFailures: group.executionFailures,
errors: group.errors,
averageToolCalls: group.toolCallSamples
? Math.round((group.toolCalls / group.toolCallSamples) * 10) / 10
: null,
averageDurationMs: group.runs ? Math.round(group.durationMs / group.runs) : 0,
}));
}
export function printSummary(rows) {
const printable = rows.map((row) => ({
adapter: row.adapter,
variant: row.variant,
runs: row.runs,
pass: `${Math.round(row.passRate * 100)}%`,
verified: `${Math.round(row.fullyVerifiedRate * 100)}%`,
exec_failures: row.executionFailures,
errors: row.errors,
avg_tools: row.averageToolCalls ?? "-",
avg_ms: row.averageDurationMs,
}));
console.table(printable);
}
+48
View File
@@ -0,0 +1,48 @@
import { loadCaseManifests } from "./case-loader.mjs";
let casesPromise;
export function allTasks({ reload = false } = {}) {
if (reload || !casesPromise) casesPromise = loadCaseManifests();
return casesPromise;
}
export function listTasks(cases) {
return cases.map(({ id, title, suite, tags, seed }) => ({ id, title, suite, tags, seed }));
}
export function getTask(taskId, cases) {
const task = cases.find(({ id }) => id === taskId);
if (!task) {
const ids = cases.map(({ id }) => id).join(", ");
throw new Error(`unknown case ${JSON.stringify(taskId)}; expected one of: ${ids}`);
}
return task;
}
export function taskUrl(task, baseUrl, runId, seed = task.seed) {
const url = new URL(task.startPath, baseUrl);
url.searchParams.set("run", runId);
if (seed !== undefined) url.searchParams.set("seed", String(seed));
return url.toString();
}
function expandTemplate(template, values) {
return Object.entries(values).reduce(
(result, [name, value]) => result.replaceAll(`{${name}}`, String(value)),
template,
);
}
export function renderPrompt(task, { baseUrl, runId, locale = "zh-CN", seed = task.seed }) {
const template = task.prompts[locale];
if (!template) {
throw new Error(`unsupported locale ${JSON.stringify(locale)}; expected en or zh-CN`);
}
return expandTemplate(template, {
baseUrl,
runId: encodeURIComponent(runId),
seed: seed ?? "",
url: taskUrl(task, baseUrl, runId, seed),
});
}