mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
test: add extensible browser evaluation harness
This commit is contained in:
1 parent
7aaa5a2d01
commit
9ad33801c7
46 files changed
+3554
No files matched your search
@@ -0,0 +1,99 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { verifyTask } from "./oracle.mjs";
|
||||
import { countPattern, runProcess, substituteArgs } from "./process.mjs";
|
||||
import { renderPrompt } from "./tasks.mjs";
|
||||
|
||||
function safeRunPart(value) {
|
||||
return value.replaceAll(/[^A-Za-z0-9._-]/g, "-").slice(0, 48);
|
||||
}
|
||||
|
||||
export async function loadAgentConfig(configPath, agentName) {
|
||||
const absolute = resolve(configPath);
|
||||
const config = JSON.parse(await readFile(absolute, "utf8"));
|
||||
const adapter = config.agents?.[agentName];
|
||||
if (!adapter) {
|
||||
const names = Object.keys(config.agents ?? {}).join(", ");
|
||||
throw new Error(`unknown agent ${JSON.stringify(agentName)}; configured agents: ${names}`);
|
||||
}
|
||||
if (typeof adapter.command !== "string" || !Array.isArray(adapter.args)) {
|
||||
throw new Error(`agent ${JSON.stringify(agentName)} must define command and args`);
|
||||
}
|
||||
return { adapter, configDirectory: dirname(absolute) };
|
||||
}
|
||||
|
||||
export async function runCommandTask({
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
adapter,
|
||||
adapterName,
|
||||
configDirectory,
|
||||
iteration,
|
||||
locale,
|
||||
seed = task.seed,
|
||||
timeoutMs,
|
||||
variantOverride,
|
||||
}) {
|
||||
const variant = variantOverride ?? adapter.variant ?? "unspecified";
|
||||
const runId = [
|
||||
safeRunPart(adapterName),
|
||||
safeRunPart(variant),
|
||||
safeRunPart(task.id),
|
||||
iteration,
|
||||
randomUUID().slice(0, 8),
|
||||
].join("-");
|
||||
server.reset(runId);
|
||||
const prompt = renderPrompt(task, { baseUrl: serverInfo.baseUrl, runId, locale, seed });
|
||||
const values = {
|
||||
prompt,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
baseUrl: serverInfo.baseUrl,
|
||||
seed: seed ?? "",
|
||||
variant,
|
||||
};
|
||||
const args = substituteArgs(adapter.args, values);
|
||||
const execution = await runProcess(adapter.command, args, {
|
||||
cwd: adapter.cwd ? resolve(configDirectory, adapter.cwd) : configDirectory,
|
||||
env: adapter.env,
|
||||
timeoutMs: adapter.timeoutMs ?? timeoutMs,
|
||||
});
|
||||
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
|
||||
const responseText = execution.stdout;
|
||||
const combinedOutput = `${execution.stdout}\n${execution.stderr}`;
|
||||
const verification = verifyTask(task, {
|
||||
events: server.snapshot(runId).events,
|
||||
responseText,
|
||||
});
|
||||
const defaultErrorPattern = "(?:\\bError:|timed out|unfinished command|session already has)";
|
||||
return {
|
||||
runId,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
suite: task.suite,
|
||||
tags: task.tags,
|
||||
seed,
|
||||
iteration,
|
||||
adapter: adapterName,
|
||||
variant,
|
||||
adapterMetadata: adapter.metadata ?? {},
|
||||
locale,
|
||||
prompt,
|
||||
execution,
|
||||
metrics: {
|
||||
errorCount: countPattern(
|
||||
combinedOutput,
|
||||
adapter.metrics?.errorPattern ?? defaultErrorPattern,
|
||||
),
|
||||
toolCallCount: countPattern(combinedOutput, adapter.metrics?.toolCallPattern),
|
||||
responseChars: responseText.length,
|
||||
},
|
||||
eventCount: server.snapshot(runId).events.length,
|
||||
verification,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
import { mkdir, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { verifyTask } from "./oracle.mjs";
|
||||
import { runProcess } from "./process.mjs";
|
||||
import { taskUrl } from "./tasks.mjs";
|
||||
|
||||
function parseJsonOutput(step) {
|
||||
try {
|
||||
return JSON.parse(step.stdout);
|
||||
} catch (error) {
|
||||
throw new Error(`could not parse bsk JSON output: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function renderStepError(step) {
|
||||
const raw = step.stderr.trim() || step.stdout.trim();
|
||||
if (!raw) return `exit ${step.exitCode}`;
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed.hint ? `${parsed.message} (hint: ${parsed.hint})` : parsed.message;
|
||||
} catch {
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
|
||||
async function hasNonemptyFile(path) {
|
||||
try {
|
||||
return (await stat(path)).size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readPath(value, path) {
|
||||
return path.split(".").reduce((current, part) => current?.[part], value);
|
||||
}
|
||||
|
||||
function resolveValue(value, variables) {
|
||||
if (typeof value !== "string") return value;
|
||||
const exact = value.match(/^\{([^{}]+)}$/);
|
||||
if (exact) {
|
||||
const resolved = readPath(variables, exact[1]);
|
||||
if (resolved === undefined) throw new Error(`unknown workflow variable ${exact[1]}`);
|
||||
return resolved;
|
||||
}
|
||||
return value.replaceAll(/\{([^{}]+)\}/g, (_match, path) => {
|
||||
const resolved = readPath(variables, path);
|
||||
if (resolved === undefined) throw new Error(`unknown workflow variable ${path}`);
|
||||
return String(resolved);
|
||||
});
|
||||
}
|
||||
|
||||
function targetArgs(step, variables) {
|
||||
if (step.selector) return ["--selector", String(resolveValue(step.selector, variables))];
|
||||
if (step.ref) return ["--ref", String(resolveValue(step.ref, variables))];
|
||||
return [];
|
||||
}
|
||||
|
||||
function tabArgs(step, variables) {
|
||||
return step.tabId === undefined ? [] : ["--tab-id", String(resolveValue(step.tabId, variables))];
|
||||
}
|
||||
|
||||
function matchesWhere(value, where = {}) {
|
||||
return Object.entries(where).every(([path, expected]) => readPath(value, path) === expected);
|
||||
}
|
||||
|
||||
export async function runBskSmokeTask({
|
||||
bskCommand,
|
||||
outputDirectory,
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
runId,
|
||||
seed = task.seed,
|
||||
timeoutMs = 60_000,
|
||||
}) {
|
||||
const steps = [];
|
||||
const evidence = {};
|
||||
const variables = {
|
||||
baseUrl: serverInfo.baseUrl,
|
||||
runId,
|
||||
seed: seed ?? "",
|
||||
url: taskUrl(task, serverInfo.baseUrl, runId, seed),
|
||||
};
|
||||
let sessionId;
|
||||
let executionError;
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
|
||||
async function bsk(args, { timeout = timeoutMs } = {}) {
|
||||
const execution = await runProcess(bskCommand, [...args, "--json"], { timeoutMs: timeout });
|
||||
steps.push(execution);
|
||||
if (execution.exitCode !== 0 || execution.timedOut || execution.error) {
|
||||
throw new Error(
|
||||
`bsk ${args.join(" ")} failed: ${execution.error ?? renderStepError(execution)}`,
|
||||
);
|
||||
}
|
||||
return parseJsonOutput(execution);
|
||||
}
|
||||
|
||||
async function waitForSiteEvent(step) {
|
||||
const timeout = step.timeoutMs ?? 3_000;
|
||||
const deadline = Date.now() + timeout;
|
||||
while (Date.now() < deadline) {
|
||||
if (
|
||||
server
|
||||
.snapshot(runId)
|
||||
.events.some(
|
||||
(event) => event.type === step.type && matchesWhere(event.data ?? {}, step.where ?? {}),
|
||||
)
|
||||
) {
|
||||
return { observed: true };
|
||||
}
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 50));
|
||||
}
|
||||
throw new Error(`local fixture did not report ${step.type} within ${timeout}ms`);
|
||||
}
|
||||
|
||||
async function executeWorkflowStep(step) {
|
||||
const session = ["--session", sessionId];
|
||||
let result;
|
||||
switch (step.action) {
|
||||
case "navigate":
|
||||
result = await bsk([
|
||||
"navigate",
|
||||
String(resolveValue(step.url ?? "{url}", variables)),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "back":
|
||||
case "forward":
|
||||
result = await bsk([`navigate-${step.action}`, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "reload":
|
||||
result = await bsk(["reload", ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "wait":
|
||||
result = await bsk(["wait-ms", String(resolveValue(step.duration, variables))]);
|
||||
break;
|
||||
case "wait-site-event":
|
||||
result = await waitForSiteEvent(step);
|
||||
break;
|
||||
case "observe":
|
||||
case "snapshot":
|
||||
case "html": {
|
||||
const command = step.action === "html" ? "get-html" : step.action;
|
||||
result = await bsk([command, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
}
|
||||
case "screenshot": {
|
||||
const name = resolveValue(step.artifactName ?? "screenshot.png", variables);
|
||||
const path = join(outputDirectory, `${runId}-${name}`);
|
||||
result = await bsk(["screenshot", ...session, ...tabArgs(step, variables), "--out", path]);
|
||||
if (step.evidence) evidence[step.evidence] = await hasNonemptyFile(path);
|
||||
break;
|
||||
}
|
||||
case "console":
|
||||
case "network":
|
||||
result = await bsk([step.action, ...session, ...tabArgs(step, variables)]);
|
||||
break;
|
||||
case "click":
|
||||
case "hover":
|
||||
result = await bsk([
|
||||
step.action,
|
||||
...targetArgs(step, variables),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "fill":
|
||||
result = await bsk([
|
||||
"fill",
|
||||
...targetArgs(step, variables),
|
||||
"--value",
|
||||
String(resolveValue(step.value, variables)),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "select":
|
||||
result = await bsk([
|
||||
"select",
|
||||
...targetArgs(step, variables),
|
||||
...step.values.flatMap((value) => ["--value", String(resolveValue(value, variables))]),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "press":
|
||||
result = await bsk([
|
||||
"press",
|
||||
String(resolveValue(step.key, variables)),
|
||||
...targetArgs(step, variables),
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
]);
|
||||
break;
|
||||
case "tab-list":
|
||||
result = await bsk(["tab", "list", ...session]);
|
||||
break;
|
||||
case "tab-create":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
"create",
|
||||
...session,
|
||||
...(step.url ? ["--url", String(resolveValue(step.url, variables))] : []),
|
||||
]);
|
||||
break;
|
||||
case "tab-select":
|
||||
case "tab-close":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
step.action.slice(4),
|
||||
String(resolveValue(step.tabId, variables)),
|
||||
...session,
|
||||
]);
|
||||
break;
|
||||
case "borrow":
|
||||
case "return":
|
||||
result = await bsk([
|
||||
"tab",
|
||||
step.action,
|
||||
String(resolveValue(step.tabId, variables)),
|
||||
...session,
|
||||
]);
|
||||
break;
|
||||
case "resize":
|
||||
result = await bsk([
|
||||
"window",
|
||||
"resize",
|
||||
...session,
|
||||
"--width",
|
||||
String(step.width),
|
||||
"--height",
|
||||
String(step.height),
|
||||
]);
|
||||
break;
|
||||
case "emulate":
|
||||
result = await bsk([
|
||||
"emulate",
|
||||
...session,
|
||||
...tabArgs(step, variables),
|
||||
"--device",
|
||||
String(resolveValue(step.device, variables)),
|
||||
]);
|
||||
break;
|
||||
case "request-help":
|
||||
result = await bsk([
|
||||
"request-help",
|
||||
...session,
|
||||
"--prompt",
|
||||
String(resolveValue(step.prompt, variables)),
|
||||
...(step.title ? ["--title", String(resolveValue(step.title, variables))] : []),
|
||||
...(step.timeout ? ["--timeout", String(step.timeout)] : []),
|
||||
]);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`unsupported smoke workflow action ${step.action}`);
|
||||
}
|
||||
if (step.saveAs) variables[step.saveAs] = result;
|
||||
if (step.evidence && step.action !== "screenshot") evidence[step.evidence] = true;
|
||||
}
|
||||
|
||||
server.reset(runId);
|
||||
try {
|
||||
if (task.smokeSteps.length === 0) throw new Error(`case ${task.id} has no smoke workflow`);
|
||||
const started = await bsk(["session", "start", "--no-focus"], { timeout: 90_000 });
|
||||
sessionId = started.session_id;
|
||||
variables.sessionId = sessionId;
|
||||
await bsk(["session", "list"]);
|
||||
for (const step of task.smokeSteps) await executeWorkflowStep(step);
|
||||
} catch (error) {
|
||||
executionError = error instanceof Error ? error.message : String(error);
|
||||
} finally {
|
||||
if (sessionId) {
|
||||
try {
|
||||
const stopped = await bsk(["session", "stop", sessionId], { timeout: 90_000 });
|
||||
evidence.sessionStopped =
|
||||
stopped.stopped?.includes(sessionId) === true && (stopped.failed?.length ?? 0) === 0;
|
||||
if (!evidence.sessionStopped) {
|
||||
executionError ??= `bsk did not confirm that session ${sessionId} stopped`;
|
||||
}
|
||||
} catch (error) {
|
||||
evidence.sessionStopped = false;
|
||||
executionError ??= error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 350));
|
||||
const responseText = steps.map(({ stdout }) => stdout).join("\n");
|
||||
const events = server.snapshot(runId).events;
|
||||
return {
|
||||
runId,
|
||||
caseId: task.id,
|
||||
taskId: task.id,
|
||||
suite: task.suite,
|
||||
tags: task.tags,
|
||||
seed,
|
||||
adapter: "bsk-direct",
|
||||
variant: "cli-direct",
|
||||
executionError,
|
||||
steps,
|
||||
evidence,
|
||||
eventCount: events.length,
|
||||
verification: verifyTask(task, { events, responseText, adapterEvidence: evidence }),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import { discoverFiles } from "./discovery.mjs";
|
||||
import { OPERATION_NAMES, WORKFLOW_ACTIONS } from "./operations.mjs";
|
||||
import { casesDirectory as defaultCasesDirectory } from "./paths.mjs";
|
||||
|
||||
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
const EVIDENCE_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
|
||||
const TOP_LEVEL_KEYS = new Set([
|
||||
"$schema",
|
||||
"schemaVersion",
|
||||
"id",
|
||||
"title",
|
||||
"order",
|
||||
"suite",
|
||||
"tags",
|
||||
"seed",
|
||||
"source",
|
||||
"fixture",
|
||||
"prompts",
|
||||
"coverage",
|
||||
"assertions",
|
||||
"smoke",
|
||||
]);
|
||||
|
||||
function isObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function checkString(errors, value, path) {
|
||||
if (typeof value !== "string" || value.trim() === "") errors.push(`${path} must be a string`);
|
||||
}
|
||||
|
||||
function validateAssertion(errors, assertion, path, kind) {
|
||||
if (!isObject(assertion)) {
|
||||
errors.push(`${path} must be an object`);
|
||||
return;
|
||||
}
|
||||
checkString(errors, assertion.label, `${path}.label`);
|
||||
if (kind === "site") {
|
||||
checkString(errors, assertion.type, `${path}.type`);
|
||||
if (!Number.isInteger(assertion.minCount) || assertion.minCount < 1) {
|
||||
errors.push(`${path}.minCount must be a positive integer`);
|
||||
}
|
||||
if (assertion.where !== undefined && !isObject(assertion.where)) {
|
||||
errors.push(`${path}.where must be an object`);
|
||||
}
|
||||
} else if (kind === "response") {
|
||||
checkString(errors, assertion.includes, `${path}.includes`);
|
||||
} else {
|
||||
checkString(errors, assertion.key, `${path}.key`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateWorkflowStep(errors, step, path) {
|
||||
if (!isObject(step)) {
|
||||
errors.push(`${path} must be an object`);
|
||||
return;
|
||||
}
|
||||
if (!WORKFLOW_ACTIONS.has(step.action)) {
|
||||
errors.push(`${path}.action is unsupported: ${JSON.stringify(step.action)}`);
|
||||
}
|
||||
if (step.saveAs !== undefined && !ID_PATTERN.test(step.saveAs)) {
|
||||
errors.push(`${path}.saveAs must use lowercase kebab-case`);
|
||||
}
|
||||
if (step.evidence !== undefined && !EVIDENCE_PATTERN.test(step.evidence)) {
|
||||
errors.push(`${path}.evidence must be an identifier`);
|
||||
}
|
||||
if (["click", "hover", "fill", "select"].includes(step.action) && !step.selector && !step.ref) {
|
||||
errors.push(`${path} requires selector or ref`);
|
||||
}
|
||||
if (step.action === "fill") checkString(errors, step.value, `${path}.value`);
|
||||
if (step.action === "select") {
|
||||
if (!Array.isArray(step.values) || step.values.length === 0) {
|
||||
errors.push(`${path}.values must be a non-empty array`);
|
||||
} else if (step.values.some((value) => typeof value !== "string")) {
|
||||
errors.push(`${path}.values must contain only strings`);
|
||||
}
|
||||
}
|
||||
if (step.action === "press") checkString(errors, step.key, `${path}.key`);
|
||||
if (step.action === "wait") checkString(errors, step.duration, `${path}.duration`);
|
||||
if (step.action === "wait-site-event") checkString(errors, step.type, `${path}.type`);
|
||||
if (["tab-select", "tab-close", "borrow", "return"].includes(step.action)) {
|
||||
checkString(errors, step.tabId, `${path}.tabId`);
|
||||
}
|
||||
if (step.action === "resize") {
|
||||
if (
|
||||
!Number.isInteger(step.width) ||
|
||||
step.width < 1 ||
|
||||
!Number.isInteger(step.height) ||
|
||||
step.height < 1
|
||||
) {
|
||||
errors.push(`${path}.width and height must be positive integers`);
|
||||
}
|
||||
}
|
||||
if (step.action === "emulate") checkString(errors, step.device, `${path}.device`);
|
||||
if (step.action === "request-help") checkString(errors, step.prompt, `${path}.prompt`);
|
||||
if (step.timeoutMs !== undefined && (!Number.isInteger(step.timeoutMs) || step.timeoutMs < 1)) {
|
||||
errors.push(`${path}.timeoutMs must be a positive integer`);
|
||||
}
|
||||
}
|
||||
|
||||
export function validateCaseManifest(manifest, { source = "case manifest" } = {}) {
|
||||
const errors = [];
|
||||
if (!isObject(manifest)) return [`${source} must contain a JSON object`];
|
||||
for (const key of Object.keys(manifest)) {
|
||||
if (!TOP_LEVEL_KEYS.has(key)) errors.push(`unknown top-level property ${key}`);
|
||||
}
|
||||
if (manifest.schemaVersion !== 1) errors.push("schemaVersion must equal 1");
|
||||
if (!ID_PATTERN.test(manifest.id ?? "")) errors.push("id must use lowercase kebab-case");
|
||||
checkString(errors, manifest.title, "title");
|
||||
if (manifest.order !== undefined && (!Number.isInteger(manifest.order) || manifest.order < 0)) {
|
||||
errors.push("order must be a non-negative integer");
|
||||
}
|
||||
if (!ID_PATTERN.test(manifest.suite ?? "")) errors.push("suite must use lowercase kebab-case");
|
||||
if (!Array.isArray(manifest.tags) || manifest.tags.some((tag) => !ID_PATTERN.test(tag))) {
|
||||
errors.push("tags must be an array of lowercase kebab-case strings");
|
||||
} else if (new Set(manifest.tags).size !== manifest.tags.length) {
|
||||
errors.push("tags must not contain duplicates");
|
||||
}
|
||||
if (!isObject(manifest.fixture)) errors.push("fixture must be an object");
|
||||
else if (
|
||||
typeof manifest.fixture.startPath !== "string" ||
|
||||
!manifest.fixture.startPath.startsWith("/")
|
||||
) {
|
||||
errors.push("fixture.startPath must be an absolute URL path");
|
||||
}
|
||||
if (!isObject(manifest.prompts)) errors.push("prompts must be an object");
|
||||
else {
|
||||
checkString(errors, manifest.prompts.en, "prompts.en");
|
||||
checkString(errors, manifest.prompts["zh-CN"], "prompts.zh-CN");
|
||||
}
|
||||
if (!Array.isArray(manifest.coverage)) errors.push("coverage must be an array");
|
||||
else {
|
||||
for (const operation of manifest.coverage) {
|
||||
if (!OPERATION_NAMES.has(operation))
|
||||
errors.push(`coverage contains unknown operation ${operation}`);
|
||||
}
|
||||
if (new Set(manifest.coverage).size !== manifest.coverage.length) {
|
||||
errors.push("coverage must not contain duplicates");
|
||||
}
|
||||
}
|
||||
if (!isObject(manifest.assertions)) errors.push("assertions must be an object");
|
||||
else {
|
||||
for (const kind of ["site", "response", "adapter"]) {
|
||||
if (!Array.isArray(manifest.assertions[kind])) {
|
||||
errors.push(`assertions.${kind} must be an array`);
|
||||
} else {
|
||||
manifest.assertions[kind].forEach((assertion, index) =>
|
||||
validateAssertion(errors, assertion, `assertions.${kind}[${index}]`, kind),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!isObject(manifest.smoke) || !Array.isArray(manifest.smoke.steps)) {
|
||||
errors.push("smoke.steps must be an array");
|
||||
} else {
|
||||
manifest.smoke.steps.forEach((step, index) =>
|
||||
validateWorkflowStep(errors, step, `smoke.steps[${index}]`),
|
||||
);
|
||||
}
|
||||
if (manifest.seed !== undefined && !Number.isSafeInteger(manifest.seed)) {
|
||||
errors.push("seed must be a safe integer");
|
||||
}
|
||||
if (manifest.source !== undefined && !isObject(manifest.source)) {
|
||||
errors.push("source must be an object");
|
||||
}
|
||||
return errors.map((error) => `${source}: ${error}`);
|
||||
}
|
||||
|
||||
function normalizeCase(manifest, sourceFile) {
|
||||
return Object.freeze({
|
||||
...manifest,
|
||||
startPath: manifest.fixture.startPath,
|
||||
siteAssertions: manifest.assertions.site,
|
||||
responseAssertions: manifest.assertions.response,
|
||||
adapterAssertions: manifest.assertions.adapter,
|
||||
smokeSteps: manifest.smoke.steps,
|
||||
sourceFile,
|
||||
});
|
||||
}
|
||||
|
||||
export async function loadCaseManifests({ casesDirectory = defaultCasesDirectory } = {}) {
|
||||
const files = await discoverFiles(casesDirectory, (path) => path.endsWith(".case.json"));
|
||||
const cases = [];
|
||||
const ids = new Map();
|
||||
const errors = [];
|
||||
for (const file of files) {
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(file, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${file}: invalid JSON: ${error.message}`);
|
||||
continue;
|
||||
}
|
||||
const manifestErrors = validateCaseManifest(manifest, { source: file });
|
||||
errors.push(...manifestErrors);
|
||||
if (typeof manifest?.id === "string") {
|
||||
if (ids.has(manifest.id)) {
|
||||
errors.push(`${file}: duplicate id also declared by ${ids.get(manifest.id)}`);
|
||||
} else {
|
||||
ids.set(manifest.id, file);
|
||||
}
|
||||
}
|
||||
if (manifestErrors.length === 0) cases.push(normalizeCase(manifest, file));
|
||||
}
|
||||
if (files.length === 0) errors.push(`${casesDirectory}: no .case.json files found`);
|
||||
if (errors.length > 0) throw new Error(`case validation failed:\n${errors.join("\n")}`);
|
||||
cases.sort(
|
||||
(left, right) =>
|
||||
left.suite.localeCompare(right.suite) ||
|
||||
(left.order ?? Number.MAX_SAFE_INTEGER) - (right.order ?? Number.MAX_SAFE_INTEGER) ||
|
||||
left.id.localeCompare(right.id),
|
||||
);
|
||||
return Object.freeze(cases);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { OPERATION_CATALOG, WORKFLOW_ACTION_OPERATIONS } from "./operations.mjs";
|
||||
|
||||
export function buildOperationCoverage(cases) {
|
||||
return OPERATION_CATALOG.map(({ operation, manualNote }) => {
|
||||
const agentCases = cases
|
||||
.filter((testCase) => testCase.coverage.includes(operation))
|
||||
.map(({ id }) => id);
|
||||
let smokeCases;
|
||||
if (["session.start", "session.stop", "session.list"].includes(operation)) {
|
||||
smokeCases = cases.filter(({ smokeSteps }) => smokeSteps.length > 0).map(({ id }) => id);
|
||||
} else {
|
||||
smokeCases = cases
|
||||
.filter(({ smokeSteps }) =>
|
||||
smokeSteps.some((step) => WORKFLOW_ACTION_OPERATIONS[step.action] === operation),
|
||||
)
|
||||
.map(({ id }) => id);
|
||||
}
|
||||
return {
|
||||
operation,
|
||||
agentCases,
|
||||
smokeCases,
|
||||
directSmoke: smokeCases.length > 0,
|
||||
note: smokeCases.length === 0 && manualNote ? `manual lane: ${manualNote}` : undefined,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
export async function discoverFiles(root, predicate) {
|
||||
const files = [];
|
||||
|
||||
async function visit(directory) {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(directory, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
||||
const path = join(directory, entry.name);
|
||||
if (entry.isDirectory()) await visit(path);
|
||||
else if (entry.isFile() && predicate(path)) files.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
await visit(root);
|
||||
return files;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import { discoverFiles } from "./discovery.mjs";
|
||||
import { casesDirectory, fixturesDirectory } from "./paths.mjs";
|
||||
|
||||
export async function loadFixtureRegistry({ roots = [fixturesDirectory, casesDirectory] } = {}) {
|
||||
const files = (
|
||||
await Promise.all(
|
||||
roots.map((root) => discoverFiles(root, (path) => path.endsWith(".fixture.mjs"))),
|
||||
)
|
||||
).flat();
|
||||
const routes = new Map();
|
||||
const definitions = [];
|
||||
const errors = [];
|
||||
|
||||
for (const file of files.sort()) {
|
||||
let definition;
|
||||
try {
|
||||
definition = (await import(pathToFileURL(file).href)).default;
|
||||
} catch (error) {
|
||||
errors.push(`${file}: could not import fixture: ${error.message}`);
|
||||
continue;
|
||||
}
|
||||
if (!definition || typeof definition.id !== "string") {
|
||||
errors.push(`${file}: default export must define id`);
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(definition.routes) || definition.routes.length === 0) {
|
||||
errors.push(`${file}: default export must define non-empty routes`);
|
||||
continue;
|
||||
}
|
||||
if (typeof definition.render !== "function") {
|
||||
errors.push(`${file}: default export must define render(context)`);
|
||||
continue;
|
||||
}
|
||||
definitions.push({ ...definition, sourceFile: file });
|
||||
for (const route of definition.routes) {
|
||||
if (typeof route !== "string" || !route.startsWith("/")) {
|
||||
errors.push(`${file}: invalid route ${JSON.stringify(route)}`);
|
||||
} else if (routes.has(route)) {
|
||||
errors.push(
|
||||
`${file}: route ${route} is already registered by ${routes.get(route).sourceFile}`,
|
||||
);
|
||||
} else {
|
||||
routes.set(route, { definition, sourceFile: file });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) throw new Error(`fixture validation failed:\n${errors.join("\n")}`);
|
||||
return Object.freeze({
|
||||
definitions: Object.freeze(definitions),
|
||||
routes: Object.freeze([...routes.keys()].sort()),
|
||||
hasRoute(pathname) {
|
||||
return routes.has(pathname);
|
||||
},
|
||||
render(pathname, context) {
|
||||
const entry = routes.get(pathname);
|
||||
return entry ? entry.definition.render({ ...context, pathname }) : null;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function validateCaseFixtureLinks(cases, fixtureRegistry) {
|
||||
return cases
|
||||
.filter(({ startPath }) => !fixtureRegistry.hasRoute(startPath))
|
||||
.map(({ id, startPath }) => `case ${id} references unregistered fixture route ${startPath}`);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
const stylesheet = `
|
||||
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||
body { margin: 0; background: #f4f7fb; color: #182033; }
|
||||
main { width: min(760px, calc(100% - 40px)); margin: 48px auto; }
|
||||
.card { background: white; border: 1px solid #d9e1ef; border-radius: 14px; padding: 24px; box-shadow: 0 8px 28px #20304a12; }
|
||||
h1 { margin-top: 0; }
|
||||
a, button { font: inherit; }
|
||||
button, .button { display: inline-block; border: 0; border-radius: 8px; padding: 10px 16px; background: #3157d5; color: white; cursor: pointer; text-decoration: none; }
|
||||
label { display: block; margin: 16px 0; font-weight: 600; }
|
||||
input, textarea, select { display: block; box-sizing: border-box; width: 100%; margin-top: 6px; border: 1px solid #9caac2; border-radius: 7px; padding: 9px; font: inherit; }
|
||||
textarea { min-height: 90px; }
|
||||
.marker { border-left: 4px solid #3157d5; padding: 8px 12px; background: #eef2ff; font-family: ui-monospace, monospace; }
|
||||
.muted { color: #5c667a; }
|
||||
#account-menu { position: relative; display: inline-block; padding-bottom: 12px; }
|
||||
#hover-panel { display: none; position: absolute; z-index: 2; top: 34px; left: 0; width: 190px; padding: 10px; background: white; border: 1px solid #9caac2; border-radius: 8px; box-shadow: 0 8px 24px #20304a22; }
|
||||
#account-menu:hover #hover-panel, #account-menu.open #hover-panel { display: block; }
|
||||
#hover-panel button { width: 100%; }
|
||||
#inspect-card { margin-top: 80px; }
|
||||
code { font-family: ui-monospace, monospace; }
|
||||
`;
|
||||
|
||||
const clientScript = `
|
||||
(() => {
|
||||
const runId = new URLSearchParams(location.search).get("run");
|
||||
const navigation = performance.getEntriesByType("navigation")[0];
|
||||
const send = (type, data = {}) => {
|
||||
if (!runId) return Promise.resolve();
|
||||
return fetch("/api/events", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ runId, type, path: location.pathname, data }),
|
||||
keepalive: true,
|
||||
}).catch(() => undefined);
|
||||
};
|
||||
window.browserEval = { runId, send };
|
||||
window.addEventListener("pageshow", (event) => {
|
||||
send("page.shown", {
|
||||
path: location.pathname,
|
||||
persisted: event.persisted,
|
||||
navigationType: navigation?.type ?? "unknown",
|
||||
});
|
||||
});
|
||||
})();
|
||||
`;
|
||||
|
||||
export function escapeHtml(value) {
|
||||
return String(value)
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
|
||||
export function withRun(path, runId, extra = {}) {
|
||||
const url = new URL(path, "http://browser-eval.local");
|
||||
url.searchParams.set("run", runId);
|
||||
for (const [name, value] of Object.entries(extra)) {
|
||||
if (value !== undefined) url.searchParams.set(name, String(value));
|
||||
}
|
||||
return `${url.pathname}${url.search}`;
|
||||
}
|
||||
|
||||
export function page({ title, body, script = "" }) {
|
||||
return `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>${escapeHtml(title)}</title>
|
||||
<style>${stylesheet}</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>${body}</main>
|
||||
<script>${clientScript}</script>
|
||||
${script ? `<script>${script}</script>` : ""}
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
export const OPERATION_CATALOG = [
|
||||
{ operation: "session.start" },
|
||||
{ operation: "session.stop" },
|
||||
{ operation: "session.list" },
|
||||
{ operation: "page.navigate" },
|
||||
{ operation: "page.back" },
|
||||
{ operation: "page.forward" },
|
||||
{ operation: "page.reload" },
|
||||
{ operation: "page.wait" },
|
||||
{ operation: "inspect.observe" },
|
||||
{ operation: "inspect.snapshot" },
|
||||
{ operation: "inspect.html" },
|
||||
{ operation: "inspect.screenshot" },
|
||||
{ operation: "inspect.console" },
|
||||
{ operation: "inspect.network" },
|
||||
{ operation: "interact.click" },
|
||||
{ operation: "interact.hover" },
|
||||
{ operation: "interact.fill" },
|
||||
{ operation: "interact.select" },
|
||||
{ operation: "interact.press" },
|
||||
{ operation: "tabs.list" },
|
||||
{ operation: "tabs.create" },
|
||||
{ operation: "tabs.select" },
|
||||
{ operation: "tabs.close" },
|
||||
{
|
||||
operation: "tabs.borrow",
|
||||
manualNote: "requires an existing user tab and ownership confirmation",
|
||||
},
|
||||
{ operation: "tabs.return", manualNote: "paired with borrowing a real user tab" },
|
||||
{ operation: "assist.resize" },
|
||||
{ operation: "assist.emulate" },
|
||||
{ operation: "assist.request-help", manualNote: "intentionally waits for human interaction" },
|
||||
];
|
||||
|
||||
export const WORKFLOW_ACTION_OPERATIONS = {
|
||||
navigate: "page.navigate",
|
||||
back: "page.back",
|
||||
forward: "page.forward",
|
||||
reload: "page.reload",
|
||||
wait: "page.wait",
|
||||
observe: "inspect.observe",
|
||||
snapshot: "inspect.snapshot",
|
||||
html: "inspect.html",
|
||||
screenshot: "inspect.screenshot",
|
||||
console: "inspect.console",
|
||||
network: "inspect.network",
|
||||
click: "interact.click",
|
||||
hover: "interact.hover",
|
||||
fill: "interact.fill",
|
||||
select: "interact.select",
|
||||
press: "interact.press",
|
||||
"tab-list": "tabs.list",
|
||||
"tab-create": "tabs.create",
|
||||
"tab-select": "tabs.select",
|
||||
"tab-close": "tabs.close",
|
||||
borrow: "tabs.borrow",
|
||||
return: "tabs.return",
|
||||
resize: "assist.resize",
|
||||
emulate: "assist.emulate",
|
||||
"request-help": "assist.request-help",
|
||||
};
|
||||
|
||||
export const WORKFLOW_ACTIONS = new Set([
|
||||
...Object.keys(WORKFLOW_ACTION_OPERATIONS),
|
||||
"wait-site-event",
|
||||
]);
|
||||
|
||||
export const OPERATION_NAMES = new Set(OPERATION_CATALOG.map(({ operation }) => operation));
|
||||
@@ -0,0 +1,63 @@
|
||||
function readPath(value, path) {
|
||||
return path.split(".").reduce((current, part) => current?.[part], value);
|
||||
}
|
||||
|
||||
function eventMatches(event, assertion) {
|
||||
if (event.type !== assertion.type) return false;
|
||||
return Object.entries(assertion.where ?? {}).every(
|
||||
([path, expected]) => readPath(event, path) === expected,
|
||||
);
|
||||
}
|
||||
|
||||
export function verifyTask(task, { events = [], responseText = "", adapterEvidence = {} } = {}) {
|
||||
const site = task.siteAssertions.map((assertion) => {
|
||||
const actual = events.filter((event) => eventMatches(event, assertion)).length;
|
||||
return {
|
||||
kind: "site",
|
||||
label: assertion.label,
|
||||
status: actual >= assertion.minCount ? "passed" : "failed",
|
||||
expected: `at least ${assertion.minCount}`,
|
||||
actual,
|
||||
};
|
||||
});
|
||||
|
||||
const response = task.responseAssertions.map((assertion) => ({
|
||||
kind: "response",
|
||||
label: assertion.label,
|
||||
status: responseText.includes(assertion.includes) ? "passed" : "failed",
|
||||
expected: `response includes ${JSON.stringify(assertion.includes)}`,
|
||||
}));
|
||||
|
||||
const adapter = task.adapterAssertions.map((assertion) => {
|
||||
if (!Object.hasOwn(adapterEvidence, assertion.key)) {
|
||||
return {
|
||||
kind: "adapter",
|
||||
label: assertion.label,
|
||||
status: "unverified",
|
||||
expected: assertion.key,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "adapter",
|
||||
label: assertion.label,
|
||||
status: adapterEvidence[assertion.key] ? "passed" : "failed",
|
||||
expected: assertion.key,
|
||||
actual: adapterEvidence[assertion.key],
|
||||
};
|
||||
});
|
||||
|
||||
const checks = [...site, ...response, ...adapter];
|
||||
const failed = checks.filter(({ status }) => status === "failed").length;
|
||||
const unverified = checks.filter(({ status }) => status === "unverified").length;
|
||||
return {
|
||||
taskId: task.id,
|
||||
status: failed > 0 ? "failed" : unverified > 0 ? "passed-with-unverified" : "passed",
|
||||
summary: {
|
||||
passed: checks.length - failed - unverified,
|
||||
failed,
|
||||
unverified,
|
||||
total: checks.length,
|
||||
},
|
||||
checks,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const evalDirectory = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
export const casesDirectory = join(evalDirectory, "cases");
|
||||
export const fixturesDirectory = join(evalDirectory, "fixtures");
|
||||
export const schemasDirectory = join(evalDirectory, "schemas");
|
||||
export const resultsDirectory = join(evalDirectory, "results");
|
||||
@@ -0,0 +1,71 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const MAX_CAPTURE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
function appendCaptured(current, chunk) {
|
||||
if (current.length >= MAX_CAPTURE_BYTES) return current;
|
||||
return `${current}${chunk}`.slice(0, MAX_CAPTURE_BYTES);
|
||||
}
|
||||
|
||||
export function runProcess(command, args, { cwd, env, timeoutMs = 300_000 } = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let settled = false;
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: { ...process.env, ...env },
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout = appendCaptured(stdout, chunk.toString());
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr = appendCaptured(stderr, chunk.toString());
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
|
||||
}, timeoutMs);
|
||||
|
||||
const finish = (result) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
resolve({
|
||||
command,
|
||||
args,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut,
|
||||
durationMs: Date.now() - startedAt,
|
||||
...result,
|
||||
});
|
||||
};
|
||||
|
||||
child.once("error", (error) => finish({ exitCode: null, signal: null, error: error.message }));
|
||||
child.once("close", (exitCode, signal) => finish({ exitCode, signal }));
|
||||
});
|
||||
}
|
||||
|
||||
export function substituteArgs(args, values) {
|
||||
return args.map((argument) => {
|
||||
let result = argument;
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
result = result.replaceAll(`{${key}}`, String(value));
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
export function countPattern(text, pattern) {
|
||||
if (!pattern) return null;
|
||||
const matches = text.match(new RegExp(pattern, "g"));
|
||||
return matches?.length ?? 0;
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { relative, sep } from "node:path";
|
||||
|
||||
import { validateCaseFixtureLinks } from "./fixture-registry.mjs";
|
||||
import { casesDirectory } from "./paths.mjs";
|
||||
|
||||
function templateRoots(value) {
|
||||
if (typeof value === "string") {
|
||||
return [...value.matchAll(/\{([^{}]+)\}/g)].map((match) => match[1].split(".")[0]);
|
||||
}
|
||||
if (Array.isArray(value)) return value.flatMap(templateRoots);
|
||||
if (value && typeof value === "object") return Object.values(value).flatMap(templateRoots);
|
||||
return [];
|
||||
}
|
||||
|
||||
export function validateRepositoryCases(cases, fixtureRegistry) {
|
||||
const errors = [...validateCaseFixtureLinks(cases, fixtureRegistry)];
|
||||
for (const testCase of cases) {
|
||||
const relativePath = relative(casesDirectory, testCase.sourceFile);
|
||||
const suiteDirectory = relativePath.split(sep)[0];
|
||||
if (suiteDirectory !== testCase.suite) {
|
||||
errors.push(
|
||||
`case ${testCase.id} declares suite ${testCase.suite} but is stored under ${suiteDirectory}`,
|
||||
);
|
||||
}
|
||||
|
||||
const availableVariables = new Set(["baseUrl", "runId", "seed", "sessionId", "url"]);
|
||||
const evidence = new Set(["sessionStopped"]);
|
||||
for (const [index, step] of testCase.smokeSteps.entries()) {
|
||||
for (const root of templateRoots(step)) {
|
||||
if (!availableVariables.has(root)) {
|
||||
errors.push(
|
||||
`case ${testCase.id} smoke.steps[${index}] references unavailable variable ${root}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (step.saveAs) availableVariables.add(step.saveAs);
|
||||
if (step.evidence) evidence.add(step.evidence);
|
||||
}
|
||||
for (const assertion of testCase.adapterAssertions) {
|
||||
if (!evidence.has(assertion.key)) {
|
||||
errors.push(
|
||||
`case ${testCase.id} expects adapter evidence ${assertion.key} but no smoke step produces it`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function repositorySummary(cases, fixtureRegistry) {
|
||||
const suites = Object.fromEntries(
|
||||
[...new Set(cases.map(({ suite }) => suite))]
|
||||
.sort()
|
||||
.map((suite) => [suite, cases.filter((testCase) => testCase.suite === suite).length]),
|
||||
);
|
||||
const tags = [...new Set(cases.flatMap(({ tags }) => tags))].sort();
|
||||
return {
|
||||
cases: cases.length,
|
||||
suites,
|
||||
tags: tags.length,
|
||||
fixtureModules: fixtureRegistry.definitions.length,
|
||||
fixtureRoutes: fixtureRegistry.routes.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { mkdir, realpath, stat, writeFile } from "node:fs/promises";
|
||||
import { join, relative, sep } from "node:path";
|
||||
|
||||
import { casesDirectory, evalDirectory } from "./paths.mjs";
|
||||
|
||||
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
|
||||
async function exists(path) {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function moduleSpecifier(fromDirectory, target) {
|
||||
const path = relative(await realpath(fromDirectory), await realpath(target))
|
||||
.split(sep)
|
||||
.join("/");
|
||||
return path.startsWith(".") ? path : `./${path}`;
|
||||
}
|
||||
|
||||
export async function scaffoldCase({
|
||||
id,
|
||||
title = id,
|
||||
suite = "regression",
|
||||
source,
|
||||
rootCasesDirectory = casesDirectory,
|
||||
}) {
|
||||
if (!ID_PATTERN.test(id ?? "")) throw new Error("case id must use lowercase kebab-case");
|
||||
if (!ID_PATTERN.test(suite ?? "")) throw new Error("suite must use lowercase kebab-case");
|
||||
const directory = join(rootCasesDirectory, suite, id);
|
||||
if (await exists(directory)) throw new Error(`case directory already exists: ${directory}`);
|
||||
await mkdir(directory, { recursive: true });
|
||||
|
||||
const marker = `CASE-${id.toUpperCase().replaceAll("-", "_")}`;
|
||||
const startPath = `/cases/${id}`;
|
||||
const manifest = {
|
||||
$schema: "../../../schemas/case.schema.json",
|
||||
schemaVersion: 1,
|
||||
id,
|
||||
title,
|
||||
order: 10,
|
||||
suite,
|
||||
tags: ["badcase", "regression"],
|
||||
...(source ? { source: { type: "user-badcase", reference: source } } : {}),
|
||||
fixture: { startPath },
|
||||
prompts: {
|
||||
en: `Open {url}, complete the minimal regression scenario, report ${marker}, and close the browser session.`,
|
||||
"zh-CN": `打开 {url},完成最小回归场景,报告 ${marker},最后关闭浏览器会话。`,
|
||||
},
|
||||
coverage: ["session.start", "session.stop", "page.navigate", "inspect.observe"],
|
||||
assertions: {
|
||||
site: [
|
||||
{
|
||||
label: "regression fixture was displayed",
|
||||
type: "page.shown",
|
||||
where: { "data.path": startPath },
|
||||
minCount: 1,
|
||||
},
|
||||
],
|
||||
response: [{ label: "reported regression marker", includes: marker }],
|
||||
adapter: [{ label: "browser session was closed", key: "sessionStopped" }],
|
||||
},
|
||||
smoke: { steps: [{ action: "navigate" }, { action: "observe" }] },
|
||||
};
|
||||
|
||||
const fixtureHelper = await moduleSpecifier(
|
||||
directory,
|
||||
join(evalDirectory, "lib", "fixtures.mjs"),
|
||||
);
|
||||
const fixture = `import { escapeHtml, page } from ${JSON.stringify(fixtureHelper)};
|
||||
|
||||
const title = ${JSON.stringify(title)};
|
||||
|
||||
export default {
|
||||
id: ${JSON.stringify(id)},
|
||||
routes: [${JSON.stringify(startPath)}],
|
||||
render() {
|
||||
return page({
|
||||
title,
|
||||
body: \`<section class="card"><h1>\${escapeHtml(title)}</h1><p class="marker">${marker}</p><p>Replace this page with a privacy-safe minimal reproduction.</p></section>\`,
|
||||
});
|
||||
},
|
||||
};
|
||||
`;
|
||||
const readme = `# ${title}
|
||||
|
||||
## Original symptom
|
||||
|
||||
Describe what the user observed without including private data or proprietary page content.
|
||||
|
||||
## Minimal reproduction
|
||||
|
||||
Explain the smallest DOM, timing, or browser-state condition that reproduces the failure.
|
||||
|
||||
## Expected behavior
|
||||
|
||||
Describe the observable success condition and update the case assertions accordingly.
|
||||
|
||||
## Regression history
|
||||
|
||||
- Source: ${source ?? "add issue, PR, or anonymized user report reference"}
|
||||
- Fix: add the fixing PR when available
|
||||
`;
|
||||
|
||||
const files = {
|
||||
manifest: join(directory, `${id}.case.json`),
|
||||
fixture: join(directory, `${id}.fixture.mjs`),
|
||||
readme: join(directory, "README.md"),
|
||||
};
|
||||
await Promise.all([
|
||||
writeFile(files.manifest, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }),
|
||||
writeFile(files.fixture, fixture, { flag: "wx" }),
|
||||
writeFile(files.readme, readme, { flag: "wx" }),
|
||||
]);
|
||||
return { directory, files };
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
function hashSeed(value) {
|
||||
const text = String(value);
|
||||
let hash = 2166136261;
|
||||
for (const character of text) {
|
||||
hash ^= character.codePointAt(0);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function mulberry32(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state += 0x6d2b79f5;
|
||||
let value = state;
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeSeed(value) {
|
||||
const numeric = Number(value);
|
||||
return Number.isSafeInteger(numeric) ? numeric : hashSeed(value);
|
||||
}
|
||||
|
||||
export function describeSeed(value) {
|
||||
const seed = normalizeSeed(value);
|
||||
const random = mulberry32(seed);
|
||||
const choose = (values) => values[Math.floor(random() * values.length)];
|
||||
return {
|
||||
seed,
|
||||
labelMode: choose(["wrapped", "for", "aria"]),
|
||||
nestingDepth: 1 + Math.floor(random() * 4),
|
||||
hydrationDelayMs: choose([0, 40, 120, 250]),
|
||||
decoy: choose(["none", "disabled-input", "duplicate-text"]),
|
||||
fieldOrder: choose(["text-first", "select-first"]),
|
||||
idSuffix: hashSeed(`${seed}:id`).toString(36).slice(0, 6),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { createServer } from "node:http";
|
||||
|
||||
import { loadFixtureRegistry } from "./fixture-registry.mjs";
|
||||
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]{1,120}$/;
|
||||
|
||||
function validRunId(value) {
|
||||
return typeof value === "string" && RUN_ID_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function json(response, status, value) {
|
||||
response.writeHead(status, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
});
|
||||
response.end(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function html(response, status, value) {
|
||||
response.writeHead(status, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "text/html; charset=utf-8",
|
||||
});
|
||||
response.end(value);
|
||||
}
|
||||
|
||||
async function readJson(request) {
|
||||
let raw = "";
|
||||
for await (const chunk of request) {
|
||||
raw += chunk;
|
||||
if (raw.length > 64 * 1024) {
|
||||
throw new Error("request body exceeds 64 KiB");
|
||||
}
|
||||
}
|
||||
return raw ? JSON.parse(raw) : {};
|
||||
}
|
||||
|
||||
export function createEvalServer({ host = "127.0.0.1", port = 0, fixtureRegistry } = {}) {
|
||||
const runs = new Map();
|
||||
let sequence = 0;
|
||||
let server;
|
||||
let registry = fixtureRegistry;
|
||||
|
||||
function eventsFor(runId) {
|
||||
if (!runs.has(runId)) runs.set(runId, []);
|
||||
return runs.get(runId);
|
||||
}
|
||||
|
||||
function record(runId, type, data = {}, metadata = {}) {
|
||||
if (!validRunId(runId)) return null;
|
||||
const event = {
|
||||
sequence: ++sequence,
|
||||
timestamp: new Date().toISOString(),
|
||||
type,
|
||||
data,
|
||||
...metadata,
|
||||
};
|
||||
eventsFor(runId).push(event);
|
||||
return event;
|
||||
}
|
||||
|
||||
function snapshot(runId) {
|
||||
return { runId, events: [...(runs.get(runId) ?? [])] };
|
||||
}
|
||||
|
||||
function reset(runId) {
|
||||
runs.set(runId, []);
|
||||
}
|
||||
|
||||
async function handle(request, response) {
|
||||
const origin = `http://${request.headers.host ?? `${host}:${port}`}`;
|
||||
const url = new URL(request.url ?? "/", origin);
|
||||
|
||||
if (request.method === "GET" && url.pathname === "/healthz") {
|
||||
return json(response, 200, { ok: true });
|
||||
}
|
||||
|
||||
if (request.method === "POST" && url.pathname === "/api/events") {
|
||||
const body = await readJson(request);
|
||||
if (!validRunId(body.runId) || typeof body.type !== "string") {
|
||||
return json(response, 400, { error: "runId and type are required" });
|
||||
}
|
||||
const event = record(body.runId, body.type, body.data ?? {}, {
|
||||
path: typeof body.path === "string" ? body.path : undefined,
|
||||
});
|
||||
return json(response, 202, { accepted: true, sequence: event.sequence });
|
||||
}
|
||||
|
||||
const runMatch = url.pathname.match(/^\/api\/runs\/([^/]+)$/);
|
||||
if (runMatch) {
|
||||
const runId = decodeURIComponent(runMatch[1]);
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
if (request.method === "GET") return json(response, 200, snapshot(runId));
|
||||
if (request.method === "DELETE") {
|
||||
reset(runId);
|
||||
return json(response, 200, { runId, reset: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (request.method === "GET" && url.pathname === "/api/ping") {
|
||||
const runId = url.searchParams.get("run");
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
record(runId, "network.ping", { token: "NETWORK-73" }, { path: url.pathname });
|
||||
return json(response, 200, { ok: true, token: "NETWORK-73" });
|
||||
}
|
||||
|
||||
if (request.method !== "GET") return json(response, 405, { error: "method not allowed" });
|
||||
|
||||
const runId = url.searchParams.get("run") ?? "manual";
|
||||
if (!validRunId(runId)) return json(response, 400, { error: "invalid run id" });
|
||||
|
||||
const fixture = registry.render(url.pathname, {
|
||||
runId,
|
||||
query: url.searchParams,
|
||||
record: (type, data, metadata) => record(runId, type, data, metadata),
|
||||
});
|
||||
if (!fixture) return html(response, 404, "<!doctype html><h1>Not found</h1>");
|
||||
record(runId, "page.request", { path: url.pathname }, { path: url.pathname });
|
||||
return html(response, 200, fixture);
|
||||
}
|
||||
|
||||
return {
|
||||
record,
|
||||
reset,
|
||||
snapshot,
|
||||
async start() {
|
||||
if (server) throw new Error("eval server is already running");
|
||||
registry ??= await loadFixtureRegistry();
|
||||
server = createServer((request, response) => {
|
||||
handle(request, response).catch((error) => {
|
||||
json(response, 500, { error: error instanceof Error ? error.message : String(error) });
|
||||
});
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, host, resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("unexpected server address");
|
||||
return { host, port: address.port, baseUrl: `http://${host}:${address.port}` };
|
||||
},
|
||||
async stop() {
|
||||
if (!server) return;
|
||||
const active = server;
|
||||
server = undefined;
|
||||
await new Promise((resolve, reject) => {
|
||||
active.close((error) => (error ? reject(error) : resolve()));
|
||||
active.closeAllConnections?.();
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
function durationFor(result) {
|
||||
if (result.execution?.durationMs !== undefined) return result.execution.durationMs;
|
||||
return (result.steps ?? []).reduce((sum, step) => sum + step.durationMs, 0);
|
||||
}
|
||||
|
||||
export function summarizeReports(reports) {
|
||||
const groups = new Map();
|
||||
for (const report of reports) {
|
||||
for (const result of report.results ?? []) {
|
||||
const adapter = result.adapter ?? report.adapter ?? "unknown";
|
||||
const variant = result.variant ?? report.variant ?? "unspecified";
|
||||
const key = `${adapter}\u0000${variant}`;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
adapter,
|
||||
variant,
|
||||
runs: 0,
|
||||
passed: 0,
|
||||
fullyVerified: 0,
|
||||
executionFailures: 0,
|
||||
errors: 0,
|
||||
durationMs: 0,
|
||||
toolCalls: 0,
|
||||
toolCallSamples: 0,
|
||||
});
|
||||
}
|
||||
const group = groups.get(key);
|
||||
group.runs += 1;
|
||||
group.durationMs += durationFor(result);
|
||||
group.errors += result.metrics?.errorCount ?? (result.executionError ? 1 : 0);
|
||||
if (result.metrics?.toolCallCount !== null && result.metrics?.toolCallCount !== undefined) {
|
||||
group.toolCalls += result.metrics.toolCallCount;
|
||||
group.toolCallSamples += 1;
|
||||
}
|
||||
const executionFailed =
|
||||
Boolean(result.executionError) ||
|
||||
result.execution?.timedOut ||
|
||||
(result.execution?.exitCode !== undefined && result.execution.exitCode !== 0);
|
||||
if (executionFailed) group.executionFailures += 1;
|
||||
if (result.verification?.status !== "failed" && !executionFailed) group.passed += 1;
|
||||
if (result.verification?.status === "passed" && !executionFailed) group.fullyVerified += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return [...groups.values()].map((group) => ({
|
||||
adapter: group.adapter,
|
||||
variant: group.variant,
|
||||
runs: group.runs,
|
||||
passRate: group.runs ? group.passed / group.runs : 0,
|
||||
fullyVerifiedRate: group.runs ? group.fullyVerified / group.runs : 0,
|
||||
executionFailures: group.executionFailures,
|
||||
errors: group.errors,
|
||||
averageToolCalls: group.toolCallSamples
|
||||
? Math.round((group.toolCalls / group.toolCallSamples) * 10) / 10
|
||||
: null,
|
||||
averageDurationMs: group.runs ? Math.round(group.durationMs / group.runs) : 0,
|
||||
}));
|
||||
}
|
||||
|
||||
export function printSummary(rows) {
|
||||
const printable = rows.map((row) => ({
|
||||
adapter: row.adapter,
|
||||
variant: row.variant,
|
||||
runs: row.runs,
|
||||
pass: `${Math.round(row.passRate * 100)}%`,
|
||||
verified: `${Math.round(row.fullyVerifiedRate * 100)}%`,
|
||||
exec_failures: row.executionFailures,
|
||||
errors: row.errors,
|
||||
avg_tools: row.averageToolCalls ?? "-",
|
||||
avg_ms: row.averageDurationMs,
|
||||
}));
|
||||
console.table(printable);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { loadCaseManifests } from "./case-loader.mjs";
|
||||
|
||||
let casesPromise;
|
||||
|
||||
export function allTasks({ reload = false } = {}) {
|
||||
if (reload || !casesPromise) casesPromise = loadCaseManifests();
|
||||
return casesPromise;
|
||||
}
|
||||
|
||||
export function listTasks(cases) {
|
||||
return cases.map(({ id, title, suite, tags, seed }) => ({ id, title, suite, tags, seed }));
|
||||
}
|
||||
|
||||
export function getTask(taskId, cases) {
|
||||
const task = cases.find(({ id }) => id === taskId);
|
||||
if (!task) {
|
||||
const ids = cases.map(({ id }) => id).join(", ");
|
||||
throw new Error(`unknown case ${JSON.stringify(taskId)}; expected one of: ${ids}`);
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
export function taskUrl(task, baseUrl, runId, seed = task.seed) {
|
||||
const url = new URL(task.startPath, baseUrl);
|
||||
url.searchParams.set("run", runId);
|
||||
if (seed !== undefined) url.searchParams.set("seed", String(seed));
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function expandTemplate(template, values) {
|
||||
return Object.entries(values).reduce(
|
||||
(result, [name, value]) => result.replaceAll(`{${name}}`, String(value)),
|
||||
template,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderPrompt(task, { baseUrl, runId, locale = "zh-CN", seed = task.seed }) {
|
||||
const template = task.prompts[locale];
|
||||
if (!template) {
|
||||
throw new Error(`unsupported locale ${JSON.stringify(locale)}; expected en or zh-CN`);
|
||||
}
|
||||
return expandTemplate(template, {
|
||||
baseUrl,
|
||||
runId: encodeURIComponent(runId),
|
||||
seed: seed ?? "",
|
||||
url: taskUrl(task, baseUrl, runId, seed),
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user