源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
Executable
+377
@@ -0,0 +1,377 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { basename, join, relative, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const evalsDir = fileURLToPath(new URL("..", import.meta.url));
|
||||
const repoRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||
const testsDir = join(evalsDir, "specs");
|
||||
const worldsDir = join(evalsDir, "results/.worlds");
|
||||
|
||||
const usage = `Usage: node evals/bin/evals.mjs [test-names...] [flags]
|
||||
|
||||
Run E2E tests:
|
||||
--with-llm-vision Judge vision claims inline (default: defer judging)
|
||||
--local Force isolated local resources and clear inherited remote placement
|
||||
--daytona Set OPENWORK_EVAL_DAYTONA=1
|
||||
--den <url> Set OPENWORK_EVAL_DEN_API_URL=<url>
|
||||
|
||||
Judge then publish evidence:
|
||||
--publish Enter judge-then-publish mode
|
||||
--pr <n> Publish to pull request n
|
||||
--test-run <value> Select a test run path, directory ID, name, or latest (default: latest)
|
||||
--dry-run Render publication output without posting
|
||||
--force Forward force to the publisher
|
||||
|
||||
Other:
|
||||
--help, -h Show this help
|
||||
|
||||
Publish mode cannot be combined with test names, --with-llm-vision, --daytona,
|
||||
--local, or --den. Named tests auto-consent to opt-in flags declared in their source;
|
||||
value-bearing environment variables are never auto-set.
|
||||
|
||||
Run exit codes:
|
||||
0 Passed, or an unfiltered E2E suite completed with expected skips
|
||||
1 One or more tests failed
|
||||
2 A named test skipped and its result is incomplete
|
||||
|
||||
Publish exit codes:
|
||||
0 Judge and publisher succeeded
|
||||
1 Failed claims were published, or publishing failed
|
||||
2 Pending claims require judging before publication
|
||||
`;
|
||||
|
||||
export function consentVarsFromSource(text) {
|
||||
const variables = new Set();
|
||||
const optInPattern = /optIn\s*:\s*\[([^\]]*)\]/gs;
|
||||
const envPattern = /process\.env\.(OPENWORK_EVAL_[A-Z0-9_]+)(?:\?\.trim\(\))?\s*===\s*"1"/g;
|
||||
|
||||
for (const match of text.matchAll(optInPattern)) {
|
||||
for (const literal of match[1].matchAll(/["'](OPENWORK_EVAL_[A-Z0-9_]+)["']/g)) {
|
||||
variables.add(literal[1]);
|
||||
}
|
||||
}
|
||||
for (const match of text.matchAll(envPattern)) variables.add(match[1]);
|
||||
|
||||
return [...variables].sort();
|
||||
}
|
||||
|
||||
function valueAfter(args, index, flag) {
|
||||
const value = args[index + 1];
|
||||
if (!value || value.startsWith("--")) throw new Error(`${flag} requires a value.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseArgs(args) {
|
||||
const options = {
|
||||
testNames: [],
|
||||
withLlmVision: false,
|
||||
local: false,
|
||||
daytona: false,
|
||||
publish: false,
|
||||
dryRun: false,
|
||||
force: false,
|
||||
help: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--help" || arg === "-h") options.help = true;
|
||||
else if (arg === "--with-llm-vision") options.withLlmVision = true;
|
||||
else if (arg === "--local") options.local = true;
|
||||
else if (arg === "--daytona") options.daytona = true;
|
||||
else if (arg === "--publish") options.publish = true;
|
||||
else if (arg === "--dry-run") options.dryRun = true;
|
||||
else if (arg === "--force") options.force = true;
|
||||
else if (arg === "--den" || arg === "--pr" || arg === "--test-run") {
|
||||
const value = valueAfter(args, index, arg);
|
||||
if (arg === "--den") options.den = value;
|
||||
else if (arg === "--pr") options.pr = value;
|
||||
else options.testRun = value;
|
||||
index += 1;
|
||||
} else if (arg.startsWith("-")) {
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
} else {
|
||||
options.testNames.push(arg);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.local && (options.daytona || options.den !== undefined)) {
|
||||
const conflicts = [];
|
||||
if (options.daytona) conflicts.push("--daytona");
|
||||
if (options.den !== undefined) conflicts.push("--den");
|
||||
throw new Error(`--local is mutually exclusive with ${conflicts.join(" and ")}.`);
|
||||
}
|
||||
|
||||
if (options.publish) {
|
||||
const conflicts = [];
|
||||
if (options.testNames.length > 0) conflicts.push("test names");
|
||||
if (options.withLlmVision) conflicts.push("--with-llm-vision");
|
||||
if (options.local) conflicts.push("--local");
|
||||
if (options.daytona) conflicts.push("--daytona");
|
||||
if (options.den !== undefined) conflicts.push("--den");
|
||||
if (conflicts.length > 0) {
|
||||
throw new Error(`--publish is mutually exclusive with ${conflicts.join(", ")}.`);
|
||||
}
|
||||
if (!options.pr && !options.dryRun && !options.help) {
|
||||
throw new Error("--pr <n> is required unless --dry-run is set.");
|
||||
}
|
||||
} else {
|
||||
const publishFlags = [];
|
||||
if (options.pr !== undefined) publishFlags.push("--pr");
|
||||
if (options.testRun !== undefined) publishFlags.push("--test-run");
|
||||
if (options.dryRun) publishFlags.push("--dry-run");
|
||||
if (options.force) publishFlags.push("--force");
|
||||
if (publishFlags.length > 0) {
|
||||
throw new Error(`${publishFlags.join(", ")} require --publish.`);
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
const REMOTE_PLACEMENT_ENV = [
|
||||
"OPENWORK_EVAL_DAYTONA",
|
||||
"OPENWORK_EVAL_DAYTONA_SANDBOX",
|
||||
"OPENWORK_EVAL_DAYTONA_SANDBOX_ID",
|
||||
"OPENWORK_EVAL_DAYTONA_DEN_SANDBOX",
|
||||
"OPENWORK_EVAL_DAYTONA_DESKTOP_SANDBOX",
|
||||
"OPENWORK_EVAL_DEN_API_URL",
|
||||
"OPENWORK_EVAL_DEN_WEB_URL",
|
||||
];
|
||||
|
||||
/** Resolve the child environment before any test process can provision resources. */
|
||||
export function resolveRunEnvironment(options, env = process.env) {
|
||||
const childEnv = { ...env };
|
||||
if (options.local) {
|
||||
for (const name of REMOTE_PLACEMENT_ENV) delete childEnv[name];
|
||||
return childEnv;
|
||||
}
|
||||
if (options.daytona) childEnv.OPENWORK_EVAL_DAYTONA = "1";
|
||||
if (options.den !== undefined) childEnv.OPENWORK_EVAL_DEN_API_URL = options.den;
|
||||
return childEnv;
|
||||
}
|
||||
|
||||
function testFiles(directory = testsDir) {
|
||||
return readdirSync(directory, { recursive: true, withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && /\.e2e\.test\.ts$/.test(entry.name))
|
||||
.map((entry) => join(entry.parentPath, entry.name))
|
||||
.sort();
|
||||
}
|
||||
|
||||
export function resolveTestNames(names, files = testFiles()) {
|
||||
const entries = files.map((file) => ({
|
||||
file,
|
||||
base: basename(file),
|
||||
relative: relative(testsDir, file).split(sep).join("/"),
|
||||
}));
|
||||
const resolved = [];
|
||||
|
||||
for (const name of names) {
|
||||
const normalized = name.replace(/^\.\//, "").replace(/^specs\//, "");
|
||||
let matches = entries.filter((entry) => entry.relative === normalized);
|
||||
if (matches.length === 0) {
|
||||
matches = entries.filter((entry) =>
|
||||
entry.base === `${normalized}.e2e.test.ts`
|
||||
);
|
||||
}
|
||||
if (matches.length === 0) {
|
||||
matches = entries.filter((entry) => entry.base.startsWith(normalized));
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
throw new Error(`Test name "${name}" is ambiguous:\n${matches.map((entry) => ` ${entry.relative}`).join("\n")}`);
|
||||
}
|
||||
if (matches.length === 0) {
|
||||
const close = entries.filter((entry) => entry.base.includes(normalized));
|
||||
throw new Error(`No test matches "${name}". Close candidates:\n${close.length > 0 ? close.map((entry) => ` ${entry.relative}`).join("\n") : " (none)"}`);
|
||||
}
|
||||
if (!resolved.includes(matches[0].file)) resolved.push(matches[0].file);
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function reportAssertions(report) {
|
||||
if (!Array.isArray(report?.testResults)) return [];
|
||||
return report.testResults.flatMap((result) => {
|
||||
if (!Array.isArray(result?.assertionResults)) return [];
|
||||
return result.assertionResults.map((assertion) => ({ assertion, result }));
|
||||
});
|
||||
}
|
||||
|
||||
function reportCount(report, field, status) {
|
||||
if (Number.isFinite(report?.[field])) return report[field];
|
||||
const assertions = reportAssertions(report);
|
||||
if (assertions.length === 0) return null;
|
||||
return assertions.filter(({ assertion }) => status.includes(assertion?.status)).length;
|
||||
}
|
||||
|
||||
export function summarize(report) {
|
||||
const assertions = reportAssertions(report);
|
||||
return {
|
||||
passed: reportCount(report, "numPassedTests", ["passed"]),
|
||||
failed: reportCount(report, "numFailedTests", ["failed"]),
|
||||
skipped: reportCount(report, "numPendingTests", ["pending", "skipped", "todo", "disabled"]),
|
||||
skips: assertions
|
||||
.filter(({ assertion }) => ["pending", "skipped", "todo", "disabled"].includes(assertion?.status))
|
||||
.map(({ assertion, result }) => ({
|
||||
file: basename(result?.name ?? result?.testFilePath ?? "unknown"),
|
||||
title: assertion?.title ?? assertion?.fullName ?? "unknown",
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function verdictFor(summary, { childExit = 0 } = {}) {
|
||||
if ((summary.failed ?? 0) > 0 || childExit !== 0) return "failed";
|
||||
if ((summary.skipped ?? 0) > 0) return "incomplete";
|
||||
return "passed";
|
||||
}
|
||||
|
||||
export function exitCodeFor(verdict, { named = false } = {}) {
|
||||
if (verdict === "failed") return 1;
|
||||
if (verdict === "incomplete" && named) return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function worldSnapshotsSince(startTime, directory = worldsDir) {
|
||||
try {
|
||||
return readdirSync(directory, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
||||
.flatMap((entry) => {
|
||||
try {
|
||||
const path = join(directory, entry.name);
|
||||
const mtime = statSync(path).mtimeMs;
|
||||
return mtime >= startTime ? [{ path, mtime }] : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
.sort((left, right) => right.mtime - left.mtime)
|
||||
.map(({ path }) => path);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function childStatus(result) {
|
||||
if (result.error) process.stderr.write(`${result.error.message}\n`);
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
function publish(options) {
|
||||
const testRun = options.testRun ?? "latest";
|
||||
const judge = spawnSync(process.execPath, [
|
||||
join(evalsDir, "packages/test-evidence/bin/test-evidence-judge.mjs"),
|
||||
"--test-run",
|
||||
testRun,
|
||||
], { cwd: repoRoot, env: process.env, stdio: "inherit" });
|
||||
const judgeStatus = childStatus(judge);
|
||||
|
||||
if (judgeStatus === 2 && !options.dryRun) {
|
||||
process.stderr.write("Pending claims need OPENAI_API_KEY or ANTHROPIC_API_KEY for judging; rerun after providing one, or use --dry-run.\n");
|
||||
return 2;
|
||||
}
|
||||
if (![0, 1, 2].includes(judgeStatus)) return judgeStatus;
|
||||
|
||||
const publishArgs = [join(evalsDir, "packages/test-artifacts/bin/publish-pr.mjs")];
|
||||
if (options.pr) publishArgs.push("--pr", options.pr);
|
||||
if (options.testRun) publishArgs.push("--test-run", options.testRun);
|
||||
if (options.dryRun) publishArgs.push("--dry-run");
|
||||
if (options.force) publishArgs.push("--force");
|
||||
const published = spawnSync(process.execPath, publishArgs, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
const publishStatus = childStatus(published);
|
||||
return judgeStatus === 1 && publishStatus === 0 ? 1 : publishStatus;
|
||||
}
|
||||
|
||||
function run(options) {
|
||||
const runStartedAt = Date.now();
|
||||
const resolved = resolveTestNames(options.testNames);
|
||||
const childEnv = resolveRunEnvironment(options);
|
||||
childEnv.OPENWORK_EVAL_E2E_TESTS = "1";
|
||||
const consented = new Set(["OPENWORK_EVAL_E2E_TESTS"]);
|
||||
|
||||
for (const file of resolved) {
|
||||
for (const variable of consentVarsFromSource(readFileSync(file, "utf8"))) {
|
||||
if (Object.hasOwn(process.env, variable)) continue;
|
||||
childEnv[variable] = "1";
|
||||
consented.add(variable);
|
||||
}
|
||||
}
|
||||
if (options.withLlmVision) delete childEnv.OPENWORK_EVAL_VISION;
|
||||
else childEnv.OPENWORK_EVAL_VISION = "defer";
|
||||
const outputDir = join(evalsDir, "results/.testkit");
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const outputFile = join(outputDir, `cli-run-${Date.now()}.json`);
|
||||
const vitestArgs = [
|
||||
"exec", "vitest", "run",
|
||||
"--config", "vitest.config.ts",
|
||||
"--project", "e2e",
|
||||
"--reporter=default",
|
||||
"--reporter=json",
|
||||
`--outputFile=${outputFile}`,
|
||||
...resolved.map((file) => relative(evalsDir, file).split(sep).join("/")),
|
||||
];
|
||||
const child = spawnSync("pnpm", vitestArgs, { cwd: evalsDir, env: childEnv, stdio: "inherit" });
|
||||
const status = childStatus(child);
|
||||
let report;
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(outputFile, "utf8"));
|
||||
if (!Number.isFinite(parsed?.numTotalTests) || !Array.isArray(parsed?.testResults)) {
|
||||
throw new Error("Invalid Vitest JSON report.");
|
||||
}
|
||||
report = parsed;
|
||||
} catch {
|
||||
report = undefined;
|
||||
}
|
||||
const summary = summarize(report);
|
||||
const verdict = verdictFor(summary, { childExit: status });
|
||||
if (verdict === "failed") {
|
||||
const snapshots = worldSnapshotsSince(runStartedAt);
|
||||
if (snapshots.length > 0) {
|
||||
const paths = snapshots.map((path) => relative(repoRoot, path).split(sep).join("/"));
|
||||
process.stderr.write(`world receipt metadata from this run: ${paths.join(", ")}\n`);
|
||||
}
|
||||
}
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
command: "evals:e2e",
|
||||
lane: "e2e",
|
||||
daytona: childEnv.OPENWORK_EVAL_DAYTONA?.trim() === "1",
|
||||
placement: options.local
|
||||
? "local"
|
||||
: options.daytona
|
||||
? "daytona"
|
||||
: options.den !== undefined
|
||||
? "attached"
|
||||
: "automatic",
|
||||
vision: options.withLlmVision ? "inline" : "defer",
|
||||
files: options.testNames.length > 0 ? options.testNames : ["all"],
|
||||
...summary,
|
||||
consented: [...consented].sort(),
|
||||
verdict,
|
||||
})}\n`);
|
||||
return exitCodeFor(verdict, { named: resolved.length > 0 });
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
let options;
|
||||
try {
|
||||
options = parseArgs(argv);
|
||||
if (options.help) {
|
||||
process.stdout.write(usage);
|
||||
return 0;
|
||||
}
|
||||
return options.publish ? publish(options) : run(options);
|
||||
} catch (error) {
|
||||
process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
process.exitCode = main();
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
consentVarsFromSource,
|
||||
exitCodeFor,
|
||||
parseArgs,
|
||||
resolveRunEnvironment,
|
||||
summarize,
|
||||
verdictFor,
|
||||
worldSnapshotsSince,
|
||||
} from "./evals.mjs";
|
||||
|
||||
test("consentVarsFromSource extracts, deduplicates, and sorts only opt-in variables", () => {
|
||||
const source = `
|
||||
needs({ optIn: ["OPENWORK_EVAL_ZETA", 'OPENWORK_EVAL_ALPHA'] });
|
||||
const requirements = {
|
||||
optIn: [
|
||||
"OPENWORK_EVAL_MULTI",
|
||||
"OPENWORK_EVAL_ALPHA",
|
||||
],
|
||||
};
|
||||
process.env.OPENWORK_EVAL_DIRECT === "1";
|
||||
process.env.OPENWORK_EVAL_TRIMMED?.trim() === "1";
|
||||
process.env.OPENWORK_EVAL_MODEL?.trim() || "";
|
||||
process.env.OPENWORK_EVAL_DEN_API_URL?.trim();
|
||||
process.env.UNRELATED === "1";
|
||||
`;
|
||||
|
||||
assert.deepEqual(consentVarsFromSource(source), [
|
||||
"OPENWORK_EVAL_ALPHA",
|
||||
"OPENWORK_EVAL_DIRECT",
|
||||
"OPENWORK_EVAL_MULTI",
|
||||
"OPENWORK_EVAL_TRIMMED",
|
||||
"OPENWORK_EVAL_ZETA",
|
||||
]);
|
||||
});
|
||||
|
||||
test("parseArgs maps run and publish flags", () => {
|
||||
assert.deepEqual(parseArgs(["app-smoke", "--with-llm-vision", "--daytona", "--den", "https://den.example"]), {
|
||||
testNames: ["app-smoke"],
|
||||
withLlmVision: true,
|
||||
local: false,
|
||||
daytona: true,
|
||||
publish: false,
|
||||
dryRun: false,
|
||||
force: false,
|
||||
help: false,
|
||||
den: "https://den.example",
|
||||
});
|
||||
assert.deepEqual(parseArgs(["--publish", "--pr", "42", "--test-run", "latest", "--dry-run", "--force"]), {
|
||||
testNames: [],
|
||||
withLlmVision: false,
|
||||
local: false,
|
||||
daytona: false,
|
||||
publish: true,
|
||||
dryRun: true,
|
||||
force: true,
|
||||
help: false,
|
||||
pr: "42",
|
||||
testRun: "latest",
|
||||
});
|
||||
});
|
||||
|
||||
test("parseArgs validates values, exclusivity, and unknown flags", () => {
|
||||
assert.throws(() => parseArgs(["--den"]), /--den requires a value/);
|
||||
assert.throws(() => parseArgs(["--publish", "--dry-run", "app-smoke"]), /mutually exclusive with test names/);
|
||||
assert.throws(() => parseArgs(["--publish", "--pr", "1", "--den", "x"]), /mutually exclusive with --den/);
|
||||
assert.throws(() => parseArgs(["app-smoke", "--local", "--daytona"]), /--local is mutually exclusive with --daytona/);
|
||||
assert.throws(() => parseArgs(["app-smoke", "--local", "--den", "https:\/\/den.example"]), /--local is mutually exclusive with --den/);
|
||||
assert.throws(() => parseArgs(["--unknown"]), /Unknown flag: --unknown/);
|
||||
});
|
||||
|
||||
test("explicit local placement removes inherited remote provisioning inputs", () => {
|
||||
const options = parseArgs(["app-smoke", "--local"]);
|
||||
const childEnv = resolveRunEnvironment(options, {
|
||||
PATH: "/bin",
|
||||
OPENWORK_EVAL_DAYTONA: "1",
|
||||
OPENWORK_EVAL_DAYTONA_SANDBOX: "desktop-sandbox",
|
||||
OPENWORK_EVAL_DAYTONA_SANDBOX_ID: "legacy-sandbox",
|
||||
OPENWORK_EVAL_DAYTONA_DEN_SANDBOX: "den-sandbox",
|
||||
OPENWORK_EVAL_DAYTONA_DESKTOP_SANDBOX: "prepared-desktop",
|
||||
OPENWORK_EVAL_DEN_API_URL: "https://den-api.example.test",
|
||||
OPENWORK_EVAL_DEN_WEB_URL: "https://den.example.test",
|
||||
});
|
||||
|
||||
assert.deepEqual(childEnv, { PATH: "/bin" });
|
||||
});
|
||||
|
||||
test("explicit Daytona and attached Den placement retain their existing behavior", () => {
|
||||
const daytona = resolveRunEnvironment(parseArgs(["app-smoke", "--daytona"]), {
|
||||
OPENWORK_EVAL_DEN_API_URL: "https://attached.example.test",
|
||||
});
|
||||
assert.equal(daytona.OPENWORK_EVAL_DAYTONA, "1");
|
||||
assert.equal(daytona.OPENWORK_EVAL_DEN_API_URL, "https://attached.example.test");
|
||||
|
||||
const attached = resolveRunEnvironment(parseArgs(["app-smoke", "--den", "https://den.example.test"]), {});
|
||||
assert.equal(attached.OPENWORK_EVAL_DEN_API_URL, "https://den.example.test");
|
||||
});
|
||||
|
||||
test("automatic placement preserves the caller environment", () => {
|
||||
const ambient = {
|
||||
OPENWORK_EVAL_DAYTONA: "1",
|
||||
OPENWORK_EVAL_DEN_API_URL: "https://den.example.test",
|
||||
};
|
||||
assert.deepEqual(resolveRunEnvironment(parseArgs(["app-smoke"]), ambient), ambient);
|
||||
});
|
||||
|
||||
test("verdict and exit mapping covers failed, incomplete, and passed runs", () => {
|
||||
const failed = verdictFor({ failed: 1, skipped: 0 });
|
||||
assert.equal(failed, "failed");
|
||||
assert.equal(exitCodeFor(failed, { named: true }), 1);
|
||||
|
||||
const incomplete = verdictFor({ failed: 0, skipped: 1 });
|
||||
assert.equal(incomplete, "incomplete");
|
||||
assert.equal(exitCodeFor(incomplete, { named: true }), 2);
|
||||
assert.equal(exitCodeFor(incomplete, { named: false }), 0);
|
||||
|
||||
const passed = verdictFor({ failed: 0, skipped: 0 });
|
||||
assert.equal(passed, "passed");
|
||||
assert.equal(exitCodeFor(passed, { named: true }), 0);
|
||||
});
|
||||
|
||||
test("summarize reads counts and skipped test details", () => {
|
||||
assert.deepEqual(summarize({
|
||||
numPassedTests: 1,
|
||||
numFailedTests: 0,
|
||||
numPendingTests: 1,
|
||||
testResults: [{
|
||||
name: "/repo/evals/specs/app-smoke.e2e.test.ts",
|
||||
assertionResults: [
|
||||
{ status: "passed", title: "runs" },
|
||||
{ status: "pending", title: "needs provider" },
|
||||
],
|
||||
}],
|
||||
}), {
|
||||
passed: 1,
|
||||
failed: 0,
|
||||
skipped: 1,
|
||||
skips: [{ file: "app-smoke.e2e.test.ts", title: "needs provider" }],
|
||||
});
|
||||
});
|
||||
|
||||
test("worldSnapshotsSince returns only snapshots written during the run, newest first", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "openwork-world-snapshots-"));
|
||||
try {
|
||||
await writeFile(join(directory, "old.json"), "{}\n");
|
||||
await utimes(join(directory, "old.json"), new Date(0), new Date(0));
|
||||
const startTime = Date.now();
|
||||
await writeFile(join(directory, "recent.json"), "{}\n");
|
||||
await writeFile(join(directory, "newer.json"), "{}\n");
|
||||
await utimes(join(directory, "recent.json"), new Date(startTime + 1_000), new Date(startTime + 1_000));
|
||||
await utimes(join(directory, "newer.json"), new Date(startTime + 2_000), new Date(startTime + 2_000));
|
||||
await writeFile(join(directory, "ignored.txt"), "not a snapshot\n");
|
||||
|
||||
assert.deepEqual(worldSnapshotsSince(startTime, directory), [
|
||||
join(directory, "newer.json"),
|
||||
join(directory, "recent.json"),
|
||||
]);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env node
|
||||
/** World CLI bootstrap. */
|
||||
|
||||
const nodeMajor = Number.parseInt(process.versions.node.split(".")[0] ?? "0", 10);
|
||||
if (nodeMajor < 24 || !process.features?.typescript) {
|
||||
console.error("Node 24+ with native TypeScript required — run `nvm use`");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { main } = await import("../packages/env/src/cli.ts");
|
||||
|
||||
try {
|
||||
process.exitCode = await main();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in New Issue
Block a user