openwork/evals/bin/evals.test.mjs

import test from "node:test";
import assert from "node:assert/strict";
import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

import {
  consentVarsFromSource,
  exitCodeFor,
  parseArgs,
  resolveRunEnvironment,
  summarize,
  verdictFor,
  worldSnapshotsSince,
} from "./evals.mjs";

test("consentVarsFromSource extracts, deduplicates, and sorts only opt-in variables", () => {
  const source = `
    needs({ optIn: ["OPENWORK_EVAL_ZETA", 'OPENWORK_EVAL_ALPHA'] });
    const requirements = {
      optIn: [
        "OPENWORK_EVAL_MULTI",
        "OPENWORK_EVAL_ALPHA",
      ],
    };
    process.env.OPENWORK_EVAL_DIRECT === "1";
    process.env.OPENWORK_EVAL_TRIMMED?.trim() === "1";
    process.env.OPENWORK_EVAL_MODEL?.trim() || "";
    process.env.OPENWORK_EVAL_DEN_API_URL?.trim();
    process.env.UNRELATED === "1";
  `;

  assert.deepEqual(consentVarsFromSource(source), [
    "OPENWORK_EVAL_ALPHA",
    "OPENWORK_EVAL_DIRECT",
    "OPENWORK_EVAL_MULTI",
    "OPENWORK_EVAL_TRIMMED",
    "OPENWORK_EVAL_ZETA",
  ]);
});

test("parseArgs maps run and publish flags", () => {
  assert.deepEqual(parseArgs(["app-smoke", "--with-llm-vision", "--daytona", "--den", "https://den.example"]), {
    testNames: ["app-smoke"],
    withLlmVision: true,
    local: false,
    daytona: true,
    publish: false,
    dryRun: false,
    force: false,
    help: false,
    den: "https://den.example",
  });
  assert.deepEqual(parseArgs(["--publish", "--pr", "42", "--test-run", "latest", "--dry-run", "--force"]), {
    testNames: [],
    withLlmVision: false,
    local: false,
    daytona: false,
    publish: true,
    dryRun: true,
    force: true,
    help: false,
    pr: "42",
    testRun: "latest",
  });
});

test("parseArgs validates values, exclusivity, and unknown flags", () => {
  assert.throws(() => parseArgs(["--den"]), /--den requires a value/);
  assert.throws(() => parseArgs(["--publish", "--dry-run", "app-smoke"]), /mutually exclusive with test names/);
  assert.throws(() => parseArgs(["--publish", "--pr", "1", "--den", "x"]), /mutually exclusive with --den/);
  assert.throws(() => parseArgs(["app-smoke", "--local", "--daytona"]), /--local is mutually exclusive with --daytona/);
  assert.throws(() => parseArgs(["app-smoke", "--local", "--den", "https:\/\/den.example"]), /--local is mutually exclusive with --den/);
  assert.throws(() => parseArgs(["--unknown"]), /Unknown flag: --unknown/);
});

test("explicit local placement removes inherited remote provisioning inputs", () => {
  const options = parseArgs(["app-smoke", "--local"]);
  const childEnv = resolveRunEnvironment(options, {
    PATH: "/bin",
    OPENWORK_EVAL_DAYTONA: "1",
    OPENWORK_EVAL_DAYTONA_SANDBOX: "desktop-sandbox",
    OPENWORK_EVAL_DAYTONA_SANDBOX_ID: "legacy-sandbox",
    OPENWORK_EVAL_DAYTONA_DEN_SANDBOX: "den-sandbox",
    OPENWORK_EVAL_DAYTONA_DESKTOP_SANDBOX: "prepared-desktop",
    OPENWORK_EVAL_DEN_API_URL: "https://den-api.example.test",
    OPENWORK_EVAL_DEN_WEB_URL: "https://den.example.test",
  });

  assert.deepEqual(childEnv, { PATH: "/bin" });
});

test("explicit Daytona and attached Den placement retain their existing behavior", () => {
  const daytona = resolveRunEnvironment(parseArgs(["app-smoke", "--daytona"]), {
    OPENWORK_EVAL_DEN_API_URL: "https://attached.example.test",
  });
  assert.equal(daytona.OPENWORK_EVAL_DAYTONA, "1");
  assert.equal(daytona.OPENWORK_EVAL_DEN_API_URL, "https://attached.example.test");

  const attached = resolveRunEnvironment(parseArgs(["app-smoke", "--den", "https://den.example.test"]), {});
  assert.equal(attached.OPENWORK_EVAL_DEN_API_URL, "https://den.example.test");
});

test("automatic placement preserves the caller environment", () => {
  const ambient = {
    OPENWORK_EVAL_DAYTONA: "1",
    OPENWORK_EVAL_DEN_API_URL: "https://den.example.test",
  };
  assert.deepEqual(resolveRunEnvironment(parseArgs(["app-smoke"]), ambient), ambient);
});

test("verdict and exit mapping covers failed, incomplete, and passed runs", () => {
  const failed = verdictFor({ failed: 1, skipped: 0 });
  assert.equal(failed, "failed");
  assert.equal(exitCodeFor(failed, { named: true }), 1);

  const incomplete = verdictFor({ failed: 0, skipped: 1 });
  assert.equal(incomplete, "incomplete");
  assert.equal(exitCodeFor(incomplete, { named: true }), 2);
  assert.equal(exitCodeFor(incomplete, { named: false }), 0);

  const passed = verdictFor({ failed: 0, skipped: 0 });
  assert.equal(passed, "passed");
  assert.equal(exitCodeFor(passed, { named: true }), 0);
});

test("summarize reads counts and skipped test details", () => {
  assert.deepEqual(summarize({
    numPassedTests: 1,
    numFailedTests: 0,
    numPendingTests: 1,
    testResults: [{
      name: "/repo/evals/specs/app-smoke.e2e.test.ts",
      assertionResults: [
        { status: "passed", title: "runs" },
        { status: "pending", title: "needs provider" },
      ],
    }],
  }), {
    passed: 1,
    failed: 0,
    skipped: 1,
    skips: [{ file: "app-smoke.e2e.test.ts", title: "needs provider" }],
  });
});

test("worldSnapshotsSince returns only snapshots written during the run, newest first", async () => {
  const directory = await mkdtemp(join(tmpdir(), "openwork-world-snapshots-"));
  try {
    await writeFile(join(directory, "old.json"), "{}\n");
    await utimes(join(directory, "old.json"), new Date(0), new Date(0));
    const startTime = Date.now();
    await writeFile(join(directory, "recent.json"), "{}\n");
    await writeFile(join(directory, "newer.json"), "{}\n");
    await utimes(join(directory, "recent.json"), new Date(startTime + 1_000), new Date(startTime + 1_000));
    await utimes(join(directory, "newer.json"), new Date(startTime + 2_000), new Date(startTime + 2_000));
    await writeFile(join(directory, "ignored.txt"), "not a snapshot\n");

    assert.deepEqual(worldSnapshotsSince(startTime, directory), [
      join(directory, "newer.json"),
      join(directory, "recent.json"),
    ]);
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});