2026-09-02 01:05:41 +08:00
commit 04bac1f6e2
2200 changed files with 489415 additions and 0 deletions
@@ -0,0 +1,350 @@
import { execFile, spawn } from "node:child_process";
import { closeSync, openSync } from "node:fs";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { createServer } from "node:net";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
const STATE_ROOT = join(REPO_ROOT, "evals", "results", ".dev-den");
const MYSQL_CONTAINER = "openwork-web-local-mysql";
const DB_ENCRYPTION_KEY = "local-dev-db-encryption-key-please-change-1234567890";
const BETTER_AUTH_SECRET = "local-dev-secret-not-for-production-use!!";
const USAGE = `Usage:
  pnpm --dir evals dev:den -- up [--port <port>] [--database <name>] [--seed]
  pnpm --dir evals dev:den -- down --port <port> [--drop-database]
The up command creates an isolated database, initializes it with
db:bootstrap, starts den-api in multi-org dev mode, waits for /health, and prints the
OPENWORK_EVAL_DEN_* exports. The generated trusted origins always include the
printed web URL; omitting it causes Better Auth 403 INVALID_ORIGIN responses.`;
interface DevDenState {
  apiUrl: string;
  database: string;
  databaseUrl: string;
  logPath: string;
  pid: number;
  port: number;
  webUrl: string;
}
function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}
function statePath(port: number): string {
  return join(STATE_ROOT, `${port}.json`);
}
function parseState(value: unknown): DevDenState | null {
  if (!isRecord(value)) return null;
  if (
    typeof value.apiUrl !== "string"
    || typeof value.database !== "string"
    || typeof value.databaseUrl !== "string"
    || typeof value.logPath !== "string"
    || typeof value.pid !== "number"
    || typeof value.port !== "number"
    || typeof value.webUrl !== "string"
  ) return null;
  return {
    apiUrl: value.apiUrl,
    database: value.database,
    databaseUrl: value.databaseUrl,
    logPath: value.logPath,
    pid: value.pid,
    port: value.port,
    webUrl: value.webUrl,
  };
}
function processAlive(pid: number): boolean {
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
}
async function run(command: string, args: string[], env: NodeJS.ProcessEnv = process.env): Promise<void> {
  await new Promise<void>((resolveRun, reject) => {
    const child = execFile(command, args, { cwd: REPO_ROOT, env }, (error) => {
      if (error) reject(error);
      else resolveRun();
    });
    child.stdout?.pipe(process.stdout);
    child.stderr?.pipe(process.stderr);
  });
}
async function pickPort(): Promise<number> {
  return new Promise<number>((resolvePort, reject) => {
    const server = createServer();
    server.once("error", reject);
    server.listen(0, "127.0.0.1", () => {
      const address = server.address();
      if (!address || typeof address === "string") {
        server.close();
        reject(new Error("Could not allocate a local Den port."));
        return;
      }
      server.close((error) => {
        if (error) reject(error);
        else resolvePort(address.port);
      });
    });
  });
}
function parsePort(value: string | undefined): number | null {
  if (value === undefined) return null;
  const port = Number(value);
  if (!Number.isInteger(port) || port < 1 || port > 65_535) {
    throw new Error(`Invalid port: ${value}`);
  }
  return port;
}
function validateDatabase(value: string): string {
  if (!/^[A-Za-z0-9_]+$/.test(value)) {
    throw new Error(`Invalid database name: ${value}. Use only letters, numbers, and underscores.`);
  }
  return value;
}
async function ensurePortFree(port: number): Promise<void> {
  await new Promise<void>((resolveFree, reject) => {
    const server = createServer();
    server.once("error", () => reject(new Error(`Port ${port} is already in use.`)));
    server.listen(port, "127.0.0.1", () => server.close((error) => error ? reject(error) : resolveFree()));
  });
}
function denEnvironment(state: DevDenState): NodeJS.ProcessEnv {
  const trustedOrigins = `${state.apiUrl},${state.webUrl}`;
  return {
    ...process.env,
    OPENWORK_DEV_MODE: "1",
    PORT: String(state.port),
    DEN_API_PORT: String(state.port),
    DEN_API_PUBLIC_URL: state.apiUrl,
    DATABASE_URL: state.databaseUrl,
    DEN_DB_ENCRYPTION_KEY: DB_ENCRYPTION_KEY,
    BETTER_AUTH_SECRET,
    BETTER_AUTH_URL: state.webUrl,
    DEN_BETTER_AUTH_TRUSTED_ORIGINS: trustedOrigins,
    CORS_ORIGINS: trustedOrigins,
    DEN_ORG_MODE: "multi_org",
    DEN_SINGLE_ORG_ALLOW_PUBLIC_SIGNUP: "true",
    PROVISIONER_MODE: "stub",
    STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY ?? "sk_test_openwork_eval",
    STRIPE_INFERENCE_PRICE_ID: process.env.STRIPE_INFERENCE_PRICE_ID ?? "price_openwork_models_eval",
    STRIPE_WEBHOOK_SECRET: process.env.STRIPE_WEBHOOK_SECRET ?? "whsec_openwork_eval",
    INFERENCE_PROXY_BASE_URL: process.env.INFERENCE_PROXY_BASE_URL ?? "http://127.0.0.1:8791",
  };
}
async function waitForHealth(state: DevDenState): Promise<void> {
  const startedAt = Date.now();
  while (Date.now() - startedAt < 180_000) {
    if (!processAlive(state.pid)) {
      throw new Error(`den-api exited before becoming healthy. See ${relative(REPO_ROOT, state.logPath)}.`);
    }
    const log = await readFile(state.logPath, "utf8").catch(() => "");
    if (log.includes('"message":"server listening"')) {
      try {
        const response = await fetch(`${state.apiUrl}/health`, { signal: AbortSignal.timeout(2_000) });
        if (response.ok) return;
      } catch {
        // den-api logged that it is listening, but health is not ready yet.
      }
    }
    await new Promise((resolveWait) => setTimeout(resolveWait, 1_000));
  }
  throw new Error(`den-api did not become healthy within 180s. See ${relative(REPO_ROOT, state.logPath)}.`);
}
function printEnvironment(state: DevDenState): void {
  const trustedOrigins = `${state.apiUrl},${state.webUrl}`;
  console.log(`Health passed: ${state.apiUrl}/health`);
  console.log(`export OPENWORK_EVAL_DEN_API_URL="${state.apiUrl}"`);
  console.log(`export OPENWORK_EVAL_DEN_WEB_URL="${state.webUrl}"`);
  console.log(`# DEN_BETTER_AUTH_TRUSTED_ORIGINS="${trustedOrigins}"`);
  console.log("# The trusted origins include OPENWORK_EVAL_DEN_WEB_URL; otherwise sign-in fails with 403 INVALID_ORIGIN.");
  console.log(`Log: ${relative(REPO_ROOT, state.logPath)}`);
  console.log(`Tear down: pnpm --dir evals dev:den -- down --port ${state.port} --drop-database`);
}
async function createDatabase(database: string): Promise<void> {
  await run("pnpm", ["dev:den:mysql"]);
  await run("docker", [
    "exec",
    MYSQL_CONTAINER,
    "mysql",
    "-uroot",
    "-ppassword",
    "-e",
    `CREATE DATABASE IF NOT EXISTS \`${database}\``,
  ]);
}
async function applySchema(state: DevDenState): Promise<void> {
  // Uses db:bootstrap, the same entrypoint a real Den install runs, rather than
  // re-deriving DDL with db:push.
  //
  // push regenerates a prefix-length index as ``token`(191)`` — doubled
  // backticks — which MySQL rejects with a 1064 parse error, so an isolated Den
  // could never finish standing up. bootstrap instead initializes an empty
  // database from the build-time schema snapshot and records the committed
  // migrations as its baseline, so the eval Den is built the way a deployment
  // is. Plain db:migrate is not an option here: the ledger is baselined, so on
  // an empty database it has nothing to create the base tables from.
  console.log(`Bootstrapping ${state.database} from the current schema snapshot...`);
  await run("pnpm", ["--filter", "@openwork-ee/den-db", "db:bootstrap"], denEnvironment(state));
}
async function seedDemoOrg(state: DevDenState): Promise<void> {
  console.log("Seeding the demo organization...");
  await run("pnpm", ["--filter", "@openwork-ee/den-api", "seed:demo-org"], {
    ...denEnvironment(state),
    DEN_DEMO_SEED_FETCH_GITHUB: "0",
  });
}
async function up(portValue: string | undefined, databaseValue: string | undefined, seed: boolean): Promise<void> {
  const port = parsePort(portValue) ?? await pickPort();
  const database = validateDatabase(databaseValue ?? `openwork_den_eval_${process.pid}_${Date.now().toString(36)}`);
  await ensurePortFree(port);
  await mkdir(STATE_ROOT, { recursive: true });
  const apiUrl = `http://127.0.0.1:${port}`;
  const state: DevDenState = {
    apiUrl,
    database,
    databaseUrl: `mysql://root:password@127.0.0.1:3306/${database}`,
    logPath: join(STATE_ROOT, `${port}.log`),
    pid: 0,
    port,
    webUrl: `http://localhost:${port}`,
  };
  await createDatabase(database);
  await applySchema(state);
  await ensurePortFree(port);
  const logFd = openSync(state.logPath, "w");
  const child = spawn("pnpm", ["--filter", "@openwork-ee/den-api", "dev:local"], {
    cwd: REPO_ROOT,
    detached: true,
    env: denEnvironment(state),
    stdio: ["ignore", logFd, logFd],
  });
  closeSync(logFd);
  if (!child.pid) throw new Error("Could not start den-api.");
  child.unref();
  state.pid = child.pid;
  await writeFile(statePath(port), `${JSON.stringify(state, null, 2)}\n`, "utf8");
  try {
    await waitForHealth(state);
    if (seed) await seedDemoOrg(state);
    printEnvironment(state);
  } catch (error) {
    try {
      process.kill(-state.pid, "SIGTERM");
    } catch {
      // Process already stopped.
    }
    throw error;
  }
}
async function readState(port: number): Promise<DevDenState> {
  let parsed: unknown;
  try {
    parsed = JSON.parse(await readFile(statePath(port), "utf8"));
  } catch {
    throw new Error(`No dev Den state found for port ${port}.`);
  }
  const state = parseState(parsed);
  if (!state) throw new Error(`Invalid dev Den state for port ${port}.`);
  return state;
}
async function stopProcess(state: DevDenState): Promise<void> {
  if (!processAlive(state.pid)) return;
  try {
    process.kill(-state.pid, "SIGTERM");
  } catch {
    process.kill(state.pid, "SIGTERM");
  }
  const startedAt = Date.now();
  while (processAlive(state.pid) && Date.now() - startedAt < 10_000) {
    await new Promise((resolveWait) => setTimeout(resolveWait, 250));
  }
  if (processAlive(state.pid)) {
    try {
      process.kill(-state.pid, "SIGKILL");
    } catch {
      process.kill(state.pid, "SIGKILL");
    }
  }
}
async function down(portValue: string | undefined, dropDatabase: boolean): Promise<void> {
  const port = parsePort(portValue);
  if (port === null) throw new Error(`--port is required for down.\n\n${USAGE}`);
  const state = await readState(port);
  await stopProcess(state);
  console.log(`Stopped den-api on port ${port}.`);
  if (dropDatabase) {
    await run("docker", [
      "exec",
      MYSQL_CONTAINER,
      "mysql",
      "-uroot",
      "-ppassword",
      "-e",
      `DROP DATABASE IF EXISTS \`${state.database}\``,
    ]);
    console.log(`Dropped database ${state.database}.`);
  }
  await rm(statePath(port), { force: true });
}
async function main(argv: string[]): Promise<void> {
  const args = argv[0] === "--" ? argv.slice(1) : argv;
  const { values, positionals } = parseArgs({
    args,
    options: {
      port: { type: "string" },
      database: { type: "string" },
      seed: { type: "boolean" },
      "drop-database": { type: "boolean" },
      help: { type: "boolean", short: "h" },
    },
    strict: true,
    allowPositionals: true,
  });
  if (values.help) {
    console.log(USAGE);
    return;
  }
  const command = positionals[0];
  if (positionals.length !== 1 || (command !== "up" && command !== "down")) {
    throw new Error(USAGE);
  }
  if (command === "up") {
    await up(values.port, values.database, values.seed === true);
    return;
  }
  await down(values.port, values["drop-database"] === true);
}
main(process.argv.slice(2)).catch((error: unknown) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exitCode = 1;
});
@@ -0,0 +1,45 @@
import { probeTls } from "@openwork/behaviors";
import { diagnoseTls, type TlsVersionFacts } from "@openwork/matchers";
function usage(): never {
  throw new Error("Usage: node evals/scripts/diagnose.mts <origin> [--insecure-ca <pemPath>]");
}
const args = process.argv.slice(2);
const origin = args[0];
if (!origin) usage();
let ca: string | undefined;
for (let index = 1; index < args.length; index += 1) {
  if (args[index] !== "--insecure-ca") usage();
  const pemPath = args[index + 1];
  if (!pemPath) usage();
  ca = process.getBuiltinModule("node:fs").readFileSync(pemPath, "utf8");
  index += 1;
}
const url = new URL(origin);
const tls = await probeTls({
  host: url.hostname,
  port: Number(url.port || 443),
  servername: url.hostname,
  ca,
});
function printVersion(label: string, facts: TlsVersionFacts): void {
  const handshake = facts.handshakeOk ? "ok" : (facts.stalled ? "STALLED" : "FAILED");
  const chain = facts.handshakeOk
    ? (facts.chainVerified ? "verified" : `UNTRUSTED (${facts.errorCode ?? "unauthorized"})`)
    : "not checked";
  console.log(`${label}: handshake ${handshake} / chain ${chain}`);
}
printVersion("TLS 1.2", tls.tls12);
printVersion("TLS 1.3", tls.tls13);
const findings = [diagnoseTls(tls)];
for (const finding of findings) {
  console.log(`${finding.ok ? "ok" : "FAIL"} tls ${finding.code}`);
  if (!finding.ok) {
    console.log(`  ${finding.summary}`);
    console.log(`  Action: ${finding.action}`);
  }
}
console.log(JSON.stringify({ origin: url.origin, tls, findings }, null, 2));
if (findings.some((finding) => !finding.ok)) process.exitCode = 1;
@@ -0,0 +1,143 @@
import { execFile } from "node:child_process";
import { createHash, timingSafeEqual } from "node:crypto";
import { readFile, rm, writeFile } from "node:fs/promises";
import http from "node:http";
import { setTimeout } from "node:timers/promises";
import {
  linuxTrustStorePlan,
  readTrustedEnterpriseTlsManifest,
  stageTrustedEnterpriseTlsRoot,
  startEnterpriseTlsReverseEdge,
  type EnterpriseTlsEdgeManifest,
  type LinuxTrustCommand,
} from "../packages/labs/src/egress.ts";
type EdgeManifest = EnterpriseTlsEdgeManifest;
function option(name: string, fallback?: string): string {
  const index = process.argv.indexOf(name);
  const value = index >= 0 ? process.argv[index + 1] : fallback;
  if (!value) throw new Error(`Missing ${name}.`);
  return value;
}
function portOption(name: string, fallback: number): number {
  const value = Number.parseInt(option(name, String(fallback)), 10);
  if (!Number.isInteger(value) || value < 1 || value > 65_535) throw new Error(`Invalid ${name}.`);
  return value;
}
async function readManifest(manifestPath: string): Promise<EdgeManifest> {
  return (await readTrustedEnterpriseTlsManifest(manifestPath)).manifest;
}
function run(command: LinuxTrustCommand): Promise<void> {
  return new Promise((resolve, reject) => {
    execFile(command.file, command.args, { encoding: "utf8" }, (error, stdout, stderr) => {
      if (!error) return resolve();
      reject(new Error(`${command.file} failed: ${String(stderr || stdout).trim() || error.message}`));
    });
  });
}
async function changeTrust(action: "install" | "remove", manifestPath: string): Promise<void> {
  const staged = action === "install" ? await stageTrustedEnterpriseTlsRoot(manifestPath) : null;
  try {
    const manifest = staged?.manifest ?? await readManifest(manifestPath);
    const plan = linuxTrustStorePlan(staged?.stagedRootPemPath ?? manifest.rootPemPath);
    const prerequisite = await plan.checkPrerequisites();
    if (!prerequisite.ok) throw new Error(prerequisite.failure);
    const commands = action === "install"
      ? plan.install(prerequisite.updateCaCertificatesPath)
      : plan.remove(prerequisite.updateCaCertificatesPath);
    for (const command of commands) await run(command);
  } finally {
    await staged?.cleanup().catch(() => undefined);
  }
}
async function stop(manifestPath: string): Promise<void> {
  const manifest = await readManifest(manifestPath);
  process.kill(manifest.pid, "SIGTERM");
  for (let attempt = 0; attempt < 100; attempt += 1) {
    try {
      await readFile(manifestPath);
    } catch {
      return;
    }
    await setTimeout(100);
  }
  throw new Error(`Enterprise TLS edge did not stop cleanly: ${manifest.pid}`);
}
function listen(server: http.Server, port: number): Promise<void> {
  return new Promise((resolve, reject) => {
    server.once("error", reject);
    server.listen(port, "127.0.0.1", () => resolve());
  });
}
function close(server: http.Server): Promise<void> {
  return new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
}
async function serve(manifestPath: string): Promise<void> {
  const adminToken = process.env.ENTERPRISE_TLS_ADMIN_TOKEN;
  if (typeof adminToken !== "string" || !/^[a-f0-9]{32,}$/.test(adminToken)) {
    throw new Error("ENTERPRISE_TLS_ADMIN_TOKEN must be at least 32 hex characters.");
  }
  const expectedAuthorizationHash = createHash("sha256").update(`Bearer ${adminToken}`).digest();
  const candidatePort = portOption("--candidate-port", 8443);
  const negativePort = portOption("--negative-port", 9443);
  const adminPort = portOption("--admin-port", 8445);
  if (new Set([candidatePort, negativePort, adminPort]).size !== 3) throw new Error("Candidate, negative, and admin ports must be distinct.");
  const edge = await startEnterpriseTlsReverseEdge({
    upstream: option("--upstream"),
    candidatePort,
    negativePort,
  });
  const adminUrl = `http://127.0.0.1:${adminPort}`;
  const manifest: EdgeManifest = {
    pid: process.pid,
    candidateUrl: edge.candidateUrl,
    negativeUrl: edge.negativeUrl,
    adminUrl,
    rootPemPath: edge.rootPemPath,
  };
  const admin = http.createServer((request, response) => {
    response.setHeader("content-type", "application/json; charset=utf-8");
    const receivedAuthorizationHash = createHash("sha256").update(request.headers.authorization ?? "").digest();
    if (!timingSafeEqual(receivedAuthorizationHash, expectedAuthorizationHash)) {
      response.writeHead(401).end(`${JSON.stringify({ error: "Unauthorized" })}\n`);
      return;
    }
    if (request.url === "/health" || request.url === "/manifest") response.end(`${JSON.stringify({ ok: true, ...manifest })}\n`);
    else if (request.url === "/requests") response.end(`${JSON.stringify(edge.requests)}\n`);
    else response.writeHead(404).end(`${JSON.stringify({ error: "not_found" })}\n`);
  });
  try {
    await listen(admin, adminPort);
    await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
    await waitForSignal();
  } finally {
    await close(admin).catch(() => undefined);
    await edge.stop();
    await rm(manifestPath, { force: true });
  }
}
function waitForSignal(): Promise<void> {
  return new Promise((resolve) => {
    process.once("SIGTERM", resolve);
    process.once("SIGINT", resolve);
  });
}
const action = process.argv[2];
const manifestPath = option("--manifest", "/tmp/openwork-enterprise-tls-edge.json");
if (action === "serve") await serve(manifestPath);
else if (action === "install" || action === "remove") await changeTrust(action, manifestPath);
else if (action === "stop") await stop(manifestPath);
else throw new Error("Usage: enterprise-tls-edge.mts <serve|install|remove|stop> [options]");
@@ -0,0 +1,167 @@
// Flake-rate reporting over repeated vitest runs.
//
// collect: run one lane with the vitest JSON reporter and save the receipt.
//   node scripts/flake-report.mjs collect [--project pr|e2e] [-- <extra vitest args>]
// report: aggregate every saved receipt into per-spec pass rates.
//   node scripts/flake-report.mjs report [--project pr|e2e] [--threshold 0.95]
//
// A spec is "flaky" when the same spec file both passed and failed across the
// collected runs. Specs below the pass-rate threshold belong on the quarantine
// list until diagnosed (see diagnose-a-red-run).
import { execFileSync } from "node:child_process"
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"
import { join, relative, resolve } from "node:path"
import { fileURLToPath } from "node:url"
const evalsRoot = fileURLToPath(new URL("..", import.meta.url))
export function runsDirectory(project) {
  return resolve(evalsRoot, "results", "flake", project)
}
function parseArguments(argv) {
  const separator = argv.indexOf("--")
  const own = separator === -1 ? argv : argv.slice(0, separator)
  const passthrough = separator === -1 ? [] : argv.slice(separator + 1)
  const mode = own[0]
  if (mode !== "collect" && mode !== "report") {
    throw new Error("usage: flake-report.mjs <collect|report> [--project pr|e2e] [--threshold 0.95] [-- vitest args]")
  }
  let project = "pr"
  let threshold = 0.95
  for (let index = 1; index < own.length; index += 1) {
    if (own[index] === "--project") {
      project = own[index + 1] ?? ""
      index += 1
    } else if (own[index] === "--threshold") {
      threshold = Number(own[index + 1])
      index += 1
    } else {
      throw new Error(`unknown argument: ${own[index]}`)
    }
  }
  if (project !== "pr" && project !== "e2e") throw new Error("--project must be pr or e2e")
  if (!Number.isFinite(threshold) || threshold <= 0 || threshold > 1) throw new Error("--threshold must be in (0, 1]")
  return { mode, project, threshold, passthrough }
}
export function normalizeRun(receipt, root) {
  if (typeof receipt !== "object" || receipt === null || !Array.isArray(receipt.testResults)) {
    throw new Error("receipt is not a vitest JSON reporter payload")
  }
  return receipt.testResults.map((file) => {
    const tests = Array.isArray(file.assertionResults) ? file.assertionResults : []
    const durationMs = tests.reduce((total, test) => total + (typeof test.duration === "number" ? test.duration : 0), 0)
    return {
      spec: relative(root, file.name).replaceAll("\\", "/"),
      passed: file.status === "passed",
      skipped: tests.length > 0 && tests.every((test) => test.status === "pending" || test.status === "skipped" || test.status === "todo"),
      durationMs,
    }
  })
}
export function aggregate(runs, threshold) {
  const bySpec = new Map()
  for (const run of runs) {
    for (const entry of run) {
      if (entry.skipped) continue
      const current = bySpec.get(entry.spec) ?? { spec: entry.spec, runs: 0, passes: 0, durationsMs: [] }
      current.runs += 1
      if (entry.passed) current.passes += 1
      current.durationsMs.push(entry.durationMs)
      bySpec.set(entry.spec, current)
    }
  }
  const specs = [...bySpec.values()]
    .map((entry) => {
      const passRate = entry.passes / entry.runs
      const meanDurationMs = entry.durationsMs.reduce((total, value) => total + value, 0) / entry.runs
      return {
        spec: entry.spec,
        runs: entry.runs,
        passes: entry.passes,
        passRate,
        flaky: entry.passes > 0 && entry.passes < entry.runs,
        quarantine: passRate < threshold,
        meanDurationMs: Math.round(meanDurationMs),
        maxDurationMs: Math.round(Math.max(...entry.durationsMs)),
      }
    })
    .sort((left, right) => left.passRate - right.passRate || right.maxDurationMs - left.maxDurationMs)
  return {
    threshold,
    totalRuns: runs.length,
    totalSpecs: specs.length,
    flakySpecs: specs.filter((entry) => entry.flaky).length,
    quarantineSpecs: specs.filter((entry) => entry.quarantine).length,
    specs,
  }
}
export function renderMarkdown(report, project) {
  const lines = [
    `# Flake report — ${project} lane`,
    "",
    `Runs aggregated: ${report.totalRuns} · Specs: ${report.totalSpecs} · Flaky: ${report.flakySpecs} · Below ${Math.round(report.threshold * 100)}% pass rate: ${report.quarantineSpecs}`,
    "",
    "| Spec | Runs | Pass rate | Flaky | Quarantine | Mean ms | Max ms |",
    "| --- | ---: | ---: | :---: | :---: | ---: | ---: |",
  ]
  for (const entry of report.specs) {
    lines.push(
      `| ${entry.spec} | ${entry.runs} | ${(entry.passRate * 100).toFixed(1)}% | ${entry.flaky ? "yes" : ""} | ${entry.quarantine ? "yes" : ""} | ${entry.meanDurationMs} | ${entry.maxDurationMs} |`,
    )
  }
  return `${lines.join("\n")}\n`
}
function collect(project, passthrough) {
  const directory = runsDirectory(project)
  mkdirSync(directory, { recursive: true })
  const outputFile = join(directory, `run-${new Date().toISOString().replaceAll(":", "-")}.json`)
  const vitestBin = resolve(evalsRoot, "node_modules", ".bin", process.platform === "win32" ? "vitest.CMD" : "vitest")
  const args = [
    "run",
    "--config",
    "vitest.config.ts",
    "--project",
    project,
    "--reporter=default",
    "--reporter=json",
    `--outputFile=${outputFile}`,
    ...passthrough,
  ]
  let failed = false
  try {
    execFileSync(vitestBin, args, { cwd: evalsRoot, stdio: "inherit" })
  } catch {
    failed = true
  }
  console.log(`\nflake-report: saved ${relative(evalsRoot, outputFile)} (${failed ? "red" : "green"} run)`)
}
function report(project, threshold) {
  const directory = runsDirectory(project)
  let files = []
  try {
    files = readdirSync(directory).filter((name) => name.startsWith("run-") && name.endsWith(".json"))
  } catch {
    throw new Error(`no collected runs in ${relative(evalsRoot, directory)}; run flake:collect first`)
  }
  if (files.length === 0) throw new Error(`no collected runs in ${relative(evalsRoot, directory)}; run flake:collect first`)
  const runs = files.map((name) => normalizeRun(JSON.parse(readFileSync(join(directory, name), "utf8")), evalsRoot))
  const result = aggregate(runs, threshold)
  const markdown = renderMarkdown(result, project)
  writeFileSync(join(directory, "flake-report.md"), markdown)
  writeFileSync(join(directory, "flake-report.json"), `${JSON.stringify(result, null, 2)}\n`)
  console.log(markdown)
  console.log(`flake-report: wrote ${relative(evalsRoot, join(directory, "flake-report.md"))}`)
}
const invokedDirectly = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)
if (invokedDirectly) {
  const { mode, project, threshold, passthrough } = parseArguments(process.argv.slice(2))
  if (mode === "collect") collect(project, passthrough)
  else report(project, threshold)
}
@@ -0,0 +1,92 @@
import assert from "node:assert/strict"
import { test } from "node:test"
import { aggregate, normalizeRun, renderMarkdown } from "./flake-report.mjs"
const root = "/repo/evals"
function receipt(entries) {
  return {
    testResults: entries.map((entry) => ({
      name: `${root}/specs/${entry.spec}`,
      status: entry.passed ? "passed" : "failed",
      assertionResults: [{ status: entry.passed ? "passed" : "failed", duration: entry.durationMs }],
    })),
  }
}
test("normalizeRun maps vitest JSON receipts to repo-relative spec results", () => {
  const run = normalizeRun(receipt([{ spec: "a.test.ts", passed: true, durationMs: 120 }]), root)
  assert.deepEqual(run, [{ spec: "specs/a.test.ts", passed: true, skipped: false, durationMs: 120 }])
})
test("normalizeRun rejects non-vitest payloads", () => {
  assert.throws(() => normalizeRun({ nope: true }, root), /vitest JSON reporter payload/)
})
test("normalizeRun marks fully skipped files as skipped", () => {
  const run = normalizeRun(
    {
      testResults: [
        {
          name: `${root}/specs/skipped.test.ts`,
          status: "passed",
          assertionResults: [{ status: "pending" }, { status: "todo" }],
        },
      ],
    },
    root,
  )
  assert.equal(run[0].skipped, true)
})
test("aggregate computes pass rate, flaky, and quarantine flags", () => {
  const runs = [
    normalizeRun(
      receipt([
        { spec: "stable.test.ts", passed: true, durationMs: 100 },
        { spec: "flaky.test.ts", passed: true, durationMs: 200 },
        { spec: "broken.test.ts", passed: false, durationMs: 50 },
      ]),
      root,
    ),
    normalizeRun(
      receipt([
        { spec: "stable.test.ts", passed: true, durationMs: 110 },
        { spec: "flaky.test.ts", passed: false, durationMs: 900 },
        { spec: "broken.test.ts", passed: false, durationMs: 60 },
      ]),
      root,
    ),
  ]
  const result = aggregate(runs, 0.95)
  assert.equal(result.totalRuns, 2)
  assert.equal(result.totalSpecs, 3)
  assert.equal(result.flakySpecs, 1)
  assert.equal(result.quarantineSpecs, 2)
  const flaky = result.specs.find((entry) => entry.spec === "specs/flaky.test.ts")
  assert.equal(flaky.passRate, 0.5)
  assert.equal(flaky.flaky, true)
  assert.equal(flaky.quarantine, true)
  assert.equal(flaky.meanDurationMs, 550)
  assert.equal(flaky.maxDurationMs, 900)
  const broken = result.specs.find((entry) => entry.spec === "specs/broken.test.ts")
  assert.equal(broken.flaky, false)
  assert.equal(broken.quarantine, true)
  const stable = result.specs.find((entry) => entry.spec === "specs/stable.test.ts")
  assert.equal(stable.passRate, 1)
  assert.equal(stable.quarantine, false)
  // Worst pass rate sorts first.
  assert.equal(result.specs[0].passRate <= result.specs[result.specs.length - 1].passRate, true)
})
test("renderMarkdown emits one row per spec with the summary line", () => {
  const runs = [normalizeRun(receipt([{ spec: "a.test.ts", passed: true, durationMs: 10 }]), root)]
  const markdown = renderMarkdown(aggregate(runs, 0.95), "pr")
  assert.match(markdown, /# Flake report — pr lane/)
  assert.match(markdown, /Runs aggregated: 1 · Specs: 1 · Flaky: 0/)
  assert.match(markdown, /\| specs\/a\.test\.ts \| 1 \| 100\.0% \|/)
})
@@ -0,0 +1,82 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
const CHROME_BIN = process.env.CHROME_BIN?.trim() || "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
const PROFILES = [
  ["ADMIN", 9333],
  ["INVITEE", 9334],
  ["MOBILE", 9335],
];
async function isListening(port) {
  try {
    const response = await fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(700) });
    return response.ok;
  } catch {
    return false;
  }
}
function alive(pid) {
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
}
async function killPidfile(userDataDir) {
  const pidPath = join(userDataDir, "openwork-eval-chrome.pid");
  let pid = 0;
  try {
    pid = Number((await readFile(pidPath, "utf8")).trim());
  } catch {
    return;
  }
  if (Number.isInteger(pid) && pid > 0 && alive(pid)) {
    try {
      process.kill(pid, "SIGTERM");
    } catch {}
  }
}
async function waitForPort(port) {
  for (let attempt = 0; attempt < 40; attempt += 1) {
    if (await isListening(port)) return;
    await new Promise((resolve) => setTimeout(resolve, 250));
  }
  throw new Error(`Chrome did not open CDP port ${port}.`);
}
async function ensureProfile(port) {
  const userDataDir = `/tmp/openwork-evals-web-${port}`;
  await mkdir(userDataDir, { recursive: true });
  if (await isListening(port)) return;
  await killPidfile(userDataDir);
  const child = spawn(CHROME_BIN, [
    `--remote-debugging-port=${port}`,
    `--user-data-dir=${userDataDir}`,
    "--no-first-run",
    "--no-default-browser-check",
    "--disable-popup-blocking",
    "about:blank",
  ], {
    detached: true,
    stdio: "ignore",
  });
  child.unref();
  await writeFile(join(userDataDir, "openwork-eval-chrome.pid"), `${child.pid}\n`);
  await waitForPort(port);
}
for (const [, port] of PROFILES) {
  await ensureProfile(port);
}
for (const [name, port] of PROFILES) {
  console.log(`export OPENWORK_EVAL_WEB_CDP_${name}=http://127.0.0.1:${port}`);
}
@@ -0,0 +1,13 @@
import { readdir, readFile } from "node:fs/promises";
const specs = new URL("../specs/", import.meta.url);
const files = (await readdir(specs))
  .filter((file) => file.endsWith(".e2e.test.ts"))
  .sort();
for (const file of files) {
  const source = await readFile(new URL(file, specs), "utf8");
  if (/import\s*\{[^}]*\bdesktop\b[^}]*\}\s*from\s*["']@openwork\/hosts["']/s.test(source)) {
    console.log(file);
  }
}
@@ -0,0 +1,127 @@
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { parseArgs } from "node:util";
import {
  deleteSandboxes,
  parseConnectorE2eTestEnv,
  provisionDenSandbox,
  provisionDesktopSandbox,
  renderConnectorE2eTestEnv,
  startMockOnSandbox,
} from "@openwork/hosts";
import type { DesktopSandbox } from "@openwork/hosts";
const USAGE = `Usage:
  node scripts/provision-org-connector-two-members.ts --ref <branch-or-commit> [--reuse-a <id>] [--reuse-b <id>] [--reuse-den <id>] [--out org-connector-two-members.env]
  node scripts/provision-org-connector-two-members.ts --delete <envfile>
  node scripts/provision-org-connector-two-members.ts --help`;
function prefixed(prefix: string, observe?: (line: string) => void): (line: string) => void {
  return (line) => {
    observe?.(line);
    console.error(`[${prefix}] ${line}`);
  };
}
async function deleteProvisioned(envFile: string): Promise<void> {
  const facts = parseConnectorE2eTestEnv(await readFile(envFile, "utf8"));
  await deleteSandboxes(facts.created);
  const created = new Set(facts.created);
  const leftAlone = [facts.sandboxA, facts.sandboxB].filter((sandbox) => !created.has(sandbox));
  console.log(`Deleted: ${facts.created.join(", ") || "(none)"}`);
  console.log(`Left alone: ${leftAlone.join(", ") || "(none recorded)"}`);
}
async function provisionSpecEnv(options: {
  ref: string;
  reuseA?: string;
  reuseB?: string;
  reuseDen?: string;
  out: string;
}): Promise<void> {
  const created: string[] = [];
  function recordCreated(sandbox: string): void {
    if (!created.includes(sandbox)) created.push(sandbox);
  }
  try {
    const denLog = prefixed("den", (line) => {
      const match = /Creating server sandbox:\s*(\S+)/.exec(line);
      if (match?.[1]) recordCreated(match[1]);
    });
    const den = await provisionDenSandbox({ ref: options.ref, reuse: options.reuseDen, log: denLog });
    if (den.created) recordCreated(den.sandbox);
    const mock = await startMockOnSandbox({ sandbox: den.sandbox, log: prefixed("mock") });
    async function desktop(name: "a" | "b", reuse: string | undefined): Promise<DesktopSandbox> {
      const desktopLog = prefixed(name, (line) => {
        const match = /desktop sandbox created:\s*(\S+)/.exec(line);
        if (match?.[1]) recordCreated(match[1]);
      });
      const result = await provisionDesktopSandbox({ ref: options.ref, name, reuse, log: desktopLog });
      if (result.created) recordCreated(result.sandbox);
      return result;
    }
    const [desktopA, desktopB] = await Promise.all([
      desktop("a", options.reuseA),
      desktop("b", options.reuseB),
    ]);
    await writeFile(options.out, renderConnectorE2eTestEnv({
      denApiUrl: den.apiUrl,
      denWebUrl: den.webUrl,
      sandboxA: desktopA.sandbox,
      sandboxB: desktopB.sandbox,
      mockUrl: mock.url,
      ref: options.ref,
      created,
    }), "utf8");
    console.log(`Wrote ${resolve(options.out)}`);
    console.log(`set -a; source ${options.out}; set +a`);
    console.log("pnpm --dir evals exec vitest run --config vitest.config.ts --project e2e specs/org-connector-two-members.e2e.test.ts");
    console.log(`node scripts/provision-org-connector-two-members.ts --delete ${options.out}`);
  } catch (error) {
    console.error(`Provisioning failed. Provision-created sandboxes so far: ${created.join(", ") || "(none)"}`);
    throw error;
  }
}
async function main(argv: string[]): Promise<void> {
  const { values } = parseArgs({
    args: argv,
    options: {
      ref: { type: "string" },
      "reuse-a": { type: "string" },
      "reuse-b": { type: "string" },
      "reuse-den": { type: "string" },
      out: { type: "string" },
      delete: { type: "string" },
      help: { type: "boolean", short: "h" },
    },
    strict: true,
    allowPositionals: false,
  });
  if (values.help) {
    console.log(USAGE);
    return;
  }
  if (values.delete) {
    await deleteProvisioned(values.delete);
    return;
  }
  if (!values.ref) throw new Error(`--ref is required.\n\n${USAGE}`);
  await provisionSpecEnv({
    ref: values.ref,
    reuseA: values["reuse-a"],
    reuseB: values["reuse-b"],
    reuseDen: values["reuse-den"],
    out: values.out ?? "org-connector-two-members.env",
  });
}
main(process.argv.slice(2)).catch((error: unknown) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exitCode = 1;
});
@@ -0,0 +1,104 @@
// Scenario coverage matrix for evals/specs.
//
//   node scripts/spec-coverage-matrix.mjs [--out <file.md>]
//
// Quantifies scenario coverage two ways:
//   1. Per contract in specs/contracts.snapshot.json: how many pr-lane,
//      e2e-lane, and live specs prove it, and which lanes are missing.
//   2. Specs not referenced by any contract (unmapped coverage), grouped by
//      slug prefix so gaps in the contract map are visible.
import { readdirSync, readFileSync, writeFileSync } from "node:fs"
import { relative, resolve } from "node:path"
import { fileURLToPath } from "node:url"
const evalsRoot = fileURLToPath(new URL("..", import.meta.url))
const specsDirectory = resolve(evalsRoot, "specs")
export function laneOf(specName) {
  if (specName.endsWith(".e2e.test.ts")) return "e2e"
  if (specName.endsWith(".live.test.ts")) return "live"
  if (specName.endsWith(".test.ts")) return "pr"
  return null
}
export function areaOf(specName) {
  return specName.split("-")[0].replace(/\..*$/, "")
}
export function buildMatrix(specNames, contracts) {
  const lanes = new Map(specNames.map((name) => [name, laneOf(name)]))
  const mapped = new Set()
  const contractRows = contracts.map((contract) => {
    const specs = contract.specs
      .map((path) => path.replace(/^evals\/specs\//, ""))
      .filter((name) => lanes.has(name))
    for (const name of specs) mapped.add(name)
    const counts = { pr: 0, e2e: 0, live: 0 }
    for (const name of specs) counts[lanes.get(name)] += 1
    const missingLanes = ["pr", "e2e"].filter((lane) => counts[lane] === 0)
    return { id: contract.id, description: contract.description, counts, specCount: specs.length, missingLanes }
  })
  const unmapped = specNames.filter((name) => !mapped.has(name))
  const unmappedByArea = new Map()
  for (const name of unmapped) {
    const area = areaOf(name)
    const entry = unmappedByArea.get(area) ?? { area, pr: 0, e2e: 0, live: 0 }
    entry[lanes.get(name)] += 1
    unmappedByArea.set(area, entry)
  }
  return {
    totalSpecs: specNames.length,
    mappedSpecs: mapped.size,
    unmappedSpecs: unmapped.length,
    contracts: contractRows.sort((left, right) => left.specCount - right.specCount || left.id.localeCompare(right.id)),
    unmappedAreas: [...unmappedByArea.values()].sort((left, right) => left.area.localeCompare(right.area)),
  }
}
export function renderMarkdown(matrix) {
  const lines = [
    "# Scenario coverage matrix",
    "",
    `Specs: ${matrix.totalSpecs} · Mapped to a contract: ${matrix.mappedSpecs} · Unmapped: ${matrix.unmappedSpecs}`,
    "",
    "## Contracts",
    "",
    "| Contract | pr | e2e | live | Missing lanes |",
    "| --- | ---: | ---: | ---: | --- |",
  ]
  for (const row of matrix.contracts) {
    lines.push(
      `| ${row.id} | ${row.counts.pr} | ${row.counts.e2e} | ${row.counts.live} | ${row.missingLanes.join(", ") || "—"} |`,
    )
  }
  lines.push(
    "",
    "## Specs without a contract (by area)",
    "",
    "These specs run but are invisible to spec-impact analysis. Add them to",
    "`specs/contracts.snapshot.json` or confirm they are intentionally standalone.",
    "",
    "| Area | pr | e2e | live |",
    "| --- | ---: | ---: | ---: |",
  )
  for (const row of matrix.unmappedAreas) {
    lines.push(`| ${row.area} | ${row.pr} | ${row.e2e} | ${row.live} |`)
  }
  return `${lines.join("\n")}\n`
}
const invokedDirectly = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)
if (invokedDirectly) {
  const outFlag = process.argv.indexOf("--out")
  const specNames = readdirSync(specsDirectory).filter((name) => laneOf(name) !== null)
  const snapshot = JSON.parse(readFileSync(resolve(specsDirectory, "contracts.snapshot.json"), "utf8"))
  const matrix = buildMatrix(specNames, snapshot.contracts)
  const markdown = renderMarkdown(matrix)
  if (outFlag !== -1) {
    const outPath = resolve(process.cwd(), process.argv[outFlag + 1])
    writeFileSync(outPath, markdown)
    console.log(`spec-coverage-matrix: wrote ${relative(process.cwd(), outPath)}`)
  } else {
    console.log(markdown)
  }
}
@@ -0,0 +1,56 @@
import assert from "node:assert/strict"
import { test } from "node:test"
import { areaOf, buildMatrix, laneOf, renderMarkdown } from "./spec-coverage-matrix.mjs"
test("laneOf classifies spec lanes", () => {
  assert.equal(laneOf("a.test.ts"), "pr")
  assert.equal(laneOf("a.e2e.test.ts"), "e2e")
  assert.equal(laneOf("a.live.test.ts"), "live")
  assert.equal(laneOf("contracts.snapshot.json"), null)
})
test("areaOf groups by slug prefix", () => {
  assert.equal(areaOf("automation-revision-revert.e2e.test.ts"), "automation")
  assert.equal(areaOf("workflows.e2e.test.ts"), "workflows")
})
test("buildMatrix counts lanes per contract and finds unmapped specs", () => {
  const specNames = [
    "automation-a.e2e.test.ts",
    "automation-b.test.ts",
    "workflow-x.e2e.test.ts",
    "orphan-thing.test.ts",
    "orphan-thing-two.e2e.test.ts",
  ]
  const contracts = [
    {
      id: "desktop.automations",
      description: "automations",
      specs: ["evals/specs/automation-a.e2e.test.ts", "evals/specs/automation-b.test.ts"],
    },
    { id: "den.workflows", description: "workflows", specs: ["evals/specs/workflow-x.e2e.test.ts"] },
  ]
  const matrix = buildMatrix(specNames, contracts)
  assert.equal(matrix.totalSpecs, 5)
  assert.equal(matrix.mappedSpecs, 3)
  assert.equal(matrix.unmappedSpecs, 2)
  const automations = matrix.contracts.find((row) => row.id === "desktop.automations")
  assert.deepEqual(automations.counts, { pr: 1, e2e: 1, live: 0 })
  assert.deepEqual(automations.missingLanes, [])
  const workflows = matrix.contracts.find((row) => row.id === "den.workflows")
  assert.deepEqual(workflows.missingLanes, ["pr"])
  assert.deepEqual(matrix.unmappedAreas, [{ area: "orphan", pr: 1, e2e: 1, live: 0 }])
})
test("renderMarkdown includes contract rows and unmapped areas", () => {
  const matrix = buildMatrix(
    ["automation-a.e2e.test.ts", "orphan-thing.test.ts"],
    [{ id: "desktop.automations", description: "automations", specs: ["evals/specs/automation-a.e2e.test.ts"] }],
  )
  const markdown = renderMarkdown(matrix)
  assert.match(markdown, /\| desktop\.automations \| 0 \| 1 \| 0 \| pr \|/)
  assert.match(markdown, /\| orphan \| 1 \| 0 \| 0 \|/)
})
@@ -0,0 +1,144 @@
import { execFileSync } from "node:child_process"
import { appendFileSync, readFileSync } from "node:fs"
import { resolve } from "node:path"
import { fileURLToPath } from "node:url"
const repoRoot = fileURLToPath(new URL("../..", import.meta.url))
const defaultSnapshot = resolve(repoRoot, "evals/specs/contracts.snapshot.json")
function matches(pathname, pattern) {
  if (pattern.endsWith("/**")) return pathname.startsWith(pattern.slice(0, -3))
  return pathname === pattern
}
function strings(value, label) {
  if (!Array.isArray(value) || value.length === 0 || value.some((entry) => typeof entry !== "string" || entry.length === 0)) {
    throw new Error(`${label} must be a non-empty string array`)
  }
  return value
}
export function validateSnapshot(value) {
  if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("snapshot must be an object")
  if (Reflect.get(value, "version") !== 1) throw new Error("snapshot version must be 1")
  const rawContracts = Reflect.get(value, "contracts")
  if (!Array.isArray(rawContracts) || rawContracts.length === 0) throw new Error("snapshot contracts must be a non-empty array")
  const ids = new Set()
  return rawContracts.map((contract, index) => {
    if (typeof contract !== "object" || contract === null || Array.isArray(contract)) {
      throw new Error(`contracts[${index}] must be an object`)
    }
    const id = Reflect.get(contract, "id")
    const description = Reflect.get(contract, "description")
    if (typeof id !== "string" || id.length === 0) throw new Error(`contracts[${index}].id must be a string`)
    if (ids.has(id)) throw new Error(`duplicate contract id: ${id}`)
    ids.add(id)
    if (typeof description !== "string" || description.length === 0) {
      throw new Error(`contracts[${index}].description must be a string`)
    }
    return {
      id,
      description,
      implementation: strings(Reflect.get(contract, "implementation"), `${id}.implementation`),
      specs: strings(Reflect.get(contract, "specs"), `${id}.specs`),
    }
  })
}
export function analyzeImpact(contracts, changedFiles) {
  const changed = [...new Set(changedFiles.map((entry) => entry.trim()).filter(Boolean))].sort()
  const changedE2eTests = changed.filter((pathname) => pathname.startsWith("evals/specs/") && pathname.endsWith(".e2e.test.ts"))
  const impacted = contracts.flatMap((contract) => {
    const implementation = changed.filter((pathname) => contract.implementation.some((pattern) => matches(pathname, pattern)))
    if (implementation.length === 0) return []
    const specs = changed.filter((pathname) => contract.specs.includes(pathname))
    return [{ ...contract, changedImplementation: implementation, changedSpecs: specs, covered: specs.length > 0 }]
  })
  return {
    changed,
    changedE2eTests,
    impacted,
    attention: impacted.filter((contract) => !contract.covered),
    matchedTests: [...new Set([...impacted.flatMap((contract) => contract.specs), ...changedE2eTests])].sort(),
  }
}
function markdown(result) {
  const lines = ["### Soft spec-impact snapshot", ""]
  if (result.impacted.length === 0) {
    lines.push(
      "No mapped implementation contract changed.",
      "Warden suggestion: add or update an `evals/specs/<feature>.e2e.test.ts` test for this change if it affects app behavior.",
      "",
    )
    return lines.join("\n")
  }
  lines.push("| Contract | Result | Changed implementation | Mapped E2E tests |", "| --- | --- | --- | --- |")
  for (const contract of result.impacted) {
    const status = contract.covered ? "Covered by a changed E2E test" : "Needs attention"
    const implementation = contract.changedImplementation.map((entry) => `\`${entry}\``).join("<br>")
    const specs = contract.specs.map((entry) => `\`${entry}\``).join("<br>")
    lines.push(`| \`${contract.id}\` | ${status} | ${implementation} | ${specs} |`)
  }
  lines.push("")
  if (result.attention.length > 0) {
    lines.push("This check is advisory: confirm the existing mapped E2E test still proves the contract, or update/add an E2E test.", "")
  }
  lines.push(`Matched E2E tests: ${result.matchedTests.map((test) => `\`${test}\``).join(", ") || "none"}`, "")
  return lines.join("\n")
}
function annotation(value) {
  return value.replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A")
}
function parseArgs(args) {
  const options = { changedFiles: [] }
  for (let index = 0; index < args.length; index += 1) {
    const argument = args[index]
    if (argument === "--strict") options.strict = true
    else if (argument === "--json") options.json = true
    else if (argument === "--matched-tests") options.matchedTests = true
    else if (["--base", "--head", "--snapshot", "--summary", "--changed-file"].includes(argument)) {
      const value = args[index + 1]
      if (!value) throw new Error(`${argument} requires a value`)
      index += 1
      if (argument === "--changed-file") options.changedFiles.push(value)
      else options[argument.slice(2)] = value
    } else throw new Error(`unknown argument: ${argument}`)
  }
  return options
}
function gitChangedFiles(base, head) {
  if (!base || !head) throw new Error("pass both --base and --head, or one or more --changed-file values")
  return execFileSync("git", ["diff", "--name-only", "--diff-filter=ACMR", `${base}...${head}`], {
    cwd: repoRoot,
    encoding: "utf8",
  }).split("\n").filter(Boolean)
}
function main() {
  const options = parseArgs(process.argv.slice(2))
  const snapshotPath = resolve(repoRoot, options.snapshot ?? defaultSnapshot)
  const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8"))
  const contracts = validateSnapshot(snapshot)
  const changedFiles = options.changedFiles.length > 0
    ? options.changedFiles
    : gitChangedFiles(options.base, options.head)
  const result = analyzeImpact(contracts, changedFiles)
  const report = markdown(result)
  if (options.summary) appendFileSync(options.summary, report)
  if (options.matchedTests) {
    process.stdout.write(`${JSON.stringify(result.matchedTests)}\n`)
    return
  }
  if (options.json) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
  else process.stdout.write(report)
  for (const contract of result.attention) {
    process.stdout.write(`::warning title=Spec impact snapshot::${annotation(`${contract.id} changed without a mapped E2E test change`)}\n`)
  }
  if (options.strict && result.attention.length > 0) process.exitCode = 2
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main()