源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/packages/test-artifacts/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./publish-pr.ts";
+export * from "./render.ts";
+export * from "./scan.ts";
+export * from "./schema.ts";

evals/packages/test-artifacts/src/publish-pr.ts

@@ -0,0 +1,256 @@
+import { spawnSync } from "node:child_process";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { renderPrMarkdown } from "./render.ts";
+import { readTestRunDirectory } from "./scan.ts";
+
+const BLOB_API_BASE = "https://blob.vercel-storage.com";
+const MARKER = "<!-- test-evidence -->";
+const LEGACY_MARKERS = ["<!-- photo-roll -->", "<!-- fraimz -->"];
+
+export interface CommandOptions {
+  input?: string;
+}
+
+export interface CommandResult {
+  status: number | null;
+  stdout: string;
+  stderr: string;
+  error?: Error;
+}
+
+export type CommandRunner = (command: string, args: string[], opts?: CommandOptions) => CommandResult;
+export type Fetcher = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export interface PublishDependencies {
+  exec?: CommandRunner;
+  fetch?: Fetcher;
+  stdout?: (markdown: string) => void;
+}
+
+export interface PublishPrOptions {
+  pr?: string | number;
+  testRunDir: string;
+  dryRun?: boolean;
+  force?: boolean;
+}
+
+export interface PublishPrResult {
+  markdown: string;
+  posted: boolean;
+  updated: boolean;
+  urls: Record<string, string>;
+}
+
+function commandRunner(command: string, args: string[], opts: CommandOptions = {}): CommandResult {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    input: opts.input,
+  });
+  return {
+    status: result.status,
+    stdout: typeof result.stdout === "string" ? result.stdout : "",
+    stderr: typeof result.stderr === "string" ? result.stderr : "",
+    error: result.error,
+  };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function formatTestRunAge(createdAt: string, now = Date.now()): string {
+  const elapsed = Math.max(0, now - Date.parse(createdAt));
+  const minutes = Math.floor(elapsed / 60_000);
+  if (minutes < 60) return `${minutes}m old`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h old`;
+  return `${Math.floor(hours / 24)}d old`;
+}
+
+function shortSha(sha: string): string {
+  return sha.slice(0, 7);
+}
+
+function resolveBlobToken(exec: CommandRunner): string | null {
+  const fromEnv = process.env.BLOB_READ_WRITE_TOKEN;
+  if (fromEnv) return fromEnv;
+  const result = exec(
+    "infisical",
+    ["secrets", "get", "BLOB_READ_WRITE_TOKEN", "--plain", "--silent"],
+  );
+  const token = result.status === 0 && !result.error ? result.stdout.trim() : "";
+  return token.length > 0 ? token : null;
+}
+
+async function uploadImages(
+  testRunDir: string,
+  testRunId: string,
+  files: string[],
+  token: string,
+  fetcher: Fetcher,
+): Promise<Record<string, string>> {
+  const urls: Record<string, string> = {};
+  const realDir = await realpath(testRunDir);
+  for (const file of files) {
+    if (basename(file) !== file || !file.toLowerCase().endsWith(".png")) {
+      throw new Error(`Refusing to upload invalid test artifact path: ${file}`);
+    }
+    const filePath = join(realDir, file);
+    const stats = await lstat(filePath).catch(() => null);
+    if (!stats?.isFile()) {
+      throw new Error(`Refusing to upload non-regular or symlinked test artifact: ${file}`);
+    }
+    const pathname = `test-artifacts/${encodeURIComponent(testRunId)}/${encodeURIComponent(file)}`;
+    const response = await fetcher(`${BLOB_API_BASE}/${pathname}`, {
+      method: "PUT",
+      headers: {
+        authorization: `Bearer ${token}`,
+        "x-content-type": "image/png",
+        "x-add-random-suffix": "0",
+      },
+      body: await readFile(filePath),
+    });
+    if (!response.ok) {
+      const detail = (await response.text()).slice(0, 300);
+      throw new Error(`Vercel Blob upload failed (${response.status}) for ${file}: ${detail}`);
+    }
+    let payload: unknown;
+    try {
+      payload = await response.json();
+    } catch {
+      throw new Error(`Vercel Blob upload for ${file}: response was not JSON`);
+    }
+    if (!isRecord(payload) || typeof payload.url !== "string" || payload.url.length === 0) {
+      throw new Error(`Vercel Blob upload for ${file}: response did not include a url`);
+    }
+    urls[file] = payload.url;
+  }
+  return urls;
+}
+
+function stickyCommentId(raw: string): string | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (!isRecord(value) || !Array.isArray(value.comments)) return null;
+  const markers = [MARKER, ...LEGACY_MARKERS];
+  for (const comment of value.comments) {
+    if (!isRecord(comment) || typeof comment.body !== "string") continue;
+    const body = comment.body;
+    if (!markers.some((marker) => body.includes(marker))) continue;
+    const directId = comment.databaseId ?? comment.id;
+    if (typeof directId === "number" && Number.isInteger(directId)) return String(directId);
+    if (typeof directId === "string" && /^\d+$/.test(directId)) return directId;
+    if (typeof comment.url === "string") {
+      const match = /#issuecomment-(\d+)$/.exec(comment.url);
+      if (match?.[1]) return match[1];
+    }
+  }
+  return null;
+}
+
+function requireSuccess(result: CommandResult, label: string): void {
+  if (result.status === 0 && !result.error) return;
+  const stderr = result.stderr.trim();
+  const detail = result.error?.message ?? (stderr || `exit ${result.status}`);
+  throw new Error(`${label} failed: ${detail}`);
+}
+
+function resolvePrHeadSha(pr: string, exec: CommandRunner): string {
+  const viewed = exec("gh", ["pr", "view", pr, "--json", "headRefOid"]);
+  if (viewed.status !== 0 || viewed.error) {
+    const detail = viewed.error?.message ?? viewed.stderr.trim();
+    throw new Error(`Unable to resolve PR head SHA with gh${detail ? `: ${detail}` : "."} Install GitHub CLI if needed, then run \`gh auth login\`.`);
+  }
+  let payload: unknown;
+  try {
+    payload = JSON.parse(viewed.stdout);
+  } catch {
+    throw new Error("Unable to resolve PR head SHA with gh: response was not JSON. Run `gh auth login` and try again.");
+  }
+  if (!isRecord(payload) || typeof payload.headRefOid !== "string" || payload.headRefOid.length === 0) {
+    throw new Error("Unable to resolve PR head SHA with gh: response did not include headRefOid. Run `gh auth login` and try again.");
+  }
+  return payload.headRefOid;
+}
+
+function postStickyComment(pr: string, markdown: string, exec: CommandRunner): boolean {
+  const viewed = exec("gh", ["pr", "view", pr, "--json", "comments"]);
+  requireSuccess(viewed, "Reading PR comments");
+  const commentId = stickyCommentId(viewed.stdout);
+  if (commentId) {
+    const updated = exec(
+      "gh",
+      ["api", "--method", "PATCH", `repos/{owner}/{repo}/issues/comments/${commentId}`, "--input", "-"],
+      { input: JSON.stringify({ body: markdown }) },
+    );
+    requireSuccess(updated, "Updating test evidence comment");
+    return true;
+  }
+  const posted = exec("gh", ["pr", "comment", pr, "--body-file", "-"], { input: markdown });
+  requireSuccess(posted, "Posting test evidence comment");
+  return false;
+}
+
+export async function publishPr(
+  options: PublishPrOptions,
+  dependencies: PublishDependencies = {},
+): Promise<PublishPrResult> {
+  const stored = await readTestRunDirectory(options.testRunDir);
+  if (!stored) throw new Error(`No valid test-run.json or legacy result found in ${options.testRunDir}`);
+  const { format, testRun } = stored;
+  const exec = dependencies.exec ?? commandRunner;
+  const fetcher = dependencies.fetch ?? globalThis.fetch;
+  const testRunId = basename(options.testRunDir);
+  const pr = options.pr === undefined ? "<n>" : String(options.pr);
+  const reproCommand = `pnpm --dir evals artifacts:publish -- --pr ${pr} --test-run ${testRunId}`;
+  const sourcePath = format === "current"
+    ? `evals/results/test-runs/${testRunId}/test-run.json`
+    : `evals/results/rolls/${testRunId}/roll.json`;
+
+  if (options.dryRun) {
+    const markdown = renderPrMarkdown(testRun, {}, {
+      reproCommand,
+      sourcePath,
+      notice: "Dry run: screenshots were not uploaded.",
+    });
+    (dependencies.stdout ?? ((body) => process.stdout.write(`${body}\n`)))(markdown);
+    return { markdown, posted: false, updated: false, urls: {} };
+  }
+  if (options.pr === undefined) throw new Error("Publishing requires --pr <n>.");
+
+  if (!testRun.gitSha) {
+    throw new Error(`Refusing to publish ${testRunId}: stored test evidence has no gitSha (${formatTestRunAge(testRun.createdAt)}).`);
+  }
+  const prHeadSha = resolvePrHeadSha(String(options.pr), exec);
+  const stale = testRun.gitSha.toLowerCase() !== prHeadSha.toLowerCase();
+  if (stale && !options.force) {
+    throw new Error(`Refusing stale evidence: test run SHA ${testRun.gitSha}, PR head SHA ${prHeadSha} (${formatTestRunAge(testRun.createdAt)}). Use --force to publish it anyway.`);
+  }
+  const staleNotice = stale
+    ? `⚠ evidence from ${shortSha(testRun.gitSha)}, PR head is ${shortSha(prHeadSha)}`
+    : undefined;
+
+  const token = resolveBlobToken(exec);
+  const urls = token
+    ? await uploadImages(
+      options.testRunDir,
+      testRunId,
+      [...new Set(testRun.artifacts.map((artifact) => artifact.fileName).filter((fileName) => fileName.length > 0))],
+      token,
+      fetcher,
+    )
+    : {};
+  const uploadNotice = token ? undefined : "screenshots not uploaded (no BLOB_READ_WRITE_TOKEN)";
+  const markdown = renderPrMarkdown(testRun, urls, {
+    reproCommand,
+    sourcePath,
+    notice: [staleNotice, uploadNotice].filter((notice) => notice !== undefined).join(" · ") || undefined,
+  });
+  const updated = postStickyComment(String(options.pr), markdown, exec);
+  return { markdown, posted: true, updated, urls };
+}

evals/packages/test-artifacts/src/render.ts

@@ -0,0 +1,142 @@
+import type { TestArtifact, TestRunRecord } from "./schema.ts";
+import type { TestArtifactIndexEntry } from "./scan.ts";
+
+export interface RenderPrOptions {
+  title?: string;
+  reproCommand?: string;
+  notice?: string;
+  sourcePath?: string;
+}
+
+function html(value: string): string {
+  return value
+    .replaceAll("&", "&")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#39;");
+}
+
+function artifactBadge(entry: TestArtifactIndexEntry): { label: string; className: string; summary: string } {
+  if (entry.kind === "legacy-artifacts") {
+    return {
+      label: "LEGACY",
+      className: "legacy",
+      summary: `${entry.pngFiles.length} loose screenshot${entry.pngFiles.length === 1 ? "" : "s"}`,
+    };
+  }
+  const summary = entry.testRun.summary;
+  if (summary.failedArtifacts > 0 || summary.failedExpectations > 0) {
+    return {
+      label: "FAILED",
+      className: "failed",
+      summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed · ${summary.failedArtifacts} failed`,
+    };
+  }
+  if (summary.pendingArtifacts > 0 || summary.pendingJudgments > 0) {
+    return {
+      label: "PENDING",
+      className: "pending",
+      summary: `${summary.pendingArtifacts}/${summary.totalArtifacts} artifacts pending · ${summary.pendingJudgments} expectations pending`,
+    };
+  }
+  if (summary.ok) {
+    return {
+      label: "PASSED",
+      className: "passed",
+      summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed`,
+    };
+  }
+  return {
+    label: "UNVALIDATED",
+    className: "unvalidated",
+    summary: `${summary.unvalidatedArtifacts}/${summary.totalArtifacts} artifacts unvalidated`,
+  };
+}
+
+export function renderArtifactIndexHtml(entries: TestArtifactIndexEntry[]): string {
+  const ordered = [...entries].sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+  const cards = ordered.map((entry) => {
+    const badge = artifactBadge(entry);
+    const caption = entry.kind === "test-run" ? entry.testRun.artifacts[0]?.caption : undefined;
+    const thumbnail = entry.thumbnailHref
+      ? `<img src="${html(entry.thumbnailHref)}" alt="${html(caption ?? entry.name)}">`
+      : `<div class="empty">No screenshot recorded</div>`;
+    return `<article class="card ${badge.className}">
+      <a class="thumb" href="${html(entry.href)}">${thumbnail}</a>
+      <div class="copy">
+        <div class="topline"><span class="badge">${badge.label}</span><time datetime="${html(entry.createdAt)}">${html(entry.createdAt)}</time></div>
+        <h2><a href="${html(entry.href)}">${html(entry.name)}</a></h2>
+        ${caption ? `<p class="caption">${html(caption)}</p>` : ""}
+        <p class="summary">${html(badge.summary)}</p>
+      </div>
+    </article>`;
+  }).join("\n");
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>OpenWork test artifacts</title><style>
+body{font:15px/1.5 system-ui,sans-serif;max-width:1100px;margin:40px auto;padding:0 20px;background:#f6f7f9;color:#17191d}header,.card{background:white;border:1px solid #dfe2e8;border-radius:12px;padding:20px;margin:0 0 24px}header h1{margin:0 0 6px}.muted,time,.summary{color:#636c76}.card{display:grid;grid-template-columns:minmax(220px,40%) 1fr;gap:20px}.card.passed{border-left:6px solid #238636}.card.failed{border-left:6px solid #cf222e}.card.pending,.card.unvalidated{border-left:6px solid #9a6700}.card.legacy{border-left:6px solid #656d76}.thumb img,.empty{display:block;width:100%;aspect-ratio:16/10;object-fit:cover;border:1px solid #dfe2e8;border-radius:8px;background:#f6f7f9}.empty{display:grid;place-items:center;color:#636c76}.topline{display:flex;align-items:center;gap:10px}.badge{font-size:12px;font-weight:700}.passed .badge{color:#1a7f37}.failed .badge{color:#cf222e}.pending .badge,.unvalidated .badge{color:#9a6700}.legacy .badge{color:#656d76}h2{margin:12px 0 6px}a{color:inherit}.caption{font-weight:600}@media(max-width:700px){.card{grid-template-columns:1fr}}
+</style></head><body><header><h1>Test artifacts</h1><p class="muted">Test evidence captured as the user experienced it, newest first.</p></header>${cards || '<p class="muted">No test artifacts found.</p>'}</body></html>\n`;
+}
+
+function summaryLine(testRun: TestRunRecord): string {
+  const summary = testRun.summary;
+  const failed = summary.failedArtifacts > 0 || summary.failedExpectations > 0;
+  const pending = summary.pendingArtifacts > 0 || summary.pendingJudgments > 0;
+  const icon = failed ? "❌" : pending ? "⏳" : summary.ok ? "✅" : "⚪";
+  return `${icon} **${artifactVerdict(testRun)}** · ${summary.passedExpectations} expectations passed · ${summary.failedExpectations} failed · ${summary.pendingJudgments} pending`;
+}
+
+function renderArtifact(artifact: TestArtifact, sequence: number, imageUrl: string | undefined): string {
+  const assertion = artifact.fileName.length === 0;
+  const pending = artifact.judgments.some((judgment) => judgment.state === "pending");
+  const marker = pending ? "⏳ PENDING" : assertion ? (artifact.ok === false ? "❌ FAIL ASSERTION" : "ℹ️ ASSERTION") : artifact.ok === false ? "❌ FAIL" : artifact.ok === true ? "✅ PASS" : "⚪ UNVALIDATED";
+  const lines = [`### ${marker} — ${sequence}. ${html(artifact.caption)}`, ""];
+  if (assertion && artifact.description) lines.push(html(artifact.description), "");
+  if (artifact.judgments.length === 0) {
+    lines.push("- ⚪ **UNVALIDATED** — no visual expectations recorded.");
+  } else {
+    for (const judgment of artifact.judgments) {
+      const judgmentMarker = judgment.state === "passed" ? "✅ **PASS**" : judgment.state === "failed" ? "❌ **FAIL**" : "⏳ **PENDING**";
+      lines.push(`- ${judgmentMarker} ${html(judgment.expectation)} — ${html(judgment.reasoning)}`);
+    }
+  }
+  if (imageUrl) {
+    lines.push("", `<a href="${html(imageUrl)}"><img src="${html(imageUrl)}" alt="${html(artifact.caption)}" width="700"></a>`);
+  }
+  return lines.join("\n");
+}
+
+function artifactVerdict(testRun: TestRunRecord): string {
+  const screenshots = testRun.artifacts.filter((artifact) => artifact.fileName.length > 0);
+  const passed = screenshots.filter((artifact) => artifact.ok === true).length;
+  const pending = screenshots.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length;
+  const assertions = testRun.artifacts.length - screenshots.length;
+  return `${passed}/${screenshots.length} screenshots passed${pending > 0 ? ` · ${pending} pending` : ""}${assertions > 0 ? ` · ${assertions} assertion${assertions === 1 ? "" : "s"}` : ""}`;
+}
+
+export function renderPrMarkdown(
+  testRun: TestRunRecord,
+  urls: Record<string, string>,
+  opts: RenderPrOptions = {},
+): string {
+  const title = opts.title ?? `Test evidence — ${testRun.name}`;
+  const lines = [
+    "<!-- test-evidence -->",
+    `## ${html(title)} — ${artifactVerdict(testRun)}`,
+    "",
+    summaryLine(testRun),
+  ];
+  if (opts.notice) lines.push("", `> ${opts.notice}`);
+  for (const [index, artifact] of testRun.artifacts.entries()) {
+    lines.push("", renderArtifact(artifact, index + 1, urls[artifact.fileName]));
+  }
+  const source = opts.sourcePath ?? `evals/results/test-runs/${testRun.dir.split(/[\\/]/).at(-1) ?? testRun.name}/test-run.json`;
+  lines.push(
+    "",
+    "---",
+    `_Test run created ${html(testRun.createdAt)} · Source: \`${html(source)}\`${opts.reproCommand ? ` · Repro: \`${html(opts.reproCommand)}\`` : ""}_`,
+    "<!-- /test-evidence -->",
+  );
+  return lines.join("\n");
+}

evals/packages/test-artifacts/src/scan.ts

@@ -0,0 +1,179 @@
+import type { Dirent } from "node:fs";
+import { lstat, readFile, readdir, stat } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { parseLegacyTestRunJson, parseTestRunJson } from "./schema.ts";
+import type { TestRunRecord } from "./schema.ts";
+
+export type StoredTestRunFormat = "current" | "legacy";
+
+export interface RecordedTestRunEntry {
+  kind: "test-run";
+  format: StoredTestRunFormat;
+  directoryName: string;
+  directoryPath: string;
+  name: string;
+  createdAt: string;
+  href: string;
+  thumbnailHref?: string;
+  testRun: TestRunRecord;
+}
+
+export interface LegacyArtifactEntry {
+  kind: "legacy-artifacts";
+  directoryName: string;
+  directoryPath: string;
+  name: string;
+  createdAt: string;
+  href: string;
+  thumbnailHref?: string;
+  pngFiles: string[];
+}
+
+export type TestArtifactIndexEntry = RecordedTestRunEntry | LegacyArtifactEntry;
+
+export interface ReadTestRunResult {
+  format: StoredTestRunFormat;
+  testRun: TestRunRecord;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isMissing(error: unknown): boolean {
+  return isRecord(error) && error.code === "ENOENT";
+}
+
+async function readDirectory(path: string): Promise<Dirent[]> {
+  try {
+    return await readdir(path, { withFileTypes: true });
+  } catch (error) {
+    if (isMissing(error)) return [];
+    throw error;
+  }
+}
+
+function segment(value: string): string {
+  return encodeURIComponent(value);
+}
+
+function timestampFromName(name: string): string | null {
+  const match = /^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z/.exec(name);
+  if (!match) return null;
+  const [, date, hour, minute, second, millis] = match;
+  if (!date || !hour || !minute || !second || !millis) return null;
+  const timestamp = `${date}${hour}:${minute}:${second}.${millis}Z`;
+  return Number.isNaN(Date.parse(timestamp)) ? null : timestamp;
+}
+
+async function readStoredTestRun(path: string, format: StoredTestRunFormat): Promise<TestRunRecord | null> {
+  try {
+    const info = await lstat(path);
+    if (!info.isFile()) return null;
+    const raw = await readFile(path, "utf8");
+    const value: unknown = JSON.parse(raw);
+    return format === "current" ? parseTestRunJson(value) : parseLegacyTestRunJson(value);
+  } catch {
+    return null;
+  }
+}
+
+export async function readTestRunFile(path: string): Promise<TestRunRecord | null> {
+  return readStoredTestRun(path, "current");
+}
+
+export async function readTestRunDirectory(directoryPath: string): Promise<ReadTestRunResult | null> {
+  const current = await readStoredTestRun(join(directoryPath, "test-run.json"), "current");
+  if (current) return { format: "current", testRun: current };
+  const legacy = await readStoredTestRun(join(directoryPath, "roll.json"), "legacy");
+  return legacy ? { format: "legacy", testRun: legacy } : null;
+}
+
+async function scanRecordedTestRuns(resultsDir: string): Promise<RecordedTestRunEntry[]> {
+  const testRunsDir = join(resultsDir, "test-runs");
+  const directories = (await readDirectory(testRunsDir)).filter((entry) => entry.isDirectory());
+  const testRuns: RecordedTestRunEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(testRunsDir, directory.name);
+    const testRun = await readStoredTestRun(join(directoryPath, "test-run.json"), "current");
+    if (!testRun) continue;
+    const firstScreenshot = testRun.artifacts.find((artifact) => artifact.fileName.length > 0);
+    testRuns.push({
+      kind: "test-run",
+      format: "current",
+      directoryName: directory.name,
+      directoryPath,
+      name: testRun.name,
+      createdAt: testRun.createdAt,
+      href: `${segment(directory.name)}/index.html`,
+      thumbnailHref: firstScreenshot ? `${segment(directory.name)}/${segment(firstScreenshot.fileName)}` : undefined,
+      testRun,
+    });
+  }
+  return testRuns;
+}
+
+async function scanLegacyTestRuns(resultsDir: string): Promise<RecordedTestRunEntry[]> {
+  const legacyDir = join(resultsDir, "rolls");
+  const directories = (await readDirectory(legacyDir)).filter((entry) => entry.isDirectory());
+  const testRuns: RecordedTestRunEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(legacyDir, directory.name);
+    const testRun = await readStoredTestRun(join(directoryPath, "roll.json"), "legacy");
+    if (!testRun) continue;
+    const firstScreenshot = testRun.artifacts.find((artifact) => artifact.fileName.length > 0);
+    testRuns.push({
+      kind: "test-run",
+      format: "legacy",
+      directoryName: directory.name,
+      directoryPath,
+      name: testRun.name,
+      createdAt: testRun.createdAt,
+      href: `../rolls/${segment(directory.name)}/index.html`,
+      thumbnailHref: firstScreenshot ? `../rolls/${segment(directory.name)}/${segment(firstScreenshot.fileName)}` : undefined,
+      testRun,
+    });
+  }
+  return testRuns;
+}
+
+async function scanLooseLegacyArtifacts(resultsDir: string): Promise<LegacyArtifactEntry[]> {
+  const directories = (await readDirectory(resultsDir))
+    .filter((entry) => entry.isDirectory() && entry.name !== "test-runs" && entry.name !== "rolls" && !entry.name.startsWith("."));
+  const legacyArtifacts: LegacyArtifactEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(resultsDir, directory.name);
+    const contents = await readDirectory(directoryPath);
+    // fraimz.html is the frozen legacy-runner report filename.
+    const hasLegacyReport = contents.some((entry) => entry.isFile() && entry.name === "fraimz.html");
+    const pngFiles = contents
+      .filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith(".png"))
+      .map((entry) => entry.name)
+      .sort();
+    if (!hasLegacyReport && pngFiles.length === 0) continue;
+    const info = await stat(directoryPath);
+    const createdAt = timestampFromName(directory.name) ?? info.mtime.toISOString();
+    const firstPng = pngFiles[0];
+    const prefix = `../${segment(directory.name)}/`;
+    legacyArtifacts.push({
+      kind: "legacy-artifacts",
+      directoryName: directory.name,
+      directoryPath,
+      name: basename(directoryPath),
+      createdAt,
+      href: `${prefix}${hasLegacyReport ? "fraimz.html" : segment(firstPng ?? "")}`,
+      thumbnailHref: firstPng ? `${prefix}${segment(firstPng)}` : undefined,
+      pngFiles,
+    });
+  }
+  return legacyArtifacts;
+}
+
+export async function scanTestRuns(resultsDir: string): Promise<TestArtifactIndexEntry[]> {
+  const entries = [
+    ...await scanRecordedTestRuns(resultsDir),
+    ...await scanLegacyTestRuns(resultsDir),
+    ...await scanLooseLegacyArtifacts(resultsDir),
+  ];
+  return entries.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+}

evals/packages/test-artifacts/src/schema.ts

@@ -0,0 +1,244 @@
+export interface ArtifactExpectationResult {
+  expectation: string;
+  passed: boolean;
+  evidence: string;
+}
+
+export type ArtifactJudgmentState = "passed" | "failed" | "pending";
+
+export interface ArtifactJudgment {
+  expectation: string;
+  state: ArtifactJudgmentState;
+  reasoning: string;
+}
+
+export interface TestArtifact {
+  caption: string;
+  fileName: string;
+  hash: string;
+  route: string;
+  at: string;
+  description: string;
+  model: string;
+  ok: boolean | null;
+  results: ArtifactExpectationResult[];
+  judgments: ArtifactJudgment[];
+}
+
+export interface TestRunSummary {
+  ok: boolean;
+  totalArtifacts: number;
+  passedArtifacts: number;
+  failedArtifacts: number;
+  unvalidatedArtifacts: number;
+  pendingArtifacts: number;
+  passedExpectations: number;
+  failedExpectations: number;
+  pendingJudgments: number;
+}
+
+export interface TestRunRecord {
+  name: string;
+  dir: string;
+  createdAt: string;
+  closedAt: string;
+  gitSha?: string;
+  branch?: string;
+  summary: TestRunSummary;
+  artifacts: TestArtifact[];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isCount(value: unknown): value is number {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+
+function parseResult(value: unknown): ArtifactExpectationResult | null {
+  if (
+    !isRecord(value)
+    || typeof value.expectation !== "string"
+    || typeof value.passed !== "boolean"
+    || typeof value.evidence !== "string"
+  ) return null;
+  return {
+    expectation: value.expectation,
+    passed: value.passed,
+    evidence: value.evidence,
+  };
+}
+
+function parseJudgment(value: unknown): ArtifactJudgment | null {
+  if (
+    !isRecord(value)
+    || typeof value.expectation !== "string"
+    || (value.state !== "passed" && value.state !== "failed" && value.state !== "pending")
+    || typeof value.reasoning !== "string"
+  ) return null;
+  return {
+    expectation: value.expectation,
+    state: value.state,
+    reasoning: value.reasoning,
+  };
+}
+
+function judgmentForResult(result: ArtifactExpectationResult): ArtifactJudgment {
+  return {
+    expectation: result.expectation,
+    state: result.passed ? "passed" : "failed",
+    reasoning: result.evidence,
+  };
+}
+
+function parseArtifact(value: unknown): TestArtifact | null {
+  if (
+    !isRecord(value)
+    || typeof value.caption !== "string"
+    || typeof value.fileName !== "string"
+    || typeof value.hash !== "string"
+    || typeof value.route !== "string"
+    || typeof value.at !== "string"
+    || typeof value.description !== "string"
+    || typeof value.model !== "string"
+    || !Array.isArray(value.results)
+  ) return null;
+  let ok: boolean | null;
+  if (value.ok === null) ok = null;
+  else if (typeof value.ok === "boolean") ok = value.ok;
+  else return null;
+  const results: ArtifactExpectationResult[] = [];
+  for (const result of value.results) {
+    const parsed = parseResult(result);
+    if (!parsed) return null;
+    results.push(parsed);
+  }
+  const judgments: ArtifactJudgment[] = [];
+  if (value.judgments === undefined) {
+    judgments.push(...results.map(judgmentForResult));
+  } else {
+    if (!Array.isArray(value.judgments)) return null;
+    for (const judgment of value.judgments) {
+      const parsed = parseJudgment(judgment);
+      if (!parsed) return null;
+      judgments.push(parsed);
+    }
+  }
+  return {
+    caption: value.caption,
+    fileName: value.fileName,
+    hash: value.hash,
+    route: value.route,
+    at: value.at,
+    description: value.description,
+    model: value.model,
+    ok,
+    results,
+    judgments,
+  };
+}
+
+function pendingCounts(artifacts: TestArtifact[]): { pendingArtifacts: number; pendingJudgments: number } {
+  return {
+    pendingArtifacts: artifacts.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length,
+    pendingJudgments: artifacts.flatMap((artifact) => artifact.judgments).filter((judgment) => judgment.state === "pending").length,
+  };
+}
+
+function parseCurrentSummary(value: unknown, artifacts: TestArtifact[]): TestRunSummary | null {
+  if (
+    !isRecord(value)
+    || typeof value.ok !== "boolean"
+    || !isCount(value.totalArtifacts)
+    || !isCount(value.passedArtifacts)
+    || !isCount(value.failedArtifacts)
+    || !isCount(value.unvalidatedArtifacts)
+    || !isCount(value.passedExpectations)
+    || !isCount(value.failedExpectations)
+  ) return null;
+  const derived = pendingCounts(artifacts);
+  const pendingArtifacts = value.pendingArtifacts === undefined ? derived.pendingArtifacts : value.pendingArtifacts;
+  const pendingJudgments = value.pendingJudgments === undefined ? derived.pendingJudgments : value.pendingJudgments;
+  if (!isCount(pendingArtifacts) || !isCount(pendingJudgments)) return null;
+  return {
+    ok: value.ok,
+    totalArtifacts: value.totalArtifacts,
+    passedArtifacts: value.passedArtifacts,
+    failedArtifacts: value.failedArtifacts,
+    unvalidatedArtifacts: value.unvalidatedArtifacts,
+    pendingArtifacts,
+    passedExpectations: value.passedExpectations,
+    failedExpectations: value.failedExpectations,
+    pendingJudgments,
+  };
+}
+
+function parseLegacySummary(value: unknown, artifacts: TestArtifact[]): TestRunSummary | null {
+  if (
+    !isRecord(value)
+    || typeof value.ok !== "boolean"
+    || !isCount(value.totalFrames)
+    || !isCount(value.passedFrames)
+    || !isCount(value.failedFrames)
+    || !isCount(value.unvalidatedFrames)
+    || !isCount(value.passedExpectations)
+    || !isCount(value.failedExpectations)
+  ) return null;
+  const pending = pendingCounts(artifacts);
+  return {
+    ok: value.ok,
+    totalArtifacts: value.totalFrames,
+    passedArtifacts: value.passedFrames,
+    failedArtifacts: value.failedFrames,
+    unvalidatedArtifacts: value.unvalidatedFrames,
+    pendingArtifacts: pending.pendingArtifacts,
+    passedExpectations: value.passedExpectations,
+    failedExpectations: value.failedExpectations,
+    pendingJudgments: pending.pendingJudgments,
+  };
+}
+
+function parseRecord(value: unknown, legacy: boolean): TestRunRecord | null {
+  if (
+    !isRecord(value)
+    || typeof value.name !== "string"
+    || typeof value.dir !== "string"
+    || typeof value.createdAt !== "string"
+    || typeof value.closedAt !== "string"
+  ) return null;
+  const entries = legacy ? value.frames : value.artifacts;
+  if (!Array.isArray(entries)) return null;
+  const artifacts: TestArtifact[] = [];
+  for (const entry of entries) {
+    if (isRecord(entry) && entry.kind === "json") {
+      if (typeof entry.label !== "string" || typeof entry.fileName !== "string") return null;
+      continue;
+    }
+    const parsed = parseArtifact(entry);
+    if (!parsed) return null;
+    artifacts.push(parsed);
+  }
+  const summary = legacy ? parseLegacySummary(value.summary, artifacts) : parseCurrentSummary(value.summary, artifacts);
+  if (!summary) return null;
+  const gitSha = typeof value.gitSha === "string" ? value.gitSha : undefined;
+  const branch = typeof value.branch === "string" ? value.branch : undefined;
+  return {
+    name: value.name,
+    dir: value.dir,
+    createdAt: value.createdAt,
+    closedAt: value.closedAt,
+    gitSha,
+    branch,
+    summary,
+    artifacts,
+  };
+}
+
+export function parseTestRunJson(value: unknown): TestRunRecord | null {
+  return parseRecord(value, false);
+}
+
+export function parseLegacyTestRunJson(value: unknown): TestRunRecord | null {
+  return parseRecord(value, true);
+}