源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/packages/test-artifacts/bin/artifact-index.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { renderArtifactIndexHtml, scanTestRuns } from "../src/index.ts";
+
+const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
+const resultsDir = join(repoRoot, "evals", "results");
+const testRunsDir = join(resultsDir, "test-runs");
+
+const args = process.argv.slice(2);
+const shouldOpen = args.includes("--open");
+const unknown = args.filter((arg) => arg !== "--open");
+if (unknown.length > 0) throw new Error(`Unknown argument: ${unknown[0]}`);
+
+await mkdir(testRunsDir, { recursive: true });
+const entries = await scanTestRuns(resultsDir);
+const indexPath = join(testRunsDir, "index.html");
+await writeFile(indexPath, renderArtifactIndexHtml(entries), "utf8");
+process.stdout.write(`${indexPath}\n`);
+
+if (shouldOpen) {
+  if (process.platform !== "darwin") throw new Error("--open is only supported on darwin.");
+  const opened = spawnSync("open", [indexPath], { stdio: "inherit" });
+  if (opened.error || opened.status !== 0) throw opened.error ?? new Error(`open exited ${opened.status}`);
+}

evals/packages/test-artifacts/bin/publish-pr.mjs

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { isAbsolute, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { formatTestRunAge, publishPr, readTestRunDirectory, scanTestRuns } from "../src/index.ts";
+
+const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
+const resultsDir = join(repoRoot, "evals", "results");
+const args = process.argv.slice(2);
+let pr;
+let testRunArg;
+let dryRun = false;
+let force = false;
+let shouldOpen = false;
+
+for (let index = 0; index < args.length; index += 1) {
+  const arg = args[index];
+  if (arg === "--") continue;
+  if (arg === "--dry-run") {
+    dryRun = true;
+    continue;
+  }
+  if (arg === "--force") {
+    force = true;
+    continue;
+  }
+  if (arg === "--open") {
+    shouldOpen = true;
+    continue;
+  }
+  if (arg === "--pr" || arg === "--test-run") {
+    const value = args[index + 1];
+    if (!value || value.startsWith("--")) throw new Error(`${arg} requires a value.`);
+    if (arg === "--pr") pr = value;
+    else testRunArg = value;
+    index += 1;
+    continue;
+  }
+  throw new Error(`Unknown argument: ${arg}`);
+}
+
+if (!dryRun && !shouldOpen && !pr) throw new Error("--pr <n> is required unless --dry-run or --open is set.");
+if (shouldOpen && (pr || dryRun || force)) throw new Error("--open cannot be combined with --pr, --dry-run, or --force.");
+const entries = (await scanTestRuns(resultsDir)).filter((entry) => entry.kind === "test-run");
+let testRunDir;
+let selectedTestRun;
+if (testRunArg && testRunArg !== "latest") {
+  const candidate = isAbsolute(testRunArg) ? testRunArg : resolve(process.cwd(), testRunArg);
+  const stored = await readTestRunDirectory(candidate);
+  if (stored) {
+    testRunDir = candidate;
+    selectedTestRun = stored.testRun;
+  } else {
+    const selected = entries.find((entry) => entry.directoryName === testRunArg || entry.name === testRunArg);
+    testRunDir = selected?.directoryPath;
+    selectedTestRun = selected?.testRun;
+  }
+} else {
+  testRunDir = entries[0]?.directoryPath;
+  selectedTestRun = entries[0]?.testRun;
+}
+if (!testRunDir || !selectedTestRun) throw new Error(`No test run found${testRunArg ? ` for ${testRunArg}` : ""}.`);
+
+process.stdout.write(`Selected test run: ${selectedTestRun.name} · SHA ${selectedTestRun.gitSha ?? "unknown"} · ${formatTestRunAge(selectedTestRun.createdAt)}\n`);
+
+if (shouldOpen) {
+  if (process.platform !== "darwin") throw new Error("--open is only supported on darwin.");
+  const indexPath = join(testRunDir, "index.html");
+  if (!existsSync(indexPath)) throw new Error(`Selected test run has no index.html: ${indexPath}`);
+  const opened = spawnSync("open", [indexPath], { stdio: "inherit" });
+  if (opened.error || opened.status !== 0) throw opened.error ?? new Error(`open exited ${opened.status}`);
+  process.exit(0);
+}
+
+const result = await publishPr({ pr, testRunDir, dryRun, force });
+if (!dryRun) process.stdout.write(`${result.updated ? "Updated" : "Posted"} test evidence for PR ${pr}.\n`);

evals/packages/test-artifacts/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@openwork/test-artifacts",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "development": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  }
+}

evals/packages/test-artifacts/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./publish-pr.ts";
+export * from "./render.ts";
+export * from "./scan.ts";
+export * from "./schema.ts";

evals/packages/test-artifacts/src/publish-pr.ts

@@ -0,0 +1,256 @@
+import { spawnSync } from "node:child_process";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { renderPrMarkdown } from "./render.ts";
+import { readTestRunDirectory } from "./scan.ts";
+
+const BLOB_API_BASE = "https://blob.vercel-storage.com";
+const MARKER = "<!-- test-evidence -->";
+const LEGACY_MARKERS = ["<!-- photo-roll -->", "<!-- fraimz -->"];
+
+export interface CommandOptions {
+  input?: string;
+}
+
+export interface CommandResult {
+  status: number | null;
+  stdout: string;
+  stderr: string;
+  error?: Error;
+}
+
+export type CommandRunner = (command: string, args: string[], opts?: CommandOptions) => CommandResult;
+export type Fetcher = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export interface PublishDependencies {
+  exec?: CommandRunner;
+  fetch?: Fetcher;
+  stdout?: (markdown: string) => void;
+}
+
+export interface PublishPrOptions {
+  pr?: string | number;
+  testRunDir: string;
+  dryRun?: boolean;
+  force?: boolean;
+}
+
+export interface PublishPrResult {
+  markdown: string;
+  posted: boolean;
+  updated: boolean;
+  urls: Record<string, string>;
+}
+
+function commandRunner(command: string, args: string[], opts: CommandOptions = {}): CommandResult {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    input: opts.input,
+  });
+  return {
+    status: result.status,
+    stdout: typeof result.stdout === "string" ? result.stdout : "",
+    stderr: typeof result.stderr === "string" ? result.stderr : "",
+    error: result.error,
+  };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function formatTestRunAge(createdAt: string, now = Date.now()): string {
+  const elapsed = Math.max(0, now - Date.parse(createdAt));
+  const minutes = Math.floor(elapsed / 60_000);
+  if (minutes < 60) return `${minutes}m old`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h old`;
+  return `${Math.floor(hours / 24)}d old`;
+}
+
+function shortSha(sha: string): string {
+  return sha.slice(0, 7);
+}
+
+function resolveBlobToken(exec: CommandRunner): string | null {
+  const fromEnv = process.env.BLOB_READ_WRITE_TOKEN;
+  if (fromEnv) return fromEnv;
+  const result = exec(
+    "infisical",
+    ["secrets", "get", "BLOB_READ_WRITE_TOKEN", "--plain", "--silent"],
+  );
+  const token = result.status === 0 && !result.error ? result.stdout.trim() : "";
+  return token.length > 0 ? token : null;
+}
+
+async function uploadImages(
+  testRunDir: string,
+  testRunId: string,
+  files: string[],
+  token: string,
+  fetcher: Fetcher,
+): Promise<Record<string, string>> {
+  const urls: Record<string, string> = {};
+  const realDir = await realpath(testRunDir);
+  for (const file of files) {
+    if (basename(file) !== file || !file.toLowerCase().endsWith(".png")) {
+      throw new Error(`Refusing to upload invalid test artifact path: ${file}`);
+    }
+    const filePath = join(realDir, file);
+    const stats = await lstat(filePath).catch(() => null);
+    if (!stats?.isFile()) {
+      throw new Error(`Refusing to upload non-regular or symlinked test artifact: ${file}`);
+    }
+    const pathname = `test-artifacts/${encodeURIComponent(testRunId)}/${encodeURIComponent(file)}`;
+    const response = await fetcher(`${BLOB_API_BASE}/${pathname}`, {
+      method: "PUT",
+      headers: {
+        authorization: `Bearer ${token}`,
+        "x-content-type": "image/png",
+        "x-add-random-suffix": "0",
+      },
+      body: await readFile(filePath),
+    });
+    if (!response.ok) {
+      const detail = (await response.text()).slice(0, 300);
+      throw new Error(`Vercel Blob upload failed (${response.status}) for ${file}: ${detail}`);
+    }
+    let payload: unknown;
+    try {
+      payload = await response.json();
+    } catch {
+      throw new Error(`Vercel Blob upload for ${file}: response was not JSON`);
+    }
+    if (!isRecord(payload) || typeof payload.url !== "string" || payload.url.length === 0) {
+      throw new Error(`Vercel Blob upload for ${file}: response did not include a url`);
+    }
+    urls[file] = payload.url;
+  }
+  return urls;
+}
+
+function stickyCommentId(raw: string): string | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (!isRecord(value) || !Array.isArray(value.comments)) return null;
+  const markers = [MARKER, ...LEGACY_MARKERS];
+  for (const comment of value.comments) {
+    if (!isRecord(comment) || typeof comment.body !== "string") continue;
+    const body = comment.body;
+    if (!markers.some((marker) => body.includes(marker))) continue;
+    const directId = comment.databaseId ?? comment.id;
+    if (typeof directId === "number" && Number.isInteger(directId)) return String(directId);
+    if (typeof directId === "string" && /^\d+$/.test(directId)) return directId;
+    if (typeof comment.url === "string") {
+      const match = /#issuecomment-(\d+)$/.exec(comment.url);
+      if (match?.[1]) return match[1];
+    }
+  }
+  return null;
+}
+
+function requireSuccess(result: CommandResult, label: string): void {
+  if (result.status === 0 && !result.error) return;
+  const stderr = result.stderr.trim();
+  const detail = result.error?.message ?? (stderr || `exit ${result.status}`);
+  throw new Error(`${label} failed: ${detail}`);
+}
+
+function resolvePrHeadSha(pr: string, exec: CommandRunner): string {
+  const viewed = exec("gh", ["pr", "view", pr, "--json", "headRefOid"]);
+  if (viewed.status !== 0 || viewed.error) {
+    const detail = viewed.error?.message ?? viewed.stderr.trim();
+    throw new Error(`Unable to resolve PR head SHA with gh${detail ? `: ${detail}` : "."} Install GitHub CLI if needed, then run \`gh auth login\`.`);
+  }
+  let payload: unknown;
+  try {
+    payload = JSON.parse(viewed.stdout);
+  } catch {
+    throw new Error("Unable to resolve PR head SHA with gh: response was not JSON. Run `gh auth login` and try again.");
+  }
+  if (!isRecord(payload) || typeof payload.headRefOid !== "string" || payload.headRefOid.length === 0) {
+    throw new Error("Unable to resolve PR head SHA with gh: response did not include headRefOid. Run `gh auth login` and try again.");
+  }
+  return payload.headRefOid;
+}
+
+function postStickyComment(pr: string, markdown: string, exec: CommandRunner): boolean {
+  const viewed = exec("gh", ["pr", "view", pr, "--json", "comments"]);
+  requireSuccess(viewed, "Reading PR comments");
+  const commentId = stickyCommentId(viewed.stdout);
+  if (commentId) {
+    const updated = exec(
+      "gh",
+      ["api", "--method", "PATCH", `repos/{owner}/{repo}/issues/comments/${commentId}`, "--input", "-"],
+      { input: JSON.stringify({ body: markdown }) },
+    );
+    requireSuccess(updated, "Updating test evidence comment");
+    return true;
+  }
+  const posted = exec("gh", ["pr", "comment", pr, "--body-file", "-"], { input: markdown });
+  requireSuccess(posted, "Posting test evidence comment");
+  return false;
+}
+
+export async function publishPr(
+  options: PublishPrOptions,
+  dependencies: PublishDependencies = {},
+): Promise<PublishPrResult> {
+  const stored = await readTestRunDirectory(options.testRunDir);
+  if (!stored) throw new Error(`No valid test-run.json or legacy result found in ${options.testRunDir}`);
+  const { format, testRun } = stored;
+  const exec = dependencies.exec ?? commandRunner;
+  const fetcher = dependencies.fetch ?? globalThis.fetch;
+  const testRunId = basename(options.testRunDir);
+  const pr = options.pr === undefined ? "<n>" : String(options.pr);
+  const reproCommand = `pnpm --dir evals artifacts:publish -- --pr ${pr} --test-run ${testRunId}`;
+  const sourcePath = format === "current"
+    ? `evals/results/test-runs/${testRunId}/test-run.json`
+    : `evals/results/rolls/${testRunId}/roll.json`;
+
+  if (options.dryRun) {
+    const markdown = renderPrMarkdown(testRun, {}, {
+      reproCommand,
+      sourcePath,
+      notice: "Dry run: screenshots were not uploaded.",
+    });
+    (dependencies.stdout ?? ((body) => process.stdout.write(`${body}\n`)))(markdown);
+    return { markdown, posted: false, updated: false, urls: {} };
+  }
+  if (options.pr === undefined) throw new Error("Publishing requires --pr <n>.");
+
+  if (!testRun.gitSha) {
+    throw new Error(`Refusing to publish ${testRunId}: stored test evidence has no gitSha (${formatTestRunAge(testRun.createdAt)}).`);
+  }
+  const prHeadSha = resolvePrHeadSha(String(options.pr), exec);
+  const stale = testRun.gitSha.toLowerCase() !== prHeadSha.toLowerCase();
+  if (stale && !options.force) {
+    throw new Error(`Refusing stale evidence: test run SHA ${testRun.gitSha}, PR head SHA ${prHeadSha} (${formatTestRunAge(testRun.createdAt)}). Use --force to publish it anyway.`);
+  }
+  const staleNotice = stale
+    ? `⚠ evidence from ${shortSha(testRun.gitSha)}, PR head is ${shortSha(prHeadSha)}`
+    : undefined;
+
+  const token = resolveBlobToken(exec);
+  const urls = token
+    ? await uploadImages(
+      options.testRunDir,
+      testRunId,
+      [...new Set(testRun.artifacts.map((artifact) => artifact.fileName).filter((fileName) => fileName.length > 0))],
+      token,
+      fetcher,
+    )
+    : {};
+  const uploadNotice = token ? undefined : "screenshots not uploaded (no BLOB_READ_WRITE_TOKEN)";
+  const markdown = renderPrMarkdown(testRun, urls, {
+    reproCommand,
+    sourcePath,
+    notice: [staleNotice, uploadNotice].filter((notice) => notice !== undefined).join(" · ") || undefined,
+  });
+  const updated = postStickyComment(String(options.pr), markdown, exec);
+  return { markdown, posted: true, updated, urls };
+}

evals/packages/test-artifacts/src/render.ts

@@ -0,0 +1,142 @@
+import type { TestArtifact, TestRunRecord } from "./schema.ts";
+import type { TestArtifactIndexEntry } from "./scan.ts";
+
+export interface RenderPrOptions {
+  title?: string;
+  reproCommand?: string;
+  notice?: string;
+  sourcePath?: string;
+}
+
+function html(value: string): string {
+  return value
+    .replaceAll("&", "&")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#39;");
+}
+
+function artifactBadge(entry: TestArtifactIndexEntry): { label: string; className: string; summary: string } {
+  if (entry.kind === "legacy-artifacts") {
+    return {
+      label: "LEGACY",
+      className: "legacy",
+      summary: `${entry.pngFiles.length} loose screenshot${entry.pngFiles.length === 1 ? "" : "s"}`,
+    };
+  }
+  const summary = entry.testRun.summary;
+  if (summary.failedArtifacts > 0 || summary.failedExpectations > 0) {
+    return {
+      label: "FAILED",
+      className: "failed",
+      summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed · ${summary.failedArtifacts} failed`,
+    };
+  }
+  if (summary.pendingArtifacts > 0 || summary.pendingJudgments > 0) {
+    return {
+      label: "PENDING",
+      className: "pending",
+      summary: `${summary.pendingArtifacts}/${summary.totalArtifacts} artifacts pending · ${summary.pendingJudgments} expectations pending`,
+    };
+  }
+  if (summary.ok) {
+    return {
+      label: "PASSED",
+      className: "passed",
+      summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed`,
+    };
+  }
+  return {
+    label: "UNVALIDATED",
+    className: "unvalidated",
+    summary: `${summary.unvalidatedArtifacts}/${summary.totalArtifacts} artifacts unvalidated`,
+  };
+}
+
+export function renderArtifactIndexHtml(entries: TestArtifactIndexEntry[]): string {
+  const ordered = [...entries].sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+  const cards = ordered.map((entry) => {
+    const badge = artifactBadge(entry);
+    const caption = entry.kind === "test-run" ? entry.testRun.artifacts[0]?.caption : undefined;
+    const thumbnail = entry.thumbnailHref
+      ? `<img src="${html(entry.thumbnailHref)}" alt="${html(caption ?? entry.name)}">`
+      : `<div class="empty">No screenshot recorded</div>`;
+    return `<article class="card ${badge.className}">
+      <a class="thumb" href="${html(entry.href)}">${thumbnail}</a>
+      <div class="copy">
+        <div class="topline"><span class="badge">${badge.label}</span><time datetime="${html(entry.createdAt)}">${html(entry.createdAt)}</time></div>
+        <h2><a href="${html(entry.href)}">${html(entry.name)}</a></h2>
+        ${caption ? `<p class="caption">${html(caption)}</p>` : ""}
+        <p class="summary">${html(badge.summary)}</p>
+      </div>
+    </article>`;
+  }).join("\n");
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>OpenWork test artifacts</title><style>
+body{font:15px/1.5 system-ui,sans-serif;max-width:1100px;margin:40px auto;padding:0 20px;background:#f6f7f9;color:#17191d}header,.card{background:white;border:1px solid #dfe2e8;border-radius:12px;padding:20px;margin:0 0 24px}header h1{margin:0 0 6px}.muted,time,.summary{color:#636c76}.card{display:grid;grid-template-columns:minmax(220px,40%) 1fr;gap:20px}.card.passed{border-left:6px solid #238636}.card.failed{border-left:6px solid #cf222e}.card.pending,.card.unvalidated{border-left:6px solid #9a6700}.card.legacy{border-left:6px solid #656d76}.thumb img,.empty{display:block;width:100%;aspect-ratio:16/10;object-fit:cover;border:1px solid #dfe2e8;border-radius:8px;background:#f6f7f9}.empty{display:grid;place-items:center;color:#636c76}.topline{display:flex;align-items:center;gap:10px}.badge{font-size:12px;font-weight:700}.passed .badge{color:#1a7f37}.failed .badge{color:#cf222e}.pending .badge,.unvalidated .badge{color:#9a6700}.legacy .badge{color:#656d76}h2{margin:12px 0 6px}a{color:inherit}.caption{font-weight:600}@media(max-width:700px){.card{grid-template-columns:1fr}}
+</style></head><body><header><h1>Test artifacts</h1><p class="muted">Test evidence captured as the user experienced it, newest first.</p></header>${cards || '<p class="muted">No test artifacts found.</p>'}</body></html>\n`;
+}
+
+function summaryLine(testRun: TestRunRecord): string {
+  const summary = testRun.summary;
+  const failed = summary.failedArtifacts > 0 || summary.failedExpectations > 0;
+  const pending = summary.pendingArtifacts > 0 || summary.pendingJudgments > 0;
+  const icon = failed ? "❌" : pending ? "⏳" : summary.ok ? "✅" : "⚪";
+  return `${icon} **${artifactVerdict(testRun)}** · ${summary.passedExpectations} expectations passed · ${summary.failedExpectations} failed · ${summary.pendingJudgments} pending`;
+}
+
+function renderArtifact(artifact: TestArtifact, sequence: number, imageUrl: string | undefined): string {
+  const assertion = artifact.fileName.length === 0;
+  const pending = artifact.judgments.some((judgment) => judgment.state === "pending");
+  const marker = pending ? "⏳ PENDING" : assertion ? (artifact.ok === false ? "❌ FAIL ASSERTION" : "ℹ️ ASSERTION") : artifact.ok === false ? "❌ FAIL" : artifact.ok === true ? "✅ PASS" : "⚪ UNVALIDATED";
+  const lines = [`### ${marker} — ${sequence}. ${html(artifact.caption)}`, ""];
+  if (assertion && artifact.description) lines.push(html(artifact.description), "");
+  if (artifact.judgments.length === 0) {
+    lines.push("- ⚪ **UNVALIDATED** — no visual expectations recorded.");
+  } else {
+    for (const judgment of artifact.judgments) {
+      const judgmentMarker = judgment.state === "passed" ? "✅ **PASS**" : judgment.state === "failed" ? "❌ **FAIL**" : "⏳ **PENDING**";
+      lines.push(`- ${judgmentMarker} ${html(judgment.expectation)} — ${html(judgment.reasoning)}`);
+    }
+  }
+  if (imageUrl) {
+    lines.push("", `<a href="${html(imageUrl)}"><img src="${html(imageUrl)}" alt="${html(artifact.caption)}" width="700"></a>`);
+  }
+  return lines.join("\n");
+}
+
+function artifactVerdict(testRun: TestRunRecord): string {
+  const screenshots = testRun.artifacts.filter((artifact) => artifact.fileName.length > 0);
+  const passed = screenshots.filter((artifact) => artifact.ok === true).length;
+  const pending = screenshots.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length;
+  const assertions = testRun.artifacts.length - screenshots.length;
+  return `${passed}/${screenshots.length} screenshots passed${pending > 0 ? ` · ${pending} pending` : ""}${assertions > 0 ? ` · ${assertions} assertion${assertions === 1 ? "" : "s"}` : ""}`;
+}
+
+export function renderPrMarkdown(
+  testRun: TestRunRecord,
+  urls: Record<string, string>,
+  opts: RenderPrOptions = {},
+): string {
+  const title = opts.title ?? `Test evidence — ${testRun.name}`;
+  const lines = [
+    "<!-- test-evidence -->",
+    `## ${html(title)} — ${artifactVerdict(testRun)}`,
+    "",
+    summaryLine(testRun),
+  ];
+  if (opts.notice) lines.push("", `> ${opts.notice}`);
+  for (const [index, artifact] of testRun.artifacts.entries()) {
+    lines.push("", renderArtifact(artifact, index + 1, urls[artifact.fileName]));
+  }
+  const source = opts.sourcePath ?? `evals/results/test-runs/${testRun.dir.split(/[\\/]/).at(-1) ?? testRun.name}/test-run.json`;
+  lines.push(
+    "",
+    "---",
+    `_Test run created ${html(testRun.createdAt)} · Source: \`${html(source)}\`${opts.reproCommand ? ` · Repro: \`${html(opts.reproCommand)}\`` : ""}_`,
+    "<!-- /test-evidence -->",
+  );
+  return lines.join("\n");
+}

evals/packages/test-artifacts/src/scan.ts

@@ -0,0 +1,179 @@
+import type { Dirent } from "node:fs";
+import { lstat, readFile, readdir, stat } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { parseLegacyTestRunJson, parseTestRunJson } from "./schema.ts";
+import type { TestRunRecord } from "./schema.ts";
+
+export type StoredTestRunFormat = "current" | "legacy";
+
+export interface RecordedTestRunEntry {
+  kind: "test-run";
+  format: StoredTestRunFormat;
+  directoryName: string;
+  directoryPath: string;
+  name: string;
+  createdAt: string;
+  href: string;
+  thumbnailHref?: string;
+  testRun: TestRunRecord;
+}
+
+export interface LegacyArtifactEntry {
+  kind: "legacy-artifacts";
+  directoryName: string;
+  directoryPath: string;
+  name: string;
+  createdAt: string;
+  href: string;
+  thumbnailHref?: string;
+  pngFiles: string[];
+}
+
+export type TestArtifactIndexEntry = RecordedTestRunEntry | LegacyArtifactEntry;
+
+export interface ReadTestRunResult {
+  format: StoredTestRunFormat;
+  testRun: TestRunRecord;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isMissing(error: unknown): boolean {
+  return isRecord(error) && error.code === "ENOENT";
+}
+
+async function readDirectory(path: string): Promise<Dirent[]> {
+  try {
+    return await readdir(path, { withFileTypes: true });
+  } catch (error) {
+    if (isMissing(error)) return [];
+    throw error;
+  }
+}
+
+function segment(value: string): string {
+  return encodeURIComponent(value);
+}
+
+function timestampFromName(name: string): string | null {
+  const match = /^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z/.exec(name);
+  if (!match) return null;
+  const [, date, hour, minute, second, millis] = match;
+  if (!date || !hour || !minute || !second || !millis) return null;
+  const timestamp = `${date}${hour}:${minute}:${second}.${millis}Z`;
+  return Number.isNaN(Date.parse(timestamp)) ? null : timestamp;
+}
+
+async function readStoredTestRun(path: string, format: StoredTestRunFormat): Promise<TestRunRecord | null> {
+  try {
+    const info = await lstat(path);
+    if (!info.isFile()) return null;
+    const raw = await readFile(path, "utf8");
+    const value: unknown = JSON.parse(raw);
+    return format === "current" ? parseTestRunJson(value) : parseLegacyTestRunJson(value);
+  } catch {
+    return null;
+  }
+}
+
+export async function readTestRunFile(path: string): Promise<TestRunRecord | null> {
+  return readStoredTestRun(path, "current");
+}
+
+export async function readTestRunDirectory(directoryPath: string): Promise<ReadTestRunResult | null> {
+  const current = await readStoredTestRun(join(directoryPath, "test-run.json"), "current");
+  if (current) return { format: "current", testRun: current };
+  const legacy = await readStoredTestRun(join(directoryPath, "roll.json"), "legacy");
+  return legacy ? { format: "legacy", testRun: legacy } : null;
+}
+
+async function scanRecordedTestRuns(resultsDir: string): Promise<RecordedTestRunEntry[]> {
+  const testRunsDir = join(resultsDir, "test-runs");
+  const directories = (await readDirectory(testRunsDir)).filter((entry) => entry.isDirectory());
+  const testRuns: RecordedTestRunEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(testRunsDir, directory.name);
+    const testRun = await readStoredTestRun(join(directoryPath, "test-run.json"), "current");
+    if (!testRun) continue;
+    const firstScreenshot = testRun.artifacts.find((artifact) => artifact.fileName.length > 0);
+    testRuns.push({
+      kind: "test-run",
+      format: "current",
+      directoryName: directory.name,
+      directoryPath,
+      name: testRun.name,
+      createdAt: testRun.createdAt,
+      href: `${segment(directory.name)}/index.html`,
+      thumbnailHref: firstScreenshot ? `${segment(directory.name)}/${segment(firstScreenshot.fileName)}` : undefined,
+      testRun,
+    });
+  }
+  return testRuns;
+}
+
+async function scanLegacyTestRuns(resultsDir: string): Promise<RecordedTestRunEntry[]> {
+  const legacyDir = join(resultsDir, "rolls");
+  const directories = (await readDirectory(legacyDir)).filter((entry) => entry.isDirectory());
+  const testRuns: RecordedTestRunEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(legacyDir, directory.name);
+    const testRun = await readStoredTestRun(join(directoryPath, "roll.json"), "legacy");
+    if (!testRun) continue;
+    const firstScreenshot = testRun.artifacts.find((artifact) => artifact.fileName.length > 0);
+    testRuns.push({
+      kind: "test-run",
+      format: "legacy",
+      directoryName: directory.name,
+      directoryPath,
+      name: testRun.name,
+      createdAt: testRun.createdAt,
+      href: `../rolls/${segment(directory.name)}/index.html`,
+      thumbnailHref: firstScreenshot ? `../rolls/${segment(directory.name)}/${segment(firstScreenshot.fileName)}` : undefined,
+      testRun,
+    });
+  }
+  return testRuns;
+}
+
+async function scanLooseLegacyArtifacts(resultsDir: string): Promise<LegacyArtifactEntry[]> {
+  const directories = (await readDirectory(resultsDir))
+    .filter((entry) => entry.isDirectory() && entry.name !== "test-runs" && entry.name !== "rolls" && !entry.name.startsWith("."));
+  const legacyArtifacts: LegacyArtifactEntry[] = [];
+  for (const directory of directories) {
+    const directoryPath = join(resultsDir, directory.name);
+    const contents = await readDirectory(directoryPath);
+    // fraimz.html is the frozen legacy-runner report filename.
+    const hasLegacyReport = contents.some((entry) => entry.isFile() && entry.name === "fraimz.html");
+    const pngFiles = contents
+      .filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith(".png"))
+      .map((entry) => entry.name)
+      .sort();
+    if (!hasLegacyReport && pngFiles.length === 0) continue;
+    const info = await stat(directoryPath);
+    const createdAt = timestampFromName(directory.name) ?? info.mtime.toISOString();
+    const firstPng = pngFiles[0];
+    const prefix = `../${segment(directory.name)}/`;
+    legacyArtifacts.push({
+      kind: "legacy-artifacts",
+      directoryName: directory.name,
+      directoryPath,
+      name: basename(directoryPath),
+      createdAt,
+      href: `${prefix}${hasLegacyReport ? "fraimz.html" : segment(firstPng ?? "")}`,
+      thumbnailHref: firstPng ? `${prefix}${segment(firstPng)}` : undefined,
+      pngFiles,
+    });
+  }
+  return legacyArtifacts;
+}
+
+export async function scanTestRuns(resultsDir: string): Promise<TestArtifactIndexEntry[]> {
+  const entries = [
+    ...await scanRecordedTestRuns(resultsDir),
+    ...await scanLegacyTestRuns(resultsDir),
+    ...await scanLooseLegacyArtifacts(resultsDir),
+  ];
+  return entries.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+}

evals/packages/test-artifacts/src/schema.ts

@@ -0,0 +1,244 @@
+export interface ArtifactExpectationResult {
+  expectation: string;
+  passed: boolean;
+  evidence: string;
+}
+
+export type ArtifactJudgmentState = "passed" | "failed" | "pending";
+
+export interface ArtifactJudgment {
+  expectation: string;
+  state: ArtifactJudgmentState;
+  reasoning: string;
+}
+
+export interface TestArtifact {
+  caption: string;
+  fileName: string;
+  hash: string;
+  route: string;
+  at: string;
+  description: string;
+  model: string;
+  ok: boolean | null;
+  results: ArtifactExpectationResult[];
+  judgments: ArtifactJudgment[];
+}
+
+export interface TestRunSummary {
+  ok: boolean;
+  totalArtifacts: number;
+  passedArtifacts: number;
+  failedArtifacts: number;
+  unvalidatedArtifacts: number;
+  pendingArtifacts: number;
+  passedExpectations: number;
+  failedExpectations: number;
+  pendingJudgments: number;
+}
+
+export interface TestRunRecord {
+  name: string;
+  dir: string;
+  createdAt: string;
+  closedAt: string;
+  gitSha?: string;
+  branch?: string;
+  summary: TestRunSummary;
+  artifacts: TestArtifact[];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isCount(value: unknown): value is number {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+
+function parseResult(value: unknown): ArtifactExpectationResult | null {
+  if (
+    !isRecord(value)
+    || typeof value.expectation !== "string"
+    || typeof value.passed !== "boolean"
+    || typeof value.evidence !== "string"
+  ) return null;
+  return {
+    expectation: value.expectation,
+    passed: value.passed,
+    evidence: value.evidence,
+  };
+}
+
+function parseJudgment(value: unknown): ArtifactJudgment | null {
+  if (
+    !isRecord(value)
+    || typeof value.expectation !== "string"
+    || (value.state !== "passed" && value.state !== "failed" && value.state !== "pending")
+    || typeof value.reasoning !== "string"
+  ) return null;
+  return {
+    expectation: value.expectation,
+    state: value.state,
+    reasoning: value.reasoning,
+  };
+}
+
+function judgmentForResult(result: ArtifactExpectationResult): ArtifactJudgment {
+  return {
+    expectation: result.expectation,
+    state: result.passed ? "passed" : "failed",
+    reasoning: result.evidence,
+  };
+}
+
+function parseArtifact(value: unknown): TestArtifact | null {
+  if (
+    !isRecord(value)
+    || typeof value.caption !== "string"
+    || typeof value.fileName !== "string"
+    || typeof value.hash !== "string"
+    || typeof value.route !== "string"
+    || typeof value.at !== "string"
+    || typeof value.description !== "string"
+    || typeof value.model !== "string"
+    || !Array.isArray(value.results)
+  ) return null;
+  let ok: boolean | null;
+  if (value.ok === null) ok = null;
+  else if (typeof value.ok === "boolean") ok = value.ok;
+  else return null;
+  const results: ArtifactExpectationResult[] = [];
+  for (const result of value.results) {
+    const parsed = parseResult(result);
+    if (!parsed) return null;
+    results.push(parsed);
+  }
+  const judgments: ArtifactJudgment[] = [];
+  if (value.judgments === undefined) {
+    judgments.push(...results.map(judgmentForResult));
+  } else {
+    if (!Array.isArray(value.judgments)) return null;
+    for (const judgment of value.judgments) {
+      const parsed = parseJudgment(judgment);
+      if (!parsed) return null;
+      judgments.push(parsed);
+    }
+  }
+  return {
+    caption: value.caption,
+    fileName: value.fileName,
+    hash: value.hash,
+    route: value.route,
+    at: value.at,
+    description: value.description,
+    model: value.model,
+    ok,
+    results,
+    judgments,
+  };
+}
+
+function pendingCounts(artifacts: TestArtifact[]): { pendingArtifacts: number; pendingJudgments: number } {
+  return {
+    pendingArtifacts: artifacts.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length,
+    pendingJudgments: artifacts.flatMap((artifact) => artifact.judgments).filter((judgment) => judgment.state === "pending").length,
+  };
+}
+
+function parseCurrentSummary(value: unknown, artifacts: TestArtifact[]): TestRunSummary | null {
+  if (
+    !isRecord(value)
+    || typeof value.ok !== "boolean"
+    || !isCount(value.totalArtifacts)
+    || !isCount(value.passedArtifacts)
+    || !isCount(value.failedArtifacts)
+    || !isCount(value.unvalidatedArtifacts)
+    || !isCount(value.passedExpectations)
+    || !isCount(value.failedExpectations)
+  ) return null;
+  const derived = pendingCounts(artifacts);
+  const pendingArtifacts = value.pendingArtifacts === undefined ? derived.pendingArtifacts : value.pendingArtifacts;
+  const pendingJudgments = value.pendingJudgments === undefined ? derived.pendingJudgments : value.pendingJudgments;
+  if (!isCount(pendingArtifacts) || !isCount(pendingJudgments)) return null;
+  return {
+    ok: value.ok,
+    totalArtifacts: value.totalArtifacts,
+    passedArtifacts: value.passedArtifacts,
+    failedArtifacts: value.failedArtifacts,
+    unvalidatedArtifacts: value.unvalidatedArtifacts,
+    pendingArtifacts,
+    passedExpectations: value.passedExpectations,
+    failedExpectations: value.failedExpectations,
+    pendingJudgments,
+  };
+}
+
+function parseLegacySummary(value: unknown, artifacts: TestArtifact[]): TestRunSummary | null {
+  if (
+    !isRecord(value)
+    || typeof value.ok !== "boolean"
+    || !isCount(value.totalFrames)
+    || !isCount(value.passedFrames)
+    || !isCount(value.failedFrames)
+    || !isCount(value.unvalidatedFrames)
+    || !isCount(value.passedExpectations)
+    || !isCount(value.failedExpectations)
+  ) return null;
+  const pending = pendingCounts(artifacts);
+  return {
+    ok: value.ok,
+    totalArtifacts: value.totalFrames,
+    passedArtifacts: value.passedFrames,
+    failedArtifacts: value.failedFrames,
+    unvalidatedArtifacts: value.unvalidatedFrames,
+    pendingArtifacts: pending.pendingArtifacts,
+    passedExpectations: value.passedExpectations,
+    failedExpectations: value.failedExpectations,
+    pendingJudgments: pending.pendingJudgments,
+  };
+}
+
+function parseRecord(value: unknown, legacy: boolean): TestRunRecord | null {
+  if (
+    !isRecord(value)
+    || typeof value.name !== "string"
+    || typeof value.dir !== "string"
+    || typeof value.createdAt !== "string"
+    || typeof value.closedAt !== "string"
+  ) return null;
+  const entries = legacy ? value.frames : value.artifacts;
+  if (!Array.isArray(entries)) return null;
+  const artifacts: TestArtifact[] = [];
+  for (const entry of entries) {
+    if (isRecord(entry) && entry.kind === "json") {
+      if (typeof entry.label !== "string" || typeof entry.fileName !== "string") return null;
+      continue;
+    }
+    const parsed = parseArtifact(entry);
+    if (!parsed) return null;
+    artifacts.push(parsed);
+  }
+  const summary = legacy ? parseLegacySummary(value.summary, artifacts) : parseCurrentSummary(value.summary, artifacts);
+  if (!summary) return null;
+  const gitSha = typeof value.gitSha === "string" ? value.gitSha : undefined;
+  const branch = typeof value.branch === "string" ? value.branch : undefined;
+  return {
+    name: value.name,
+    dir: value.dir,
+    createdAt: value.createdAt,
+    closedAt: value.closedAt,
+    gitSha,
+    branch,
+    summary,
+    artifacts,
+  };
+}
+
+export function parseTestRunJson(value: unknown): TestRunRecord | null {
+  return parseRecord(value, false);
+}
+
+export function parseLegacyTestRunJson(value: unknown): TestRunRecord | null {
+  return parseRecord(value, true);
+}

evals/packages/test-artifacts/test/publish-pr.test.ts

@@ -0,0 +1,164 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { publishPr } from "../src/publish-pr.ts";
+import type { CommandRunner, Fetcher } from "../src/publish-pr.ts";
+import type { TestRunRecord } from "../src/schema.ts";
+
+const TEST_RUN_SHA = "1111111111111111111111111111111111111111";
+
+function testRunRecord(dir: string): TestRunRecord {
+  return {
+    name: "Publication proof",
+    dir,
+    createdAt: "2026-07-02T10:00:00.000Z",
+    closedAt: "2026-07-02T10:01:00.000Z",
+    gitSha: TEST_RUN_SHA,
+    branch: "feat/proof",
+    summary: {
+      ok: true,
+      totalArtifacts: 1,
+      passedArtifacts: 1,
+      failedArtifacts: 0,
+      unvalidatedArtifacts: 0,
+      pendingArtifacts: 0,
+      passedExpectations: 1,
+      failedExpectations: 0,
+      pendingJudgments: 0,
+    },
+    artifacts: [{
+      caption: "Published validation",
+      fileName: "01-published.png",
+      hash: "hash",
+      route: "#/published",
+      at: "2026-07-02T10:00:00.000Z",
+      description: "Visible state",
+      model: "test-model",
+      ok: true,
+      results: [{ expectation: "State is visible", passed: true, evidence: "Visible" }],
+      judgments: [{ expectation: "State is visible", state: "passed", reasoning: "Visible" }],
+    }],
+  };
+}
+
+test("publishPr dry-run emits the new marker without upload or gh calls", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-publish-"));
+  try {
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    let commandCalled = false;
+    let fetchCalled = false;
+    let output = "";
+    const exec: CommandRunner = () => {
+      commandCalled = true;
+      return { status: 1, stdout: "", stderr: "unexpected command" };
+    };
+    const fetcher: Fetcher = async () => {
+      fetchCalled = true;
+      throw new Error("unexpected fetch");
+    };
+    const result = await publishPr(
+      { testRunDir, dryRun: true },
+      { exec, fetch: fetcher, stdout: (markdown) => { output = markdown; } },
+    );
+    assert.equal(commandCalled, false);
+    assert.equal(fetchCalled, false);
+    assert.equal(result.posted, false);
+    assert.match(output, /<!-- test-evidence -->/);
+    assert.match(output, /Dry run: screenshots were not uploaded/);
+  } finally {
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr recognizes an old sticky marker and uploads under the test-artifacts prefix", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-current-"));
+  const previousToken = process.env.BLOB_READ_WRITE_TOKEN;
+  try {
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    await writeFile(join(testRunDir, "01-published.png"), Buffer.from("regular png"));
+    process.env.BLOB_READ_WRITE_TOKEN = "test-token";
+    const uploads: string[] = [];
+    let patchedBody = "";
+    const fetcher: Fetcher = async (input) => {
+      uploads.push(String(input));
+      return new Response(JSON.stringify({ url: "https://example.test/published.png" }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    };
+    const exec: CommandRunner = (_command, args, opts) => {
+      if (args.includes("headRefOid")) return { status: 0, stdout: JSON.stringify({ headRefOid: TEST_RUN_SHA }), stderr: "" };
+      if (args.includes("comments")) {
+        return { status: 0, stdout: JSON.stringify({ comments: [{ databaseId: 77, body: "<!-- photo-roll --> old" }] }), stderr: "" };
+      }
+      patchedBody = opts?.input ?? "";
+      return { status: 0, stdout: "updated", stderr: "" };
+    };
+    const result = await publishPr({ pr: 17, testRunDir }, { exec, fetch: fetcher });
+    assert.equal(result.updated, true);
+    assert.equal(uploads.length, 1);
+    assert.match(uploads[0] ?? "", /\/test-artifacts\/[^/]+\/01-published\.png$/);
+    assert.match(patchedBody, /<!-- test-evidence -->/);
+    assert.doesNotMatch(patchedBody, /<!-- photo-roll -->/);
+  } finally {
+    if (previousToken === undefined) delete process.env.BLOB_READ_WRITE_TOKEN;
+    else process.env.BLOB_READ_WRITE_TOKEN = previousToken;
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr reads persisted legacy roll.json input", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-legacy-"));
+  try {
+    const current = testRunRecord(testRunDir);
+    await writeFile(join(testRunDir, "roll.json"), JSON.stringify({
+      ...current,
+      summary: {
+        ok: true,
+        totalFrames: 1,
+        passedFrames: 1,
+        failedFrames: 0,
+        unvalidatedFrames: 0,
+        passedExpectations: 1,
+        failedExpectations: 0,
+      },
+      frames: current.artifacts,
+      artifacts: undefined,
+    }));
+    let output = "";
+    await publishPr({ testRunDir, dryRun: true }, { stdout: (markdown) => { output = markdown; } });
+    assert.match(output, /evals\/results\/rolls\/.*\/roll\.json/);
+    assert.match(output, /<!-- test-evidence -->/);
+  } finally {
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr refuses a symlinked screenshot before upload", async () => {
+  const root = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-symlink-"));
+  const testRunDir = join(root, "test-run");
+  const previousToken = process.env.BLOB_READ_WRITE_TOKEN;
+  try {
+    await mkdir(testRunDir);
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    const outside = join(root, "private-key");
+    await writeFile(outside, "private material");
+    await symlink(outside, join(testRunDir, "01-published.png"));
+    process.env.BLOB_READ_WRITE_TOKEN = "test-token";
+    const exec: CommandRunner = (_command, args) => ({
+      status: 0,
+      stdout: args.includes("headRefOid") ? JSON.stringify({ headRefOid: TEST_RUN_SHA }) : "",
+      stderr: "",
+    });
+    await assert.rejects(
+      () => publishPr({ pr: 17, testRunDir }, { exec }),
+      /Refusing to upload non-regular or symlinked test artifact: 01-published\.png/,
+    );
+  } finally {
+    if (previousToken === undefined) delete process.env.BLOB_READ_WRITE_TOKEN;
+    else process.env.BLOB_READ_WRITE_TOKEN = previousToken;
+    await rm(root, { recursive: true, force: true });
+  }
+});

evals/packages/test-artifacts/test/scan-render.test.ts

@@ -0,0 +1,119 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { renderArtifactIndexHtml, renderPrMarkdown } from "../src/render.ts";
+import { scanTestRuns } from "../src/scan.ts";
+import type { TestRunRecord } from "../src/schema.ts";
+
+function record(name: string, dir: string, createdAt: string, passed: boolean): TestRunRecord {
+  return {
+    name,
+    dir,
+    createdAt,
+    closedAt: createdAt,
+    summary: {
+      ok: passed,
+      totalArtifacts: 1,
+      passedArtifacts: passed ? 1 : 0,
+      failedArtifacts: passed ? 0 : 1,
+      unvalidatedArtifacts: 0,
+      pendingArtifacts: 0,
+      passedExpectations: passed ? 1 : 0,
+      failedExpectations: passed ? 0 : 1,
+      pendingJudgments: 0,
+    },
+    artifacts: [{
+      caption: `${name} first validation`,
+      fileName: "01-first.png",
+      hash: `${name}-hash`,
+      route: "#/workspace/test",
+      at: createdAt,
+      description: `${name} description`,
+      model: "test-model",
+      ok: passed,
+      results: [{ expectation: `${name} expectation`, passed, evidence: `${name} evidence` }],
+      judgments: [{ expectation: `${name} expectation`, state: passed ? "passed" : "failed", reasoning: `${name} evidence` }],
+    }],
+  };
+}
+
+function legacyRecord(testRun: TestRunRecord): Record<string, unknown> {
+  const frames = testRun.artifacts.map((artifact) => {
+    const { judgments, ...legacyArtifact } = artifact;
+    void judgments;
+    return legacyArtifact;
+  });
+  return {
+    ...testRun,
+    summary: {
+      ok: testRun.summary.ok,
+      totalFrames: testRun.summary.totalArtifacts,
+      passedFrames: testRun.summary.passedArtifacts,
+      failedFrames: testRun.summary.failedArtifacts,
+      unvalidatedFrames: testRun.summary.unvalidatedArtifacts,
+      passedExpectations: testRun.summary.passedExpectations,
+      failedExpectations: testRun.summary.failedExpectations,
+    },
+    frames,
+    artifacts: undefined,
+  };
+}
+
+test("scanTestRuns reads current and persisted legacy results while tolerating corrupt data", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-scan-"));
+  try {
+    const currentDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-current");
+    const oldDir = join(resultsDir, "rolls", "2026-07-01T10-00-00-000Z-old");
+    const corruptDir = join(resultsDir, "test-runs", "2026-07-03T10-00-00-000Z-corrupt");
+    const looseDir = join(resultsDir, "2026-06-01T10-00-00-000Z-legacy");
+    await Promise.all([currentDir, oldDir, corruptDir, looseDir].map((directory) => mkdir(directory, { recursive: true })));
+    await Promise.all([
+      writeFile(join(currentDir, "test-run.json"), JSON.stringify(record("Current", currentDir, "2026-07-02T10:00:00.000Z", false))),
+      writeFile(join(oldDir, "roll.json"), JSON.stringify(legacyRecord(record("Persisted", oldDir, "2026-07-01T10:00:00.000Z", true)))),
+      writeFile(join(corruptDir, "test-run.json"), "{not-json"),
+      writeFile(join(looseDir, "fraimz.html"), "legacy"),
+      writeFile(join(looseDir, "frame.png"), Buffer.from("legacy-png")),
+    ]);
+
+    const entries = await scanTestRuns(resultsDir);
+    assert.deepEqual(entries.map((entry) => entry.name), ["Current", "Persisted", "2026-06-01T10-00-00-000Z-legacy"]);
+    const storedEntries = entries.filter((entry) => entry.kind === "test-run");
+    assert.deepEqual(storedEntries.map((entry) => entry.format), ["current", "legacy"]);
+    assert.deepEqual(storedEntries[1]?.testRun.artifacts[0]?.judgments, [{
+      expectation: "Persisted expectation",
+      state: "passed",
+      reasoning: "Persisted evidence",
+    }]);
+
+    const rendered = renderArtifactIndexHtml(entries);
+    for (const expected of ["Current first validation", "FAILED", "PASSED", "current/index.html", "../rolls/", "fraimz.html"]) {
+      assert.match(rendered, new RegExp(expected));
+    }
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});
+
+test("renderPrMarkdown writes only the new test-evidence marker", () => {
+  const testRun = record("PR proof", "/tmp/2026-pr-proof", "2026-07-02T10:00:00.000Z", false);
+  const markdown = renderPrMarkdown(testRun, { "01-first.png": "https://example.test/01-first.png" });
+  assert.match(markdown, /<!-- test-evidence -->/);
+  assert.doesNotMatch(markdown, /<!-- photo-roll -->|<!-- fraimz -->/);
+  assert.match(markdown, /PR proof first validation/);
+});
+
+test("scanTestRuns skips a symlinked test-run.json", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-symlink-"));
+  try {
+    const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-symlinked");
+    await mkdir(testRunDir, { recursive: true });
+    const outside = join(resultsDir, "outside-test-run.json");
+    await writeFile(outside, JSON.stringify(record("Outside", testRunDir, "2026-07-02T10:00:00.000Z", true)));
+    await symlink(outside, join(testRunDir, "test-run.json"));
+    assert.deepEqual(await scanTestRuns(resultsDir), []);
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});

evals/packages/test-artifacts/test/test-run-compat.test.ts

@@ -0,0 +1,112 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import type { ScreenshotArtifact } from "../../test-evidence/src/screenshot.ts";
+import { createTestEvidence } from "../../test-evidence/src/test-evidence.ts";
+import { renderArtifactIndexHtml, renderPrMarkdown } from "../src/render.ts";
+import { scanTestRuns } from "../src/scan.ts";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+test("scanTestRuns reads screenshots and assertions produced by the real test-evidence writer", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-compat-"));
+  const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-writer-compat");
+  try {
+    const png = Buffer.from("synthetic screenshot pixels");
+    const screenshotArtifact: ScreenshotArtifact = {
+      png,
+      hash: createHash("sha256").update(png).digest("hex"),
+      route: "#/writer-compat",
+      visibleText: "Writer compatibility",
+      at: "2026-07-02T10:00:00.000Z",
+    };
+    const testEvidence = createTestEvidence({ name: "Writer compatibility", outDir: testRunDir });
+    testEvidence.recordScreenshot(screenshotArtifact);
+    testEvidence.recordVisualValidation(screenshotArtifact.hash, {
+      ok: true,
+      description: "Writer compatibility is visible.",
+      results: [{ expectation: "Writer compatibility screenshot", passed: true, evidence: "Writer compatibility" }],
+      why: "",
+      model: "compat-model",
+      cached: false,
+    });
+    testEvidence.recordAssertionEvidence("Witness compatibility", "The API returned HTTP 200.", true);
+    testEvidence.recordJsonArtifact("Writer state", { status: "ready" });
+    await testEvidence.close();
+
+    const raw = await readFile(join(testRunDir, "test-run.json"), "utf8");
+    assert.match(raw, /"gitSha":/);
+    assert.match(raw, /"artifacts":/);
+    const entries = await scanTestRuns(resultsDir);
+    assert.equal(entries.length, 1);
+    const entry = entries[0];
+    assert.equal(entry?.kind, "test-run");
+    if (!entry || entry.kind !== "test-run") throw new Error("Writer test run was not scanned.");
+    assert.equal(entry.testRun.artifacts[0]?.caption, "Writer compatibility screenshot");
+    assert.equal(entry.testRun.artifacts[1]?.fileName, "");
+    assert.equal(entry.testRun.artifacts.length, 2);
+    assert.equal(entry.format, "current");
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});
+
+test("scan and render retain pending judgments from a deferred test-evidence record", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-pending-"));
+  const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-pending");
+  try {
+    const png = Buffer.from("pending screenshot pixels");
+    const screenshotArtifact: ScreenshotArtifact = {
+      png,
+      hash: createHash("sha256").update(png).digest("hex"),
+      route: "#/pending",
+      visibleText: "Pending visual evidence",
+      at: "2026-07-02T10:00:00.000Z",
+    };
+    const testEvidence = createTestEvidence({ name: "Pending writer compatibility", outDir: testRunDir });
+    testEvidence.recordScreenshot(screenshotArtifact);
+    testEvidence.recordVisualValidation(screenshotArtifact.hash, {
+      ok: true,
+      description: "",
+      results: [],
+      why: "vision judgment deferred",
+      model: "compat-model",
+      cached: false,
+      deferred: true,
+      pendingExpectations: ["Pending writer expectation"],
+    });
+    await testEvidence.close();
+
+    const testRunPath = join(testRunDir, "test-run.json");
+    const value: unknown = JSON.parse(await readFile(testRunPath, "utf8"));
+    if (!isRecord(value) || !isRecord(value.summary)) throw new Error("Writer did not produce a test-run summary.");
+    delete value.summary.pendingArtifacts;
+    delete value.summary.pendingJudgments;
+    await writeFile(testRunPath, JSON.stringify(value));
+
+    const entries = await scanTestRuns(resultsDir);
+    const entry = entries[0];
+    if (!entry || entry.kind !== "test-run") throw new Error("Pending writer test run was not scanned.");
+    assert.equal(entry.testRun.summary.pendingArtifacts, 1);
+    assert.equal(entry.testRun.summary.pendingJudgments, 1);
+    assert.deepEqual(entry.testRun.artifacts[0]?.judgments, [{
+      expectation: "Pending writer expectation",
+      state: "pending",
+      reasoning: "vision judgment deferred",
+    }]);
+
+    const index = renderArtifactIndexHtml(entries);
+    const markdown = renderPrMarkdown(entry.testRun, {});
+    assert.match(index, /PENDING/);
+    assert.match(markdown, /⏳ PENDING/);
+    assert.match(markdown, /⏳ \*\*PENDING\*\* Pending writer expectation — vision judgment deferred/);
+    assert.doesNotMatch(markdown, /no visual expectations recorded/);
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});