源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/packages/test-artifacts/test/publish-pr.test.ts

@@ -0,0 +1,164 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { publishPr } from "../src/publish-pr.ts";
+import type { CommandRunner, Fetcher } from "../src/publish-pr.ts";
+import type { TestRunRecord } from "../src/schema.ts";
+
+const TEST_RUN_SHA = "1111111111111111111111111111111111111111";
+
+function testRunRecord(dir: string): TestRunRecord {
+  return {
+    name: "Publication proof",
+    dir,
+    createdAt: "2026-07-02T10:00:00.000Z",
+    closedAt: "2026-07-02T10:01:00.000Z",
+    gitSha: TEST_RUN_SHA,
+    branch: "feat/proof",
+    summary: {
+      ok: true,
+      totalArtifacts: 1,
+      passedArtifacts: 1,
+      failedArtifacts: 0,
+      unvalidatedArtifacts: 0,
+      pendingArtifacts: 0,
+      passedExpectations: 1,
+      failedExpectations: 0,
+      pendingJudgments: 0,
+    },
+    artifacts: [{
+      caption: "Published validation",
+      fileName: "01-published.png",
+      hash: "hash",
+      route: "#/published",
+      at: "2026-07-02T10:00:00.000Z",
+      description: "Visible state",
+      model: "test-model",
+      ok: true,
+      results: [{ expectation: "State is visible", passed: true, evidence: "Visible" }],
+      judgments: [{ expectation: "State is visible", state: "passed", reasoning: "Visible" }],
+    }],
+  };
+}
+
+test("publishPr dry-run emits the new marker without upload or gh calls", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-publish-"));
+  try {
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    let commandCalled = false;
+    let fetchCalled = false;
+    let output = "";
+    const exec: CommandRunner = () => {
+      commandCalled = true;
+      return { status: 1, stdout: "", stderr: "unexpected command" };
+    };
+    const fetcher: Fetcher = async () => {
+      fetchCalled = true;
+      throw new Error("unexpected fetch");
+    };
+    const result = await publishPr(
+      { testRunDir, dryRun: true },
+      { exec, fetch: fetcher, stdout: (markdown) => { output = markdown; } },
+    );
+    assert.equal(commandCalled, false);
+    assert.equal(fetchCalled, false);
+    assert.equal(result.posted, false);
+    assert.match(output, /<!-- test-evidence -->/);
+    assert.match(output, /Dry run: screenshots were not uploaded/);
+  } finally {
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr recognizes an old sticky marker and uploads under the test-artifacts prefix", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-current-"));
+  const previousToken = process.env.BLOB_READ_WRITE_TOKEN;
+  try {
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    await writeFile(join(testRunDir, "01-published.png"), Buffer.from("regular png"));
+    process.env.BLOB_READ_WRITE_TOKEN = "test-token";
+    const uploads: string[] = [];
+    let patchedBody = "";
+    const fetcher: Fetcher = async (input) => {
+      uploads.push(String(input));
+      return new Response(JSON.stringify({ url: "https://example.test/published.png" }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    };
+    const exec: CommandRunner = (_command, args, opts) => {
+      if (args.includes("headRefOid")) return { status: 0, stdout: JSON.stringify({ headRefOid: TEST_RUN_SHA }), stderr: "" };
+      if (args.includes("comments")) {
+        return { status: 0, stdout: JSON.stringify({ comments: [{ databaseId: 77, body: "<!-- photo-roll --> old" }] }), stderr: "" };
+      }
+      patchedBody = opts?.input ?? "";
+      return { status: 0, stdout: "updated", stderr: "" };
+    };
+    const result = await publishPr({ pr: 17, testRunDir }, { exec, fetch: fetcher });
+    assert.equal(result.updated, true);
+    assert.equal(uploads.length, 1);
+    assert.match(uploads[0] ?? "", /\/test-artifacts\/[^/]+\/01-published\.png$/);
+    assert.match(patchedBody, /<!-- test-evidence -->/);
+    assert.doesNotMatch(patchedBody, /<!-- photo-roll -->/);
+  } finally {
+    if (previousToken === undefined) delete process.env.BLOB_READ_WRITE_TOKEN;
+    else process.env.BLOB_READ_WRITE_TOKEN = previousToken;
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr reads persisted legacy roll.json input", async () => {
+  const testRunDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-legacy-"));
+  try {
+    const current = testRunRecord(testRunDir);
+    await writeFile(join(testRunDir, "roll.json"), JSON.stringify({
+      ...current,
+      summary: {
+        ok: true,
+        totalFrames: 1,
+        passedFrames: 1,
+        failedFrames: 0,
+        unvalidatedFrames: 0,
+        passedExpectations: 1,
+        failedExpectations: 0,
+      },
+      frames: current.artifacts,
+      artifacts: undefined,
+    }));
+    let output = "";
+    await publishPr({ testRunDir, dryRun: true }, { stdout: (markdown) => { output = markdown; } });
+    assert.match(output, /evals\/results\/rolls\/.*\/roll\.json/);
+    assert.match(output, /<!-- test-evidence -->/);
+  } finally {
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});
+
+test("publishPr refuses a symlinked screenshot before upload", async () => {
+  const root = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-symlink-"));
+  const testRunDir = join(root, "test-run");
+  const previousToken = process.env.BLOB_READ_WRITE_TOKEN;
+  try {
+    await mkdir(testRunDir);
+    await writeFile(join(testRunDir, "test-run.json"), JSON.stringify(testRunRecord(testRunDir)));
+    const outside = join(root, "private-key");
+    await writeFile(outside, "private material");
+    await symlink(outside, join(testRunDir, "01-published.png"));
+    process.env.BLOB_READ_WRITE_TOKEN = "test-token";
+    const exec: CommandRunner = (_command, args) => ({
+      status: 0,
+      stdout: args.includes("headRefOid") ? JSON.stringify({ headRefOid: TEST_RUN_SHA }) : "",
+      stderr: "",
+    });
+    await assert.rejects(
+      () => publishPr({ pr: 17, testRunDir }, { exec }),
+      /Refusing to upload non-regular or symlinked test artifact: 01-published\.png/,
+    );
+  } finally {
+    if (previousToken === undefined) delete process.env.BLOB_READ_WRITE_TOKEN;
+    else process.env.BLOB_READ_WRITE_TOKEN = previousToken;
+    await rm(root, { recursive: true, force: true });
+  }
+});

evals/packages/test-artifacts/test/scan-render.test.ts

@@ -0,0 +1,119 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { renderArtifactIndexHtml, renderPrMarkdown } from "../src/render.ts";
+import { scanTestRuns } from "../src/scan.ts";
+import type { TestRunRecord } from "../src/schema.ts";
+
+function record(name: string, dir: string, createdAt: string, passed: boolean): TestRunRecord {
+  return {
+    name,
+    dir,
+    createdAt,
+    closedAt: createdAt,
+    summary: {
+      ok: passed,
+      totalArtifacts: 1,
+      passedArtifacts: passed ? 1 : 0,
+      failedArtifacts: passed ? 0 : 1,
+      unvalidatedArtifacts: 0,
+      pendingArtifacts: 0,
+      passedExpectations: passed ? 1 : 0,
+      failedExpectations: passed ? 0 : 1,
+      pendingJudgments: 0,
+    },
+    artifacts: [{
+      caption: `${name} first validation`,
+      fileName: "01-first.png",
+      hash: `${name}-hash`,
+      route: "#/workspace/test",
+      at: createdAt,
+      description: `${name} description`,
+      model: "test-model",
+      ok: passed,
+      results: [{ expectation: `${name} expectation`, passed, evidence: `${name} evidence` }],
+      judgments: [{ expectation: `${name} expectation`, state: passed ? "passed" : "failed", reasoning: `${name} evidence` }],
+    }],
+  };
+}
+
+function legacyRecord(testRun: TestRunRecord): Record<string, unknown> {
+  const frames = testRun.artifacts.map((artifact) => {
+    const { judgments, ...legacyArtifact } = artifact;
+    void judgments;
+    return legacyArtifact;
+  });
+  return {
+    ...testRun,
+    summary: {
+      ok: testRun.summary.ok,
+      totalFrames: testRun.summary.totalArtifacts,
+      passedFrames: testRun.summary.passedArtifacts,
+      failedFrames: testRun.summary.failedArtifacts,
+      unvalidatedFrames: testRun.summary.unvalidatedArtifacts,
+      passedExpectations: testRun.summary.passedExpectations,
+      failedExpectations: testRun.summary.failedExpectations,
+    },
+    frames,
+    artifacts: undefined,
+  };
+}
+
+test("scanTestRuns reads current and persisted legacy results while tolerating corrupt data", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-scan-"));
+  try {
+    const currentDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-current");
+    const oldDir = join(resultsDir, "rolls", "2026-07-01T10-00-00-000Z-old");
+    const corruptDir = join(resultsDir, "test-runs", "2026-07-03T10-00-00-000Z-corrupt");
+    const looseDir = join(resultsDir, "2026-06-01T10-00-00-000Z-legacy");
+    await Promise.all([currentDir, oldDir, corruptDir, looseDir].map((directory) => mkdir(directory, { recursive: true })));
+    await Promise.all([
+      writeFile(join(currentDir, "test-run.json"), JSON.stringify(record("Current", currentDir, "2026-07-02T10:00:00.000Z", false))),
+      writeFile(join(oldDir, "roll.json"), JSON.stringify(legacyRecord(record("Persisted", oldDir, "2026-07-01T10:00:00.000Z", true)))),
+      writeFile(join(corruptDir, "test-run.json"), "{not-json"),
+      writeFile(join(looseDir, "fraimz.html"), "legacy"),
+      writeFile(join(looseDir, "frame.png"), Buffer.from("legacy-png")),
+    ]);
+
+    const entries = await scanTestRuns(resultsDir);
+    assert.deepEqual(entries.map((entry) => entry.name), ["Current", "Persisted", "2026-06-01T10-00-00-000Z-legacy"]);
+    const storedEntries = entries.filter((entry) => entry.kind === "test-run");
+    assert.deepEqual(storedEntries.map((entry) => entry.format), ["current", "legacy"]);
+    assert.deepEqual(storedEntries[1]?.testRun.artifacts[0]?.judgments, [{
+      expectation: "Persisted expectation",
+      state: "passed",
+      reasoning: "Persisted evidence",
+    }]);
+
+    const rendered = renderArtifactIndexHtml(entries);
+    for (const expected of ["Current first validation", "FAILED", "PASSED", "current/index.html", "../rolls/", "fraimz.html"]) {
+      assert.match(rendered, new RegExp(expected));
+    }
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});
+
+test("renderPrMarkdown writes only the new test-evidence marker", () => {
+  const testRun = record("PR proof", "/tmp/2026-pr-proof", "2026-07-02T10:00:00.000Z", false);
+  const markdown = renderPrMarkdown(testRun, { "01-first.png": "https://example.test/01-first.png" });
+  assert.match(markdown, /<!-- test-evidence -->/);
+  assert.doesNotMatch(markdown, /<!-- photo-roll -->|<!-- fraimz -->/);
+  assert.match(markdown, /PR proof first validation/);
+});
+
+test("scanTestRuns skips a symlinked test-run.json", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-symlink-"));
+  try {
+    const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-symlinked");
+    await mkdir(testRunDir, { recursive: true });
+    const outside = join(resultsDir, "outside-test-run.json");
+    await writeFile(outside, JSON.stringify(record("Outside", testRunDir, "2026-07-02T10:00:00.000Z", true)));
+    await symlink(outside, join(testRunDir, "test-run.json"));
+    assert.deepEqual(await scanTestRuns(resultsDir), []);
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});

evals/packages/test-artifacts/test/test-run-compat.test.ts

@@ -0,0 +1,112 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import type { ScreenshotArtifact } from "../../test-evidence/src/screenshot.ts";
+import { createTestEvidence } from "../../test-evidence/src/test-evidence.ts";
+import { renderArtifactIndexHtml, renderPrMarkdown } from "../src/render.ts";
+import { scanTestRuns } from "../src/scan.ts";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+test("scanTestRuns reads screenshots and assertions produced by the real test-evidence writer", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-compat-"));
+  const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-writer-compat");
+  try {
+    const png = Buffer.from("synthetic screenshot pixels");
+    const screenshotArtifact: ScreenshotArtifact = {
+      png,
+      hash: createHash("sha256").update(png).digest("hex"),
+      route: "#/writer-compat",
+      visibleText: "Writer compatibility",
+      at: "2026-07-02T10:00:00.000Z",
+    };
+    const testEvidence = createTestEvidence({ name: "Writer compatibility", outDir: testRunDir });
+    testEvidence.recordScreenshot(screenshotArtifact);
+    testEvidence.recordVisualValidation(screenshotArtifact.hash, {
+      ok: true,
+      description: "Writer compatibility is visible.",
+      results: [{ expectation: "Writer compatibility screenshot", passed: true, evidence: "Writer compatibility" }],
+      why: "",
+      model: "compat-model",
+      cached: false,
+    });
+    testEvidence.recordAssertionEvidence("Witness compatibility", "The API returned HTTP 200.", true);
+    testEvidence.recordJsonArtifact("Writer state", { status: "ready" });
+    await testEvidence.close();
+
+    const raw = await readFile(join(testRunDir, "test-run.json"), "utf8");
+    assert.match(raw, /"gitSha":/);
+    assert.match(raw, /"artifacts":/);
+    const entries = await scanTestRuns(resultsDir);
+    assert.equal(entries.length, 1);
+    const entry = entries[0];
+    assert.equal(entry?.kind, "test-run");
+    if (!entry || entry.kind !== "test-run") throw new Error("Writer test run was not scanned.");
+    assert.equal(entry.testRun.artifacts[0]?.caption, "Writer compatibility screenshot");
+    assert.equal(entry.testRun.artifacts[1]?.fileName, "");
+    assert.equal(entry.testRun.artifacts.length, 2);
+    assert.equal(entry.format, "current");
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});
+
+test("scan and render retain pending judgments from a deferred test-evidence record", async () => {
+  const resultsDir = await mkdtemp(join(tmpdir(), "openwork-test-artifacts-pending-"));
+  const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-pending");
+  try {
+    const png = Buffer.from("pending screenshot pixels");
+    const screenshotArtifact: ScreenshotArtifact = {
+      png,
+      hash: createHash("sha256").update(png).digest("hex"),
+      route: "#/pending",
+      visibleText: "Pending visual evidence",
+      at: "2026-07-02T10:00:00.000Z",
+    };
+    const testEvidence = createTestEvidence({ name: "Pending writer compatibility", outDir: testRunDir });
+    testEvidence.recordScreenshot(screenshotArtifact);
+    testEvidence.recordVisualValidation(screenshotArtifact.hash, {
+      ok: true,
+      description: "",
+      results: [],
+      why: "vision judgment deferred",
+      model: "compat-model",
+      cached: false,
+      deferred: true,
+      pendingExpectations: ["Pending writer expectation"],
+    });
+    await testEvidence.close();
+
+    const testRunPath = join(testRunDir, "test-run.json");
+    const value: unknown = JSON.parse(await readFile(testRunPath, "utf8"));
+    if (!isRecord(value) || !isRecord(value.summary)) throw new Error("Writer did not produce a test-run summary.");
+    delete value.summary.pendingArtifacts;
+    delete value.summary.pendingJudgments;
+    await writeFile(testRunPath, JSON.stringify(value));
+
+    const entries = await scanTestRuns(resultsDir);
+    const entry = entries[0];
+    if (!entry || entry.kind !== "test-run") throw new Error("Pending writer test run was not scanned.");
+    assert.equal(entry.testRun.summary.pendingArtifacts, 1);
+    assert.equal(entry.testRun.summary.pendingJudgments, 1);
+    assert.deepEqual(entry.testRun.artifacts[0]?.judgments, [{
+      expectation: "Pending writer expectation",
+      state: "pending",
+      reasoning: "vision judgment deferred",
+    }]);
+
+    const index = renderArtifactIndexHtml(entries);
+    const markdown = renderPrMarkdown(entry.testRun, {});
+    assert.match(index, /PENDING/);
+    assert.match(markdown, /⏳ PENDING/);
+    assert.match(markdown, /⏳ \*\*PENDING\*\* Pending writer expectation — vision judgment deferred/);
+    assert.doesNotMatch(markdown, /no visual expectations recorded/);
+  } finally {
+    await rm(resultsDir, { recursive: true, force: true });
+  }
+});