源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/packages/test-evidence/test/ambient.test.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { currentTestEvidence, withTestEvidence } from "../src/ambient.ts";
+import type { ScreenshotArtifact } from "../src/screenshot.ts";
+import { createTestEvidence } from "../src/test-evidence.ts";
+
+function screenshotArtifact(contents: string): ScreenshotArtifact {
+  const png = Buffer.from(contents);
+  return {
+    png,
+    hash: createHash("sha256").update(png).digest("hex"),
+    route: `#/${contents}`,
+    visibleText: contents,
+    at: "2026-08-02T12:00:00.000Z",
+  };
+}
+
+function pause(milliseconds: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+async function hashes(dir: string): Promise<unknown[]> {
+  const value: unknown = JSON.parse(await readFile(join(dir, "test-run.json"), "utf8"));
+  assert.ok(typeof value === "object" && value !== null && !Array.isArray(value));
+  const artifacts = Reflect.get(value, "artifacts");
+  assert.ok(Array.isArray(artifacts));
+  return artifacts.map((artifact) => typeof artifact === "object" && artifact !== null ? Reflect.get(artifact, "hash") : null);
+}
+
+test("concurrent withTestEvidence scopes do not cross-record", async () => {
+  const root = await mkdtemp(join(tmpdir(), "openwork-ambient-"));
+  const firstDir = join(root, "first");
+  const secondDir = join(root, "second");
+  try {
+    const first = createTestEvidence({ name: "first", outDir: firstDir });
+    const second = createTestEvidence({ name: "second", outDir: secondDir });
+    const firstScreenshot = screenshotArtifact("first screenshot");
+    const secondScreenshot = screenshotArtifact("second screenshot");
+    assert.equal(currentTestEvidence(), null);
+
+    await Promise.all([
+      withTestEvidence(first, async () => {
+        await pause(15);
+        const active = currentTestEvidence();
+        assert.equal(active, first);
+        active.recordScreenshot(firstScreenshot);
+      }),
+      withTestEvidence(second, async () => {
+        const active = currentTestEvidence();
+        assert.equal(active, second);
+        active.recordScreenshot(secondScreenshot);
+        await pause(25);
+        assert.equal(currentTestEvidence(), second);
+      }),
+    ]);
+
+    assert.equal(currentTestEvidence(), null);
+    await Promise.all([first.close(), second.close()]);
+    assert.deepEqual(await hashes(firstDir), [firstScreenshot.hash]);
+    assert.deepEqual(await hashes(secondDir), [secondScreenshot.hash]);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

evals/packages/test-evidence/test/judge-cli.test.ts

@@ -0,0 +1,33 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+import { createTestEvidence } from "../src/test-evidence.ts";
+
+const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
+const testRunsDir = join(repoRoot, "evals", "results", "test-runs");
+const judgeCli = fileURLToPath(new URL("../bin/test-evidence-judge.mjs", import.meta.url));
+
+test("judge CLI resolves a completed test run by record name", async () => {
+  await mkdir(testRunsDir, { recursive: true });
+  const testRunDir = await mkdtemp(join(testRunsDir, "judge-cli-name-"));
+  const name = `Judge CLI name ${randomUUID()}`;
+  try {
+    const evidence = createTestEvidence({ name, outDir: testRunDir });
+    evidence.recordAssertionEvidence("No visual judgment is pending", "The test run contains assertion evidence only.", true);
+    await evidence.close();
+
+    const result = spawnSync(process.execPath, [judgeCli, "--test-run", name], {
+      cwd: repoRoot,
+      encoding: "utf8",
+    });
+    assert.equal(result.status, 0, result.stderr || result.stdout);
+    assert.match(result.stdout, /Judged 0 visual validation\(s\): 0 failed, 0 pending\./);
+    assert.match(result.stdout, new RegExp(`${testRunDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/test-run\\.json`));
+  } finally {
+    await rm(testRunDir, { recursive: true, force: true });
+  }
+});

evals/packages/test-evidence/test/test-evidence.test.ts

@@ -0,0 +1,198 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import type { CdpClient, Surface } from "@openwork/cdp";
+import { withTestEvidence } from "../src/ambient.ts";
+import { screenshot } from "../src/screenshot.ts";
+import type { ScreenshotArtifact } from "../src/screenshot.ts";
+import { createTestEvidence } from "../src/test-evidence.ts";
+import type { VisualEvidenceResult } from "../src/validate.ts";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function screenshotArtifact(contents: string): ScreenshotArtifact {
+  const png = Buffer.from(contents);
+  return {
+    png,
+    hash: createHash("sha256").update(png).digest("hex"),
+    route: `#/${contents}`,
+    visibleText: contents,
+    at: "2026-08-02T12:00:00.000Z",
+  };
+}
+
+function seen(expectation: string, passed: boolean, evidence = `${expectation} evidence`): VisualEvidenceResult {
+  return {
+    ok: passed,
+    description: `Description for ${expectation}`,
+    results: [{ expectation, passed, evidence }],
+    why: passed ? "" : `${expectation} failed`,
+    model: "injected-vision-model",
+    cached: false,
+  };
+}
+
+async function payload(dir: string): Promise<Record<string, unknown>> {
+  const value: unknown = JSON.parse(await readFile(join(dir, "test-run.json"), "utf8"));
+  assert.ok(isRecord(value));
+  return value;
+}
+
+test("test evidence writes visual validations, assertions, failures, and unvalidated screenshots", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-"));
+  try {
+    const testEvidence = createTestEvidence({ name: "body cam", outDir: dir });
+    const passing = screenshotArtifact("passing");
+    const failing = screenshotArtifact("failing");
+    testEvidence.recordScreenshot(passing);
+    testEvidence.recordScreenshot(failing);
+    testEvidence.recordScreenshot(screenshotArtifact("unvalidated"));
+    testEvidence.recordVisualValidation(passing.hash, seen("Passing screenshot", true));
+    testEvidence.recordVisualValidation(failing.hash, seen("Failing screenshot", false));
+    testEvidence.recordAssertionEvidence("API returned success", "HTTP 200", true);
+    await testEvidence.close();
+
+    const testRun = await payload(dir);
+    assert.deepEqual(testRun.summary, {
+      ok: false,
+      totalArtifacts: 4,
+      passedArtifacts: 2,
+      failedArtifacts: 1,
+      unvalidatedArtifacts: 1,
+      pendingArtifacts: 0,
+      passedExpectations: 2,
+      failedExpectations: 1,
+      pendingJudgments: 0,
+    });
+    assert.ok(Array.isArray(testRun.artifacts));
+    assert.deepEqual(
+      testRun.artifacts.map((artifact) => isRecord(artifact) ? artifact.caption : null),
+      ["Passing screenshot", "Failing screenshot", "API returned success", "body cam artifact 3"],
+    );
+    const failedArtifact = testRun.artifacts[1];
+    const assertionArtifact = testRun.artifacts[2];
+    assert.ok(isRecord(failedArtifact));
+    assert.equal(failedArtifact.ok, false);
+    assert.ok(isRecord(assertionArtifact));
+    assert.equal(assertionArtifact.fileName, "");
+    await stat(join(dir, "01-passing-screenshot.png"));
+    await stat(join(dir, "02-failing-screenshot.png"));
+    await stat(join(dir, "03-body-cam-artifact-3.png"));
+
+    const index = await readFile(join(dir, "index.html"), "utf8");
+    assert.match(index, /API returned success/);
+    assert.match(index, /unvalidated artifacts \(1\)/);
+    assert.doesNotMatch(index, /<img src=""/);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("test evidence writes a JSON artifact and lists it in the test run", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-json-"));
+  try {
+    const testEvidence = createTestEvidence({ name: "world evidence", outDir: dir });
+    testEvidence.recordJsonArtifact("world-snapshot primary", { version: 1, name: "primary" });
+    await testEvidence.close();
+
+    assert.deepEqual(JSON.parse(await readFile(join(dir, "01-world-snapshot-primary.json"), "utf8")), {
+      version: 1,
+      name: "primary",
+    });
+    const testRun = await payload(dir);
+    assert.deepEqual(testRun.artifacts, [{
+      kind: "json",
+      label: "world-snapshot primary",
+      fileName: "01-world-snapshot-primary.json",
+    }]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("test evidence accepts unchanged screenshots and only lets one validation use their pixel hash", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-retake-"));
+  try {
+    const testEvidence = createTestEvidence({ name: "retakes", outDir: dir });
+    const duplicate = screenshotArtifact("same pixels");
+    testEvidence.recordScreenshot(duplicate);
+    testEvidence.recordScreenshot(duplicate);
+    testEvidence.recordVisualValidation(duplicate.hash, seen("Same validation", false, "first judgment"));
+    testEvidence.recordVisualValidation(duplicate.hash, seen("Same validation", true, "replacement judgment"));
+    assert.throws(
+      () => testEvidence.recordVisualValidation(duplicate.hash, seen("Different validation", true)),
+      /Different validation.*different visual validation.*Same validation/i,
+    );
+    await testEvidence.close();
+
+    const testRun = await payload(dir);
+    assert.deepEqual(testRun.summary, {
+      ok: false,
+      totalArtifacts: 2,
+      passedArtifacts: 1,
+      failedArtifacts: 0,
+      unvalidatedArtifacts: 1,
+      pendingArtifacts: 0,
+      passedExpectations: 1,
+      failedExpectations: 0,
+      pendingJudgments: 0,
+    });
+    assert.ok(Array.isArray(testRun.artifacts));
+    const validated = testRun.artifacts[0];
+    assert.ok(isRecord(validated));
+    assert.equal(validated.ok, true);
+    assert.ok(Array.isArray(validated.results));
+    assert.deepEqual(validated.results[0], {
+      expectation: "Same validation",
+      passed: true,
+      evidence: "replacement judgment",
+    });
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("screenshot automatically records an artifact in ambient test evidence", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-screenshot-"));
+  try {
+    const png = Buffer.from("ambient screenshot pixels");
+    const client: CdpClient = {
+      close() {},
+      async send(method) {
+        if (method === "Page.captureScreenshot") return { data: png.toString("base64") };
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { route: "#/ambient", visibleText: "Ambient screenshot" } } };
+        }
+        throw new Error(`Unexpected CDP method: ${method}`);
+      },
+    };
+    const app: Surface = {
+      handle: { name: "fake", kind: "chrome", hostKind: "test", cdpUrl: "http://127.0.0.1" },
+      client,
+    };
+    const testEvidence = createTestEvidence({ name: "ambient screenshot", outDir: dir });
+    const captured = await withTestEvidence(testEvidence, () => screenshot(app));
+    assert.equal(captured.route, "#/ambient");
+    await testEvidence.close();
+
+    const testRun = await payload(dir);
+    assert.deepEqual(testRun.summary, {
+      ok: false,
+      totalArtifacts: 1,
+      passedArtifacts: 0,
+      failedArtifacts: 0,
+      unvalidatedArtifacts: 1,
+      pendingArtifacts: 0,
+      passedExpectations: 0,
+      failedExpectations: 0,
+      pendingJudgments: 0,
+    });
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});

evals/packages/test-evidence/test/validate.test.ts

@@ -0,0 +1,182 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+import { withTestEvidence } from "../src/ambient.ts";
+import { createTestEvidence, judgeTestRun } from "../src/test-evidence.ts";
+import { validate } from "../src/validate.ts";
+import type { VisionRequest } from "../src/validate.ts";
+import type { ScreenshotArtifact } from "../src/screenshot.ts";
+
+function testScreenshot(hash: string): ScreenshotArtifact {
+  return {
+    png: Buffer.from("canned png"),
+    hash,
+    route: "#/workspace/ws_test/session",
+    visibleText: "OpenWork composer",
+    at: "2026-07-29T12:00:00.000Z",
+  };
+}
+
+test("validate describes, matches, computes failures, and caches", async () => {
+  const unique = randomUUID();
+  const expectations = [`Composer is visible ${unique}`, `No error is visible ${unique}`];
+  const requests: VisionRequest[] = [];
+  const ask = async (request: VisionRequest): Promise<string> => {
+    requests.push(request);
+    if (requests.length === 1) return JSON.stringify({ description: "A composer and an error banner are visible." });
+    return JSON.stringify({
+      results: [
+        { expectation: expectations[0], passed: true, evidence: "The composer is centered on screen." },
+        { expectation: expectations[1], passed: false, evidence: "An error banner is visible." },
+      ],
+    });
+  };
+
+  const first = await validate(testScreenshot(unique), expectations, { ask });
+  assert.equal(first.cached, false);
+  assert.equal(first.ok, false);
+  assert.equal(first.description, "A composer and an error banner are visible.");
+  assert.equal(first.results[0]?.passed, true);
+  assert.equal(first.results[1]?.passed, false);
+  assert.match(first.why, /No error is visible/);
+  assert.match(first.why, /An error banner is visible/);
+  assert.equal(requests.length, 2);
+  assert.equal(requests[0]?.prompt.includes(unique), false, "description request must not know the expectations");
+  assert.equal(requests[1]?.prompt.includes(unique), true);
+
+  const second = await validate(testScreenshot(unique), expectations, { ask });
+  assert.equal(second.cached, true);
+  assert.equal(second.ok, false);
+  assert.equal(requests.length, 2, "cache hit must not call the transport");
+});
+
+test("validate rejects malformed model verdicts clearly", async () => {
+  let calls = 0;
+  const ask = async (): Promise<string> => {
+    calls += 1;
+    return calls === 1 ? JSON.stringify({ description: "A visible app window." }) : "not-json";
+  };
+  await assert.rejects(
+    () => validate(testScreenshot(randomUUID()), [`Visible app ${randomUUID()}`], { ask }),
+    /Vision model verdict response was not valid JSON/,
+  );
+});
+
+test("defer mode judges a caller-provided deterministic witness inline", async () => {
+  const previousMode = process.env.OPENWORK_EVAL_VISION;
+  try {
+    process.env.OPENWORK_EVAL_VISION = "defer";
+    const expectation = `Synthetic screenshot is visible ${randomUUID()}`;
+    let calls = 0;
+    const visualEvidence = await validate(testScreenshot(randomUUID()), [expectation], {
+      bypassCache: true,
+      ask: async () => {
+        calls += 1;
+        return calls === 1
+          ? JSON.stringify({ description: "A synthetic screenshot is visible." })
+          : JSON.stringify({ results: [{ expectation, passed: false, evidence: "The expected detail is absent." }] });
+      },
+    });
+
+    assert.equal(visualEvidence.ok, false);
+    assert.equal(visualEvidence.results[0]?.passed, false);
+    assert.equal(visualEvidence.deferred, undefined);
+    assert.equal(calls, 2);
+  } finally {
+    if (previousMode === undefined) delete process.env.OPENWORK_EVAL_VISION;
+    else process.env.OPENWORK_EVAL_VISION = previousMode;
+  }
+});
+
+test("deferred validation records pending expectations that the judge resolves", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-deferred-vision-"));
+  const previousMode = process.env.OPENWORK_EVAL_VISION;
+  try {
+    const expectation = `Synthetic screenshot is visible ${randomUUID()}`;
+    const screenshot = testScreenshot(randomUUID());
+    const testEvidence = createTestEvidence({ name: "deferred vision", outDir: dir });
+    testEvidence.recordScreenshot(screenshot);
+    process.env.OPENWORK_EVAL_VISION = "defer";
+    const deferred = await withTestEvidence(testEvidence, () => validate(screenshot, [expectation], { bypassCache: true }));
+    assert.equal(deferred.ok, true);
+    assert.equal(deferred.deferred, true);
+    assert.equal(deferred.why, "vision judgment deferred");
+    await testEvidence.close();
+
+    const pendingTestRun = await readFile(join(dir, "test-run.json"), "utf8");
+    assert.match(pendingTestRun, /"state": "pending"/);
+    assert.match(pendingTestRun, /"pendingArtifacts": 1/);
+    assert.match(pendingTestRun, /"pendingJudgments": 1/);
+
+    const providerError = await judgeTestRun(dir, {
+      ask: async () => {
+        throw new Error("stub provider unavailable");
+      },
+    });
+    assert.equal(providerError.judgedValidations, 0);
+    assert.equal(providerError.pendingValidations, 1);
+    assert.match(providerError.errors[0], /stub provider unavailable/);
+
+    const requests: VisionRequest[] = [];
+    const judged = await judgeTestRun(dir, {
+      ask: async (request) => {
+        requests.push(request);
+        return requests.length === 1
+          ? JSON.stringify({ description: "A synthetic screenshot is visible." })
+          : JSON.stringify({ results: [{ expectation, passed: true, evidence: "The synthetic screenshot is present." }] });
+      },
+    });
+    assert.equal(judged.judgedValidations, 1);
+    assert.equal(judged.failedValidations, 0);
+    assert.equal(judged.pendingValidations, 0);
+    assert.equal(requests.length, 2);
+
+    const resolvedTestRun = await readFile(join(dir, "test-run.json"), "utf8");
+    assert.match(resolvedTestRun, /"state": "passed"/);
+    assert.match(resolvedTestRun, /"reasoning": "The synthetic screenshot is present\."/);
+    assert.match(resolvedTestRun, /"pendingJudgments": 0/);
+
+    let idempotentCalls = 0;
+    const unchanged = await judgeTestRun(dir, {
+      ask: async () => {
+        idempotentCalls += 1;
+        throw new Error("resolved validations must not be re-judged");
+      },
+    });
+    assert.equal(unchanged.judgedValidations, 0);
+    assert.equal(unchanged.failedValidations, 0);
+    assert.equal(idempotentCalls, 0);
+
+    const cli = spawnSync(process.execPath, [
+      fileURLToPath(new URL("../bin/test-evidence-judge.mjs", import.meta.url)),
+      "--test-run",
+      dir,
+    ], { encoding: "utf8" });
+    assert.equal(cli.status, 0, cli.stderr);
+    assert.match(cli.stdout, /Judged 0 visual validation\(s\): 0 failed, 0 pending\./);
+
+    let forcedCalls = 0;
+    const forced = await judgeTestRun(dir, {
+      force: true,
+      ask: async (request) => {
+        forcedCalls += 1;
+        return forcedCalls === 1
+          ? JSON.stringify({ description: "A synthetic screenshot is visible." })
+          : JSON.stringify({ results: [{ expectation, passed: false, evidence: "The expected detail is absent." }] });
+      },
+    });
+    assert.equal(forced.judgedValidations, 1);
+    assert.equal(forced.failedValidations, 1);
+    assert.equal(forced.pendingValidations, 0);
+    assert.equal(forcedCalls, 2, "--force must bypass the cached judgment");
+  } finally {
+    if (previousMode === undefined) delete process.env.OPENWORK_EVAL_VISION;
+    else process.env.OPENWORK_EVAL_VISION = previousMode;
+    await rm(dir, { recursive: true, force: true });
+  }
+});

evals/packages/test-evidence/test/visual-evidence.test.ts

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { createVisualEvidence } from "../src/visual-evidence.ts";
+import type { ScreenshotArtifact } from "../src/screenshot.ts";
+import type { VisualEvidenceResult } from "../src/validate.ts";
+
+function screenshotArtifact(contents: string): ScreenshotArtifact {
+  const png = Buffer.from(contents);
+  return {
+    png,
+    hash: createHash("sha256").update(png).digest("hex"),
+    route: `#/workspace/${contents}/session`,
+    visibleText: contents,
+    at: "2026-07-29T12:00:00.000Z",
+  };
+}
+
+function seen(caption: string, passed: boolean): VisualEvidenceResult {
+  return {
+    ok: passed,
+    description: `Description for ${caption}`,
+    results: [{ expectation: caption, passed, evidence: `${caption} evidence` }],
+    why: passed ? "" : `${caption} failed`,
+    model: "injected-vision-model",
+    cached: false,
+  };
+}
+
+test("visual evidence writes distinct screenshots, verdicts, and idempotent indexes", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-visual-evidence-"));
+  try {
+    const visualEvidence = createVisualEvidence("unit test run", { outDir: dir });
+    const firstPath = await visualEvidence.recordScreenshot(screenshotArtifact("first"), seen("First caption", true));
+    const secondPath = await visualEvidence.recordScreenshot(screenshotArtifact("second"), seen("Second caption", false));
+
+    const firstClose = await visualEvidence.close();
+    const secondClose = await visualEvidence.close();
+    await stat(firstPath);
+    await stat(secondPath);
+    assert.equal(firstClose, join(dir, "index.html"));
+    assert.equal(secondClose, firstClose);
+
+    const index = await readFile(join(dir, "index.html"), "utf8");
+    const json = await readFile(join(dir, "test-run.json"), "utf8");
+    for (const expected of ["First caption", "Second caption", "PASS", "FAIL"]) {
+      assert.match(index, new RegExp(expected));
+    }
+    for (const expected of ["First caption", "Second caption", '"passed": true', '"passed": false']) {
+      assert.match(json, new RegExp(expected));
+    }
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("visual evidence rejects duplicate pixels assigned to different validations", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-visual-evidence-duplicate-"));
+  try {
+    const visualEvidence = createVisualEvidence("duplicate test run", { outDir: dir });
+    const duplicate = screenshotArtifact("same pixels");
+    await visualEvidence.recordScreenshot(duplicate, seen("Original screenshot", true));
+    await assert.rejects(
+      () => visualEvidence.recordScreenshot(duplicate, seen("Copied screenshot", true)),
+      /Copied screenshot.*Original screenshot/,
+    );
+    await visualEvidence.close();
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});