源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env node
|
||||
import { access, readFile, readdir } from "node:fs/promises";
|
||||
import { isAbsolute, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { judgeTestRun } from "../src/test-evidence.ts";
|
||||
|
||||
const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
|
||||
const testRunsDir = join(repoRoot, "evals", "results", "test-runs");
|
||||
const args = process.argv.slice(2);
|
||||
let testRunArg;
|
||||
let force = false;
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--") continue;
|
||||
if (arg === "--force") {
|
||||
force = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--test-run") {
|
||||
const value = args[index + 1];
|
||||
if (!value || value.startsWith("--")) throw new Error("--test-run requires a value.");
|
||||
testRunArg = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
if (!testRunArg) throw new Error("--test-run <path|directory-id|latest|name> is required.");
|
||||
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
async function hasTestRun(directory) {
|
||||
try {
|
||||
await access(join(directory, "test-run.json"));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function recordedTestRuns() {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(testRunsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const testRuns = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const directory = join(testRunsDir, entry.name);
|
||||
try {
|
||||
const value = JSON.parse(await readFile(join(directory, "test-run.json"), "utf8"));
|
||||
if (!isRecord(value) || typeof value.name !== "string" || typeof value.createdAt !== "string") continue;
|
||||
testRuns.push({ directory, directoryId: entry.name, name: value.name, createdAt: value.createdAt });
|
||||
} catch {
|
||||
// Ignore incomplete or corrupt test-run directories while resolving a selection.
|
||||
}
|
||||
}
|
||||
return testRuns.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
|
||||
}
|
||||
|
||||
async function resolveTestRun() {
|
||||
if (testRunArg === "latest") return (await recordedTestRuns())[0]?.directory;
|
||||
const candidates = [
|
||||
isAbsolute(testRunArg) ? testRunArg : resolve(process.cwd(), testRunArg),
|
||||
resolve(repoRoot, testRunArg),
|
||||
join(testRunsDir, testRunArg),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (await hasTestRun(candidate)) return candidate;
|
||||
}
|
||||
const testRuns = await recordedTestRuns();
|
||||
return testRuns.find((testRun) => testRun.directoryId === testRunArg || testRun.name === testRunArg)?.directory;
|
||||
}
|
||||
|
||||
const testRunDir = await resolveTestRun();
|
||||
if (!testRunDir) throw new Error(`No test run found for ${testRunArg}.`);
|
||||
|
||||
const result = await judgeTestRun(testRunDir, { force });
|
||||
for (const error of result.errors) process.stderr.write(`Pending: ${error}\n`);
|
||||
process.stdout.write(`Judged ${result.judgedValidations} visual validation(s): ${result.failedValidations} failed, ${result.pendingValidations} pending.\n${result.testRunPath}\n`);
|
||||
process.exitCode = result.pendingValidations > 0 ? 2 : result.failedValidations > 0 ? 1 : 0;
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@openwork/test-evidence",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"test-evidence-judge": "./bin/test-evidence-judge.mjs"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./src/index.ts",
|
||||
"development": "./src/index.ts",
|
||||
"default": "./src/index.ts"
|
||||
},
|
||||
"./vitest": {
|
||||
"types": "./src/vitest.ts",
|
||||
"development": "./src/vitest.ts",
|
||||
"default": "./src/vitest.ts"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@openwork/cdp": "workspace:*",
|
||||
"@openwork/timeline": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import type { TestEvidenceRecorder } from "./test-evidence.ts";
|
||||
|
||||
const testEvidenceStorage = new AsyncLocalStorage<TestEvidenceRecorder>();
|
||||
|
||||
export function currentTestEvidence(): TestEvidenceRecorder | null {
|
||||
return testEvidenceStorage.getStore() ?? null;
|
||||
}
|
||||
|
||||
export function withTestEvidence<Result>(testEvidence: TestEvidenceRecorder, fn: () => Result): Result {
|
||||
return testEvidenceStorage.run(testEvidence, fn);
|
||||
}
|
||||
|
||||
export function enterTestEvidence(testEvidence: TestEvidenceRecorder): () => void {
|
||||
const previous = testEvidenceStorage.getStore();
|
||||
testEvidenceStorage.enterWith(testEvidence);
|
||||
return () => {
|
||||
testEvidenceStorage.disable();
|
||||
if (previous) testEvidenceStorage.enterWith(previous);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export { currentTestEvidence, withTestEvidence } from "./ambient.ts";
|
||||
export * from "./screenshot.ts";
|
||||
export * from "./test-evidence.ts";
|
||||
export * from "./validate.ts";
|
||||
export * from "./visual-evidence.ts";
|
||||
@@ -0,0 +1,37 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { captureScreenshot, evaluate } from "@openwork/cdp";
|
||||
import type { Surface } from "@openwork/cdp";
|
||||
import { currentTestEvidence } from "./ambient.ts";
|
||||
|
||||
export interface ScreenshotArtifact {
|
||||
png: Buffer;
|
||||
hash: string;
|
||||
route: string;
|
||||
visibleText: string;
|
||||
at: string;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export async function screenshot(app: Surface): Promise<ScreenshotArtifact> {
|
||||
const at = new Date().toISOString();
|
||||
const png = await captureScreenshot(app.client);
|
||||
const page = await evaluate(app.client, `({
|
||||
route: window.location.hash,
|
||||
visibleText: document.body.innerText,
|
||||
})`);
|
||||
if (!isRecord(page) || typeof page.route !== "string" || typeof page.visibleText !== "string") {
|
||||
throw new Error("CDP did not return the current route and visible text for the screenshot.");
|
||||
}
|
||||
const screenshotArtifact: ScreenshotArtifact = {
|
||||
png,
|
||||
hash: createHash("sha256").update(png).digest("hex"),
|
||||
route: page.route,
|
||||
visibleText: page.visibleText,
|
||||
at,
|
||||
};
|
||||
currentTestEvidence()?.recordScreenshot(screenshotArtifact);
|
||||
return screenshotArtifact;
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { ScreenshotArtifact } from "./screenshot.ts";
|
||||
import { judgeVision } from "./validate.ts";
|
||||
import type { ValidateOptions, VisualEvidenceResult, VisualExpectationResult } from "./validate.ts";
|
||||
|
||||
const REPO_ROOT = fileURLToPath(new URL("../../../..", import.meta.url));
|
||||
|
||||
export type EvidenceJudgmentState = "passed" | "failed" | "pending";
|
||||
|
||||
export interface EvidenceJudgment {
|
||||
expectation: string;
|
||||
state: EvidenceJudgmentState;
|
||||
reasoning: string;
|
||||
}
|
||||
|
||||
export interface TestArtifact {
|
||||
kind?: never;
|
||||
caption: string;
|
||||
fileName: string;
|
||||
hash: string;
|
||||
route: string;
|
||||
at: string;
|
||||
description: string;
|
||||
model: string;
|
||||
ok: boolean | null;
|
||||
results: VisualExpectationResult[];
|
||||
judgments: EvidenceJudgment[];
|
||||
}
|
||||
|
||||
export interface JsonArtifact {
|
||||
kind: "json";
|
||||
label: string;
|
||||
fileName: string;
|
||||
}
|
||||
|
||||
export interface TestRunSummary {
|
||||
ok: boolean;
|
||||
totalArtifacts: number;
|
||||
passedArtifacts: number;
|
||||
failedArtifacts: number;
|
||||
unvalidatedArtifacts: number;
|
||||
pendingArtifacts: number;
|
||||
passedExpectations: number;
|
||||
failedExpectations: number;
|
||||
pendingJudgments: number;
|
||||
}
|
||||
|
||||
export interface TestRunRecord {
|
||||
name: string;
|
||||
dir: string;
|
||||
createdAt: string;
|
||||
closedAt: string;
|
||||
gitSha?: string;
|
||||
branch?: string;
|
||||
summary: TestRunSummary;
|
||||
artifacts: (TestArtifact | JsonArtifact)[];
|
||||
}
|
||||
|
||||
interface StoredTestArtifact extends TestArtifact {
|
||||
sequence: number;
|
||||
png: Buffer | null;
|
||||
validationKey: string | null;
|
||||
}
|
||||
|
||||
interface StoredJsonArtifact extends JsonArtifact {
|
||||
sequence: number;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
export interface TestEvidenceRecorder {
|
||||
readonly dir: string;
|
||||
recordScreenshot(screenshotArtifact: ScreenshotArtifact): string;
|
||||
recordVisualValidation(screenshotHash: string, visualEvidence: VisualEvidenceResult): string;
|
||||
recordAssertionEvidence(assertion: string, evidence: string, passed: boolean): void;
|
||||
recordJsonArtifact(label: string, value: unknown): void;
|
||||
close(): Promise<string>;
|
||||
[Symbol.asyncDispose](): Promise<void>;
|
||||
}
|
||||
|
||||
export interface JudgeTestRunOptions extends ValidateOptions {
|
||||
force?: boolean;
|
||||
}
|
||||
|
||||
export interface JudgeTestRunResult {
|
||||
testRunPath: string;
|
||||
judgedValidations: number;
|
||||
failedValidations: number;
|
||||
pendingValidations: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function slug(value: string): string {
|
||||
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80) || "artifact";
|
||||
}
|
||||
|
||||
function html(value: string): string {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
function fileName(sequence: number, caption: string): string {
|
||||
return `${String(sequence).padStart(2, "0")}-${slug(caption)}.png`;
|
||||
}
|
||||
|
||||
function jsonFileName(sequence: number, label: string): string {
|
||||
return `${String(sequence).padStart(2, "0")}-${slug(label)}.json`;
|
||||
}
|
||||
|
||||
function judgmentsForVisualEvidence(visualEvidence: VisualEvidenceResult): EvidenceJudgment[] {
|
||||
if (visualEvidence.deferred) {
|
||||
if (!visualEvidence.pendingExpectations) throw new Error("Deferred visual evidence must include pending expectations.");
|
||||
return visualEvidence.pendingExpectations.map((expectation) => ({
|
||||
expectation,
|
||||
state: "pending",
|
||||
reasoning: visualEvidence.why,
|
||||
}));
|
||||
}
|
||||
return visualEvidence.results.map((result) => ({
|
||||
expectation: result.expectation,
|
||||
state: result.passed ? "passed" : "failed",
|
||||
reasoning: result.evidence,
|
||||
}));
|
||||
}
|
||||
|
||||
function artifactCaption(name: string, sequence: number, visualEvidence?: VisualEvidenceResult): string {
|
||||
return visualEvidence?.results[0]?.expectation.trim()
|
||||
|| visualEvidence?.pendingExpectations?.[0]?.trim()
|
||||
|| `${name} artifact ${sequence}`;
|
||||
}
|
||||
|
||||
function visualValidationKey(visualEvidence: VisualEvidenceResult): string {
|
||||
return JSON.stringify(judgmentsForVisualEvidence(visualEvidence).map((judgment) => judgment.expectation.trim()));
|
||||
}
|
||||
|
||||
function gitValue(args: string[]): string {
|
||||
const result = spawnSync("git", ["rev-parse", ...args], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
});
|
||||
return result.status === 0 && !result.error ? result.stdout.trim() : "";
|
||||
}
|
||||
|
||||
function testArtifact(artifact: StoredTestArtifact): TestArtifact {
|
||||
return {
|
||||
caption: artifact.caption,
|
||||
fileName: artifact.fileName,
|
||||
hash: artifact.hash,
|
||||
route: artifact.route,
|
||||
at: artifact.at,
|
||||
description: artifact.description,
|
||||
model: artifact.model,
|
||||
ok: artifact.ok,
|
||||
results: artifact.results,
|
||||
judgments: artifact.judgments,
|
||||
};
|
||||
}
|
||||
|
||||
function summarize(artifacts: TestArtifact[]): TestRunSummary {
|
||||
const judgments = artifacts.flatMap((artifact) => artifact.judgments);
|
||||
const pendingArtifacts = artifacts.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length;
|
||||
return {
|
||||
ok: artifacts.length > 0 && artifacts.every((artifact) => artifact.ok === true),
|
||||
totalArtifacts: artifacts.length,
|
||||
passedArtifacts: artifacts.filter((artifact) => artifact.ok === true).length,
|
||||
failedArtifacts: artifacts.filter((artifact) => artifact.ok === false).length,
|
||||
unvalidatedArtifacts: artifacts.filter((artifact) => artifact.ok === null).length,
|
||||
pendingArtifacts,
|
||||
passedExpectations: judgments.filter((judgment) => judgment.state === "passed").length,
|
||||
failedExpectations: judgments.filter((judgment) => judgment.state === "failed").length,
|
||||
pendingJudgments: judgments.filter((judgment) => judgment.state === "pending").length,
|
||||
};
|
||||
}
|
||||
|
||||
function renderArtifact(artifact: TestArtifact): string {
|
||||
const pending = artifact.judgments.some((judgment) => judgment.state === "pending");
|
||||
const stateClass = pending ? "pending" : artifact.ok === true ? "passed" : artifact.ok === false ? "failed" : "unvalidated";
|
||||
const description = artifact.description || (pending ? "Visual validation pending." : "Not visually validated.");
|
||||
return `
|
||||
<article class="artifact ${stateClass}">
|
||||
<h2>${html(artifact.caption)}</h2>
|
||||
<p class="meta">${html(artifact.route)} · ${html(artifact.at)}${artifact.model ? ` · ${html(artifact.model)}` : ""}</p>
|
||||
${artifact.fileName ? `<img src="${html(artifact.fileName)}" alt="${html(artifact.caption)}">` : ""}
|
||||
<p>${html(description)}</p>
|
||||
<ul>${artifact.judgments.map((judgment) => `<li class="${judgment.state}"><strong>${judgment.state.toUpperCase()}</strong> ${html(judgment.expectation)} — ${html(judgment.reasoning)}</li>`).join("")}</ul>
|
||||
</article>`;
|
||||
}
|
||||
|
||||
function renderIndex(record: TestRunRecord): string {
|
||||
const summary = record.summary;
|
||||
const testArtifacts = record.artifacts.filter((artifact): artifact is TestArtifact => artifact.kind !== "json");
|
||||
const jsonArtifacts = record.artifacts.filter((artifact): artifact is JsonArtifact => artifact.kind === "json");
|
||||
const validatedArtifacts = testArtifacts.filter((artifact) => artifact.judgments.length > 0).map(renderArtifact).join("");
|
||||
const unvalidatedArtifacts = testArtifacts.filter((artifact) => artifact.judgments.length === 0);
|
||||
const unvalidatedMarkup = unvalidatedArtifacts.length > 0
|
||||
? `<details class="unvalidated-artifacts unvalidated"><summary>unvalidated artifacts (${unvalidatedArtifacts.length})</summary>${unvalidatedArtifacts.map(renderArtifact).join("")}</details>`
|
||||
: "";
|
||||
const jsonMarkup = jsonArtifacts.length > 0
|
||||
? `<details><summary>JSON artifacts (${jsonArtifacts.length})</summary><ul>${jsonArtifacts.map((artifact) => `<li><a href="${html(artifact.fileName)}">${html(artifact.fileName)}</a></li>`).join("")}</ul></details>`
|
||||
: "";
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>${html(record.name)} test evidence</title><style>
|
||||
body{font:15px/1.5 system-ui,sans-serif;max-width:1100px;margin:40px auto;padding:0 20px;background:#f6f7f9;color:#17191d}header,.artifact,details{background:white;border:1px solid #dfe2e8;border-radius:12px;padding:20px;margin:0 0 24px}.artifact.passed{border-left:6px solid #238636}.artifact.failed{border-left:6px solid #cf222e}.artifact.pending,.artifact.unvalidated,details.unvalidated{border-left:6px solid #9a6700}details .artifact{margin-top:20px}summary{cursor:pointer;font-weight:700}img{display:block;width:100%;height:auto;border:1px solid #dfe2e8;border-radius:8px}.meta{color:#636c76}.passed strong{color:#1a7f37}.failed strong{color:#cf222e}.pending strong{color:#9a6700}li{margin:8px 0}
|
||||
</style></head><body><header><h1>${html(record.name)}</h1><p>${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed; ${summary.failedArtifacts} failed; ${summary.pendingArtifacts} pending; ${summary.unvalidatedArtifacts - summary.pendingArtifacts} unvalidated. ${summary.passedExpectations} expectations passed, ${summary.failedExpectations} failed, and ${summary.pendingJudgments} pending.</p></header>${validatedArtifacts}${unvalidatedMarkup}${jsonMarkup}</body></html>
|
||||
`;
|
||||
}
|
||||
|
||||
async function writeTestRun(record: TestRunRecord, testRunDir: string): Promise<void> {
|
||||
await writeFile(join(testRunDir, "test-run.json"), `${JSON.stringify(record, null, 2)}\n`, "utf8");
|
||||
await writeFile(join(testRunDir, "index.html"), renderIndex(record), "utf8");
|
||||
}
|
||||
|
||||
function parseVisualExpectationResult(value: unknown): VisualExpectationResult | null {
|
||||
if (
|
||||
!isRecord(value)
|
||||
|| typeof value.expectation !== "string"
|
||||
|| typeof value.passed !== "boolean"
|
||||
|| typeof value.evidence !== "string"
|
||||
) return null;
|
||||
return { expectation: value.expectation, passed: value.passed, evidence: value.evidence };
|
||||
}
|
||||
|
||||
function parseJudgment(value: unknown): EvidenceJudgment | null {
|
||||
if (
|
||||
!isRecord(value)
|
||||
|| typeof value.expectation !== "string"
|
||||
|| (value.state !== "passed" && value.state !== "failed" && value.state !== "pending")
|
||||
|| typeof value.reasoning !== "string"
|
||||
) return null;
|
||||
return { expectation: value.expectation, state: value.state, reasoning: value.reasoning };
|
||||
}
|
||||
|
||||
function judgmentForResult(result: VisualExpectationResult): EvidenceJudgment {
|
||||
return {
|
||||
expectation: result.expectation,
|
||||
state: result.passed ? "passed" : "failed",
|
||||
reasoning: result.evidence,
|
||||
};
|
||||
}
|
||||
|
||||
function parseTestArtifact(value: unknown): TestArtifact | null {
|
||||
if (
|
||||
!isRecord(value)
|
||||
|| typeof value.caption !== "string"
|
||||
|| typeof value.fileName !== "string"
|
||||
|| typeof value.hash !== "string"
|
||||
|| typeof value.route !== "string"
|
||||
|| typeof value.at !== "string"
|
||||
|| typeof value.description !== "string"
|
||||
|| typeof value.model !== "string"
|
||||
|| !Array.isArray(value.results)
|
||||
) return null;
|
||||
let ok: boolean | null;
|
||||
if (value.ok === null) ok = null;
|
||||
else if (typeof value.ok === "boolean") ok = value.ok;
|
||||
else return null;
|
||||
const results: VisualExpectationResult[] = [];
|
||||
for (const result of value.results) {
|
||||
const parsed = parseVisualExpectationResult(result);
|
||||
if (!parsed) return null;
|
||||
results.push(parsed);
|
||||
}
|
||||
const judgments: EvidenceJudgment[] = [];
|
||||
if (Array.isArray(value.judgments)) {
|
||||
for (const judgment of value.judgments) {
|
||||
const parsed = parseJudgment(judgment);
|
||||
if (!parsed) return null;
|
||||
judgments.push(parsed);
|
||||
}
|
||||
} else {
|
||||
judgments.push(...results.map(judgmentForResult));
|
||||
}
|
||||
return {
|
||||
caption: value.caption,
|
||||
fileName: value.fileName,
|
||||
hash: value.hash,
|
||||
route: value.route,
|
||||
at: value.at,
|
||||
description: value.description,
|
||||
model: value.model,
|
||||
ok,
|
||||
results,
|
||||
judgments,
|
||||
};
|
||||
}
|
||||
|
||||
function parseTestRun(value: unknown): TestRunRecord | null {
|
||||
if (
|
||||
!isRecord(value)
|
||||
|| typeof value.name !== "string"
|
||||
|| typeof value.dir !== "string"
|
||||
|| typeof value.createdAt !== "string"
|
||||
|| typeof value.closedAt !== "string"
|
||||
|| !Array.isArray(value.artifacts)
|
||||
) return null;
|
||||
const artifacts: (TestArtifact | JsonArtifact)[] = [];
|
||||
for (const artifact of value.artifacts) {
|
||||
if (
|
||||
isRecord(artifact)
|
||||
&& artifact.kind === "json"
|
||||
&& typeof artifact.label === "string"
|
||||
&& typeof artifact.fileName === "string"
|
||||
) {
|
||||
artifacts.push({ kind: "json", label: artifact.label, fileName: artifact.fileName });
|
||||
continue;
|
||||
}
|
||||
const parsed = parseTestArtifact(artifact);
|
||||
if (!parsed) return null;
|
||||
artifacts.push(parsed);
|
||||
}
|
||||
const gitSha = typeof value.gitSha === "string" ? value.gitSha : undefined;
|
||||
const branch = typeof value.branch === "string" ? value.branch : undefined;
|
||||
return {
|
||||
name: value.name,
|
||||
dir: value.dir,
|
||||
createdAt: value.createdAt,
|
||||
closedAt: value.closedAt,
|
||||
gitSha,
|
||||
branch,
|
||||
summary: summarize(artifacts.filter((artifact): artifact is TestArtifact => artifact.kind !== "json")),
|
||||
artifacts,
|
||||
};
|
||||
}
|
||||
|
||||
function artifactOk(judgments: EvidenceJudgment[]): boolean | null {
|
||||
if (judgments.length === 0 || judgments.some((judgment) => judgment.state === "pending")) return null;
|
||||
return judgments.every((judgment) => judgment.state === "passed");
|
||||
}
|
||||
|
||||
function resultForJudgment(judgment: EvidenceJudgment): VisualExpectationResult[] {
|
||||
if (judgment.state === "pending") return [];
|
||||
return [{
|
||||
expectation: judgment.expectation,
|
||||
passed: judgment.state === "passed",
|
||||
evidence: judgment.reasoning,
|
||||
}];
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
export async function judgeTestRun(testRunDir: string, opts: JudgeTestRunOptions = {}): Promise<JudgeTestRunResult> {
|
||||
const testRunPath = join(testRunDir, "test-run.json");
|
||||
const value: unknown = JSON.parse(await readFile(testRunPath, "utf8"));
|
||||
const record = parseTestRun(value);
|
||||
if (!record) throw new Error(`Invalid test run: ${testRunPath}`);
|
||||
let touched = false;
|
||||
let judgedValidations = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const artifact of record.artifacts) {
|
||||
if (artifact.kind === "json") continue;
|
||||
if (!artifact.fileName) continue;
|
||||
const targetIndexes = artifact.judgments.flatMap((judgment, index) => (
|
||||
opts.force || judgment.state === "pending" ? [index] : []
|
||||
));
|
||||
if (targetIndexes.length === 0) continue;
|
||||
touched = true;
|
||||
const expectations = targetIndexes.map((index) => artifact.judgments[index].expectation);
|
||||
try {
|
||||
const png = await readFile(join(testRunDir, artifact.fileName));
|
||||
const visualEvidence = await judgeVision({
|
||||
png,
|
||||
hash: artifact.hash,
|
||||
route: artifact.route,
|
||||
visibleText: "",
|
||||
at: artifact.at,
|
||||
}, expectations, { ask: opts.ask, bypassCache: opts.force });
|
||||
for (const [resultIndex, result] of visualEvidence.results.entries()) {
|
||||
const judgmentIndex = targetIndexes[resultIndex];
|
||||
artifact.judgments[judgmentIndex] = {
|
||||
expectation: result.expectation,
|
||||
state: result.passed ? "passed" : "failed",
|
||||
reasoning: result.evidence,
|
||||
};
|
||||
}
|
||||
artifact.description = visualEvidence.description;
|
||||
artifact.model = visualEvidence.model;
|
||||
judgedValidations += targetIndexes.length;
|
||||
} catch (error) {
|
||||
const message = errorMessage(error);
|
||||
errors.push(`${artifact.caption}: ${message}`);
|
||||
for (const index of targetIndexes) {
|
||||
const judgment = artifact.judgments[index];
|
||||
artifact.judgments[index] = { ...judgment, state: "pending", reasoning: `Provider error: ${message}` };
|
||||
}
|
||||
}
|
||||
artifact.results = artifact.judgments.flatMap(resultForJudgment);
|
||||
artifact.ok = artifactOk(artifact.judgments);
|
||||
}
|
||||
|
||||
const testArtifacts = record.artifacts.filter((artifact): artifact is TestArtifact => artifact.kind !== "json");
|
||||
record.summary = summarize(testArtifacts);
|
||||
if (touched) await writeTestRun(record, testRunDir);
|
||||
const visualJudgments = testArtifacts.filter((artifact) => artifact.fileName).flatMap((artifact) => artifact.judgments);
|
||||
return {
|
||||
testRunPath,
|
||||
judgedValidations,
|
||||
failedValidations: visualJudgments.filter((judgment) => judgment.state === "failed").length,
|
||||
pendingValidations: visualJudgments.filter((judgment) => judgment.state === "pending").length,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export function createTestEvidence(meta: { name: string; outDir?: string }): TestEvidenceRecorder {
|
||||
const { name } = meta;
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const dir = meta.outDir ?? join(REPO_ROOT, "evals", "results", "test-runs", `${stamp}-${slug(name)}`);
|
||||
const artifacts: StoredTestArtifact[] = [];
|
||||
const jsonArtifacts: StoredJsonArtifact[] = [];
|
||||
const createdAt = new Date().toISOString();
|
||||
const gitSha = gitValue(["HEAD"]);
|
||||
const branch = gitValue(["--abbrev-ref", "HEAD"]);
|
||||
let nextSequence = 1;
|
||||
let closing: Promise<string> | null = null;
|
||||
|
||||
const assertOpen = (): void => {
|
||||
if (closing) throw new Error(`Cannot record test evidence after "${name}" is closed.`);
|
||||
};
|
||||
|
||||
const close = (): Promise<string> => {
|
||||
if (closing) return closing;
|
||||
closing = (async () => {
|
||||
await mkdir(dir, { recursive: true });
|
||||
for (const artifact of artifacts) {
|
||||
if (artifact.png) await writeFile(join(dir, artifact.fileName), artifact.png);
|
||||
}
|
||||
for (const artifact of jsonArtifacts) {
|
||||
await writeFile(join(dir, artifact.fileName), `${JSON.stringify(artifact.value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
const orderedArtifacts = [
|
||||
...artifacts.filter((artifact) => artifact.validationKey !== null),
|
||||
...artifacts.filter((artifact) => artifact.validationKey === null),
|
||||
].map(testArtifact);
|
||||
const record: TestRunRecord = {
|
||||
name,
|
||||
dir,
|
||||
createdAt,
|
||||
closedAt: new Date().toISOString(),
|
||||
gitSha,
|
||||
branch,
|
||||
summary: summarize(orderedArtifacts),
|
||||
artifacts: [...orderedArtifacts, ...jsonArtifacts.map(({ kind, label, fileName: artifactFileName }) => ({
|
||||
kind,
|
||||
label,
|
||||
fileName: artifactFileName,
|
||||
}))],
|
||||
};
|
||||
await writeTestRun(record, dir);
|
||||
return join(dir, "index.html");
|
||||
})();
|
||||
return closing;
|
||||
};
|
||||
|
||||
return {
|
||||
dir,
|
||||
recordScreenshot(screenshotArtifact) {
|
||||
assertOpen();
|
||||
const sequence = nextSequence;
|
||||
nextSequence += 1;
|
||||
const caption = artifactCaption(name, sequence);
|
||||
const screenshotFileName = fileName(sequence, caption);
|
||||
artifacts.push({
|
||||
caption,
|
||||
fileName: screenshotFileName,
|
||||
hash: screenshotArtifact.hash,
|
||||
route: screenshotArtifact.route,
|
||||
at: screenshotArtifact.at,
|
||||
description: "",
|
||||
model: "",
|
||||
ok: null,
|
||||
results: [],
|
||||
judgments: [],
|
||||
sequence,
|
||||
png: screenshotArtifact.png,
|
||||
validationKey: null,
|
||||
});
|
||||
return join(dir, screenshotFileName);
|
||||
},
|
||||
recordVisualValidation(screenshotHash, visualEvidence) {
|
||||
assertOpen();
|
||||
const key = visualValidationKey(visualEvidence);
|
||||
const judgments = judgmentsForVisualEvidence(visualEvidence);
|
||||
const validated = artifacts.find((artifact) => artifact.png !== null && artifact.hash === screenshotHash && artifact.validationKey !== null);
|
||||
if (validated) {
|
||||
const caption = artifactCaption(name, validated.sequence, visualEvidence);
|
||||
if (validated.validationKey !== key) {
|
||||
throw new Error(`Screenshot pixels for "${caption}" already back the different visual validation "${validated.caption}".`);
|
||||
}
|
||||
validated.caption = caption;
|
||||
validated.fileName = fileName(validated.sequence, caption);
|
||||
validated.description = visualEvidence.description;
|
||||
validated.model = visualEvidence.model;
|
||||
validated.ok = visualEvidence.deferred ? null : visualEvidence.ok;
|
||||
validated.results = visualEvidence.results;
|
||||
validated.judgments = judgments;
|
||||
return join(dir, validated.fileName);
|
||||
}
|
||||
const screenshotArtifact = artifacts.find((artifact) => artifact.png !== null && artifact.hash === screenshotHash && artifact.validationKey === null);
|
||||
if (!screenshotArtifact) throw new Error(`No recorded screenshot has hash "${screenshotHash}".`);
|
||||
const caption = artifactCaption(name, screenshotArtifact.sequence, visualEvidence);
|
||||
screenshotArtifact.caption = caption;
|
||||
screenshotArtifact.fileName = fileName(screenshotArtifact.sequence, caption);
|
||||
screenshotArtifact.description = visualEvidence.description;
|
||||
screenshotArtifact.model = visualEvidence.model;
|
||||
screenshotArtifact.ok = visualEvidence.deferred ? null : visualEvidence.ok;
|
||||
screenshotArtifact.results = visualEvidence.results;
|
||||
screenshotArtifact.judgments = judgments;
|
||||
screenshotArtifact.validationKey = key;
|
||||
return join(dir, screenshotArtifact.fileName);
|
||||
},
|
||||
recordAssertionEvidence(assertion, evidence, passed) {
|
||||
assertOpen();
|
||||
const sequence = nextSequence;
|
||||
nextSequence += 1;
|
||||
const caption = assertion.trim() || `${name} assertion ${sequence}`;
|
||||
artifacts.push({
|
||||
caption,
|
||||
fileName: "",
|
||||
hash: "",
|
||||
route: "",
|
||||
at: new Date().toISOString(),
|
||||
description: evidence,
|
||||
model: "",
|
||||
ok: passed,
|
||||
results: [{ expectation: caption, evidence, passed }],
|
||||
judgments: [{ expectation: caption, state: passed ? "passed" : "failed", reasoning: evidence }],
|
||||
sequence,
|
||||
png: null,
|
||||
validationKey: JSON.stringify([caption]),
|
||||
});
|
||||
},
|
||||
recordJsonArtifact(label, value) {
|
||||
assertOpen();
|
||||
const sequence = nextSequence;
|
||||
nextSequence += 1;
|
||||
jsonArtifacts.push({
|
||||
kind: "json",
|
||||
label,
|
||||
fileName: jsonFileName(sequence, label),
|
||||
sequence,
|
||||
value,
|
||||
});
|
||||
},
|
||||
close,
|
||||
async [Symbol.asyncDispose]() {
|
||||
await close();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
import { timed } from "@openwork/timeline";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { currentTestEvidence } from "./ambient.ts";
|
||||
import type { ScreenshotArtifact } from "./screenshot.ts";
|
||||
|
||||
const REPO_ROOT = fileURLToPath(new URL("../../../..", import.meta.url));
|
||||
const CACHE_DIR = join(REPO_ROOT, "evals", "results", ".vision-cache");
|
||||
// models.dev's checked-in catalog marks both current defaults as accepting
|
||||
// image input. The provider-specific IDs keep the no-SDK transport portable.
|
||||
const OPENAI_DEFAULT_MODEL = "gpt-5.6";
|
||||
const ANTHROPIC_DEFAULT_MODEL = "claude-sonnet-4-6";
|
||||
|
||||
export interface VisualExpectationResult {
|
||||
expectation: string;
|
||||
passed: boolean;
|
||||
evidence: string;
|
||||
}
|
||||
|
||||
export interface VisualEvidenceResult {
|
||||
ok: boolean;
|
||||
description: string;
|
||||
results: VisualExpectationResult[];
|
||||
why: string;
|
||||
model: string;
|
||||
cached: boolean;
|
||||
deferred?: boolean;
|
||||
pendingExpectations?: string[];
|
||||
}
|
||||
|
||||
export interface VisionRequest {
|
||||
prompt: string;
|
||||
png: Buffer;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface ValidateOptions {
|
||||
ask?: (req: VisionRequest) => Promise<string>;
|
||||
bypassCache?: boolean;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseJsonResponse(raw: string, label: string): unknown {
|
||||
const trimmed = raw.trim();
|
||||
const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
|
||||
const candidate = fenced?.[1] ?? trimmed;
|
||||
try {
|
||||
return JSON.parse(candidate);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`${label} was not valid JSON: ${message}. Response: ${trimmed.slice(0, 300)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseDescription(raw: string): string {
|
||||
const value = parseJsonResponse(raw, "Vision model description response");
|
||||
if (!isRecord(value) || typeof value.description !== "string" || !value.description.trim()) {
|
||||
throw new Error(`Vision model description response must be {"description":"..."}. Response: ${raw.trim().slice(0, 300)}`);
|
||||
}
|
||||
return value.description.trim();
|
||||
}
|
||||
|
||||
function parseResults(value: unknown, expectations: string[], label: string): VisualExpectationResult[] {
|
||||
if (!isRecord(value) || !Array.isArray(value.results) || value.results.length !== expectations.length) {
|
||||
throw new Error(`${label} must contain exactly ${expectations.length} result entries.`);
|
||||
}
|
||||
const parsed: VisualExpectationResult[] = [];
|
||||
for (let index = 0; index < expectations.length; index += 1) {
|
||||
const expectation = expectations[index];
|
||||
const entry = value.results[index];
|
||||
if (
|
||||
!expectation
|
||||
|| !isRecord(entry)
|
||||
|| entry.expectation !== expectation
|
||||
|| typeof entry.passed !== "boolean"
|
||||
|| typeof entry.evidence !== "string"
|
||||
|| !entry.evidence.trim()
|
||||
) {
|
||||
throw new Error(`${label} result ${index + 1} must preserve the expectation and include boolean passed plus non-empty evidence.`);
|
||||
}
|
||||
parsed.push({ expectation, passed: entry.passed, evidence: entry.evidence.trim() });
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseVerdict(raw: string, expectations: string[]): VisualExpectationResult[] {
|
||||
return parseResults(parseJsonResponse(raw, "Vision model verdict response"), expectations, "Vision model verdict response");
|
||||
}
|
||||
|
||||
function parseCachedVisualEvidence(value: unknown, expectations: string[], model: string): VisualEvidenceResult {
|
||||
if (
|
||||
!isRecord(value)
|
||||
|| typeof value.description !== "string"
|
||||
|| typeof value.why !== "string"
|
||||
|| typeof value.ok !== "boolean"
|
||||
|| value.model !== model
|
||||
) {
|
||||
throw new Error("Vision cache entry has an invalid VisualEvidenceResult shape.");
|
||||
}
|
||||
const results = parseResults(value, expectations, "Vision cache entry");
|
||||
const ok = results.every((result) => result.passed);
|
||||
if (value.ok !== ok || (ok && value.why !== "")) {
|
||||
throw new Error("Vision cache entry has an inconsistent verdict.");
|
||||
}
|
||||
return {
|
||||
ok,
|
||||
description: value.description,
|
||||
results,
|
||||
why: value.why,
|
||||
model,
|
||||
cached: true,
|
||||
};
|
||||
}
|
||||
|
||||
function providerJson(raw: string, provider: string): unknown {
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error(`${provider} returned a non-JSON response: ${raw.slice(0, 300)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isTimeoutError(error: unknown): boolean {
|
||||
if (error instanceof Error && error.name === "TimeoutError") return true;
|
||||
return error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError");
|
||||
}
|
||||
|
||||
async function askOpenAi(req: VisionRequest, key: string): Promise<string> {
|
||||
const response = await fetch("https://api.openai.com/v1/chat/completions", {
|
||||
method: "POST",
|
||||
// Unbounded vision calls stalled runs for undici's full 300s header
|
||||
// timeout; 120s comfortably covers slow multimodal responses.
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
headers: {
|
||||
authorization: `Bearer ${key}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: req.model,
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: req.prompt },
|
||||
{ type: "image_url", image_url: { url: `data:image/png;base64,${req.png.toString("base64")}` } },
|
||||
],
|
||||
}],
|
||||
response_format: { type: "json_object" },
|
||||
}),
|
||||
});
|
||||
const raw = await response.text();
|
||||
if (!response.ok) throw new Error(`OpenAI vision request failed with HTTP ${response.status}: ${raw.slice(0, 500)}`);
|
||||
const body = providerJson(raw, "OpenAI");
|
||||
const choices = isRecord(body) ? body.choices : null;
|
||||
const first = Array.isArray(choices) ? choices[0] : null;
|
||||
const message = isRecord(first) ? first.message : null;
|
||||
const content = isRecord(message) ? message.content : null;
|
||||
if (typeof content !== "string" || !content.trim()) {
|
||||
throw new Error(`OpenAI vision response did not contain message content: ${raw.slice(0, 500)}`);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
async function askAnthropic(req: VisionRequest, key: string): Promise<string> {
|
||||
const response = await fetch("https://api.anthropic.com/v1/messages", {
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
headers: {
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
"x-api-key": key,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: req.model,
|
||||
max_tokens: 2_048,
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: req.png.toString("base64") } },
|
||||
{ type: "text", text: req.prompt },
|
||||
],
|
||||
}],
|
||||
}),
|
||||
});
|
||||
const raw = await response.text();
|
||||
if (!response.ok) throw new Error(`Anthropic vision request failed with HTTP ${response.status}: ${raw.slice(0, 500)}`);
|
||||
const body = providerJson(raw, "Anthropic");
|
||||
const content = isRecord(body) ? body.content : null;
|
||||
const text = Array.isArray(content)
|
||||
? content.flatMap((part) => isRecord(part) && part.type === "text" && typeof part.text === "string" ? [part.text] : []).join("\n")
|
||||
: "";
|
||||
if (!text.trim()) throw new Error(`Anthropic vision response did not contain text content: ${raw.slice(0, 500)}`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function failureSummary(results: VisualExpectationResult[]): string {
|
||||
const failures = results.filter((result) => !result.passed);
|
||||
if (failures.length === 0) return "";
|
||||
return `Failed ${failures.length}/${results.length} visual expectations: ${failures
|
||||
.map((result) => `${result.expectation} — ${result.evidence}`)
|
||||
.join("; ")}`;
|
||||
}
|
||||
|
||||
function visionModel(): string {
|
||||
const openAiKey = process.env.OPENAI_API_KEY?.trim() ?? "";
|
||||
const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim() ?? "";
|
||||
return process.env.OPENWORK_EVAL_VISION_MODEL?.trim()
|
||||
|| (anthropicKey && !openAiKey ? ANTHROPIC_DEFAULT_MODEL : OPENAI_DEFAULT_MODEL);
|
||||
}
|
||||
|
||||
export async function judgeVision(
|
||||
screenshotArtifact: ScreenshotArtifact,
|
||||
expectations: string[],
|
||||
opts: ValidateOptions = {},
|
||||
): Promise<VisualEvidenceResult> {
|
||||
// Vision latency is the main per-screenshot cost; record it so results show it.
|
||||
const openAiKey = process.env.OPENAI_API_KEY?.trim() ?? "";
|
||||
const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim() ?? "";
|
||||
const model = visionModel();
|
||||
let ask = opts.ask;
|
||||
if (!ask) {
|
||||
if (openAiKey) ask = (req) => askOpenAi(req, openAiKey);
|
||||
else if (anthropicKey) ask = (req) => askAnthropic(req, anthropicKey);
|
||||
else {
|
||||
throw new Error(
|
||||
"Vision validation requires OPENAI_API_KEY or ANTHROPIC_API_KEY, unless ValidateOptions.ask is provided.",
|
||||
);
|
||||
}
|
||||
}
|
||||
const askOnce = ask;
|
||||
ask = async (req) => {
|
||||
try {
|
||||
return await askOnce(req);
|
||||
} catch (error) {
|
||||
if (!isTimeoutError(error)) throw error;
|
||||
await delay(10_000);
|
||||
try {
|
||||
return await askOnce(req);
|
||||
} catch (retryError) {
|
||||
if (!isTimeoutError(retryError)) throw retryError;
|
||||
throw new Error("Vision request timed out on both attempts.", { cause: retryError });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const key = createHash("sha256")
|
||||
.update(screenshotArtifact.hash + JSON.stringify(expectations) + model)
|
||||
.digest("hex");
|
||||
const cachePath = join(CACHE_DIR, `${key}.json`);
|
||||
let visualEvidence: VisualEvidenceResult | null = null;
|
||||
if (!opts.bypassCache) {
|
||||
try {
|
||||
const cached = JSON.parse(await readFile(cachePath, "utf8"));
|
||||
visualEvidence = parseCachedVisualEvidence(cached, expectations, model);
|
||||
} catch (error) {
|
||||
if (isRecord(error) && error.code === "ENOENT") {
|
||||
// Cache miss: perform the two independent model requests below.
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!visualEvidence) {
|
||||
const description = parseDescription(await ask({
|
||||
prompt: [
|
||||
"Objectively describe only what is visibly present in this screenshot.",
|
||||
"Do not infer hidden state, correctness, intent, or expected behavior.",
|
||||
"Return only JSON matching: {\"description\":\"concise visible description\"}",
|
||||
].join("\n"),
|
||||
png: screenshotArtifact.png,
|
||||
model,
|
||||
}));
|
||||
const results = parseVerdict(await ask({
|
||||
prompt: [
|
||||
"Judge each expectation against the screenshot and the expectation-free description below.",
|
||||
`Independent description: ${JSON.stringify(description)}`,
|
||||
`Expectations: ${JSON.stringify(expectations)}`,
|
||||
"Return only JSON matching: {\"results\":[{\"expectation\":\"exact expectation text\",\"passed\":true,\"evidence\":\"short visible quote or observation\"}]}",
|
||||
"Return exactly one result per expectation, in the same order. Treat negative expectations literally and do not assume success.",
|
||||
].join("\n"),
|
||||
png: screenshotArtifact.png,
|
||||
model,
|
||||
}), expectations);
|
||||
visualEvidence = {
|
||||
ok: results.every((result) => result.passed),
|
||||
description,
|
||||
results,
|
||||
why: failureSummary(results),
|
||||
model,
|
||||
cached: false,
|
||||
};
|
||||
await mkdir(CACHE_DIR, { recursive: true });
|
||||
await writeFile(cachePath, `${JSON.stringify(visualEvidence, null, 2)}\n`, "utf8");
|
||||
}
|
||||
return visualEvidence;
|
||||
}
|
||||
|
||||
export async function validate(
|
||||
screenshotArtifact: ScreenshotArtifact,
|
||||
expectations: string[],
|
||||
opts: ValidateOptions = {},
|
||||
): Promise<VisualEvidenceResult> {
|
||||
// A caller-provided ask is a deterministic witness, not an LLM call, so defer never applies to it.
|
||||
if (!opts.ask && process.env.OPENWORK_EVAL_VISION?.trim() === "defer") {
|
||||
const visualEvidence: VisualEvidenceResult = {
|
||||
ok: true,
|
||||
description: "",
|
||||
results: [],
|
||||
why: "vision judgment deferred",
|
||||
model: visionModel(),
|
||||
cached: false,
|
||||
deferred: true,
|
||||
pendingExpectations: expectations,
|
||||
};
|
||||
currentTestEvidence()?.recordVisualValidation(screenshotArtifact.hash, visualEvidence);
|
||||
return visualEvidence;
|
||||
}
|
||||
|
||||
const visualEvidence = await judgeVision(screenshotArtifact, expectations, opts);
|
||||
currentTestEvidence()?.recordVisualValidation(screenshotArtifact.hash, visualEvidence);
|
||||
return visualEvidence;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { ScreenshotArtifact } from "./screenshot.ts";
|
||||
import { createTestEvidence } from "./test-evidence.ts";
|
||||
import type { VisualEvidenceResult } from "./validate.ts";
|
||||
|
||||
export interface VisualEvidenceRecorder {
|
||||
readonly dir: string;
|
||||
recordScreenshot(screenshotArtifact: ScreenshotArtifact, visualEvidence?: VisualEvidenceResult): Promise<string>;
|
||||
close(): Promise<string>;
|
||||
[Symbol.asyncDispose](): Promise<void>;
|
||||
}
|
||||
|
||||
export function createVisualEvidence(name: string, opts: { outDir?: string } = {}): VisualEvidenceRecorder {
|
||||
const testEvidence = createTestEvidence({ name, outDir: opts.outDir });
|
||||
return {
|
||||
dir: testEvidence.dir,
|
||||
async recordScreenshot(screenshotArtifact, visualEvidence) {
|
||||
const screenshotPath = testEvidence.recordScreenshot(screenshotArtifact);
|
||||
return visualEvidence
|
||||
? testEvidence.recordVisualValidation(screenshotArtifact.hash, visualEvidence)
|
||||
: screenshotPath;
|
||||
},
|
||||
close: () => testEvidence.close(),
|
||||
async [Symbol.asyncDispose]() {
|
||||
await testEvidence.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { expect, test as base } from "vitest";
|
||||
import { enterTestEvidence } from "./ambient.ts";
|
||||
import { createTestEvidence } from "./test-evidence.ts";
|
||||
import type { TestEvidenceRecorder } from "./test-evidence.ts";
|
||||
import type { VisualEvidenceResult } from "./validate.ts";
|
||||
|
||||
function slug(value: string): string {
|
||||
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80) || "test";
|
||||
}
|
||||
|
||||
export const test = base.extend<{ evidence: TestEvidenceRecorder }>({
|
||||
evidence: [async ({ task }, use) => {
|
||||
const testEvidence = createTestEvidence({ name: slug(task.name) });
|
||||
const leaveTestEvidence = enterTestEvidence(testEvidence);
|
||||
try {
|
||||
await use(testEvidence);
|
||||
} finally {
|
||||
leaveTestEvidence();
|
||||
await testEvidence.close();
|
||||
}
|
||||
}, { auto: true }],
|
||||
});
|
||||
|
||||
export function expectVisualEvidence(visualEvidence: VisualEvidenceResult): void {
|
||||
expect(visualEvidence.ok, visualEvidence.why).toBe(true);
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { currentTestEvidence, withTestEvidence } from "../src/ambient.ts";
|
||||
import type { ScreenshotArtifact } from "../src/screenshot.ts";
|
||||
import { createTestEvidence } from "../src/test-evidence.ts";
|
||||
|
||||
function screenshotArtifact(contents: string): ScreenshotArtifact {
|
||||
const png = Buffer.from(contents);
|
||||
return {
|
||||
png,
|
||||
hash: createHash("sha256").update(png).digest("hex"),
|
||||
route: `#/${contents}`,
|
||||
visibleText: contents,
|
||||
at: "2026-08-02T12:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
function pause(milliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
async function hashes(dir: string): Promise<unknown[]> {
|
||||
const value: unknown = JSON.parse(await readFile(join(dir, "test-run.json"), "utf8"));
|
||||
assert.ok(typeof value === "object" && value !== null && !Array.isArray(value));
|
||||
const artifacts = Reflect.get(value, "artifacts");
|
||||
assert.ok(Array.isArray(artifacts));
|
||||
return artifacts.map((artifact) => typeof artifact === "object" && artifact !== null ? Reflect.get(artifact, "hash") : null);
|
||||
}
|
||||
|
||||
test("concurrent withTestEvidence scopes do not cross-record", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "openwork-ambient-"));
|
||||
const firstDir = join(root, "first");
|
||||
const secondDir = join(root, "second");
|
||||
try {
|
||||
const first = createTestEvidence({ name: "first", outDir: firstDir });
|
||||
const second = createTestEvidence({ name: "second", outDir: secondDir });
|
||||
const firstScreenshot = screenshotArtifact("first screenshot");
|
||||
const secondScreenshot = screenshotArtifact("second screenshot");
|
||||
assert.equal(currentTestEvidence(), null);
|
||||
|
||||
await Promise.all([
|
||||
withTestEvidence(first, async () => {
|
||||
await pause(15);
|
||||
const active = currentTestEvidence();
|
||||
assert.equal(active, first);
|
||||
active.recordScreenshot(firstScreenshot);
|
||||
}),
|
||||
withTestEvidence(second, async () => {
|
||||
const active = currentTestEvidence();
|
||||
assert.equal(active, second);
|
||||
active.recordScreenshot(secondScreenshot);
|
||||
await pause(25);
|
||||
assert.equal(currentTestEvidence(), second);
|
||||
}),
|
||||
]);
|
||||
|
||||
assert.equal(currentTestEvidence(), null);
|
||||
await Promise.all([first.close(), second.close()]);
|
||||
assert.deepEqual(await hashes(firstDir), [firstScreenshot.hash]);
|
||||
assert.deepEqual(await hashes(secondDir), [secondScreenshot.hash]);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
import { createTestEvidence } from "../src/test-evidence.ts";
|
||||
|
||||
const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
|
||||
const testRunsDir = join(repoRoot, "evals", "results", "test-runs");
|
||||
const judgeCli = fileURLToPath(new URL("../bin/test-evidence-judge.mjs", import.meta.url));
|
||||
|
||||
test("judge CLI resolves a completed test run by record name", async () => {
|
||||
await mkdir(testRunsDir, { recursive: true });
|
||||
const testRunDir = await mkdtemp(join(testRunsDir, "judge-cli-name-"));
|
||||
const name = `Judge CLI name ${randomUUID()}`;
|
||||
try {
|
||||
const evidence = createTestEvidence({ name, outDir: testRunDir });
|
||||
evidence.recordAssertionEvidence("No visual judgment is pending", "The test run contains assertion evidence only.", true);
|
||||
await evidence.close();
|
||||
|
||||
const result = spawnSync(process.execPath, [judgeCli, "--test-run", name], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
assert.match(result.stdout, /Judged 0 visual validation\(s\): 0 failed, 0 pending\./);
|
||||
assert.match(result.stdout, new RegExp(`${testRunDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/test-run\\.json`));
|
||||
} finally {
|
||||
await rm(testRunDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import type { CdpClient, Surface } from "@openwork/cdp";
|
||||
import { withTestEvidence } from "../src/ambient.ts";
|
||||
import { screenshot } from "../src/screenshot.ts";
|
||||
import type { ScreenshotArtifact } from "../src/screenshot.ts";
|
||||
import { createTestEvidence } from "../src/test-evidence.ts";
|
||||
import type { VisualEvidenceResult } from "../src/validate.ts";
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function screenshotArtifact(contents: string): ScreenshotArtifact {
|
||||
const png = Buffer.from(contents);
|
||||
return {
|
||||
png,
|
||||
hash: createHash("sha256").update(png).digest("hex"),
|
||||
route: `#/${contents}`,
|
||||
visibleText: contents,
|
||||
at: "2026-08-02T12:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
function seen(expectation: string, passed: boolean, evidence = `${expectation} evidence`): VisualEvidenceResult {
|
||||
return {
|
||||
ok: passed,
|
||||
description: `Description for ${expectation}`,
|
||||
results: [{ expectation, passed, evidence }],
|
||||
why: passed ? "" : `${expectation} failed`,
|
||||
model: "injected-vision-model",
|
||||
cached: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function payload(dir: string): Promise<Record<string, unknown>> {
|
||||
const value: unknown = JSON.parse(await readFile(join(dir, "test-run.json"), "utf8"));
|
||||
assert.ok(isRecord(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
test("test evidence writes visual validations, assertions, failures, and unvalidated screenshots", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-"));
|
||||
try {
|
||||
const testEvidence = createTestEvidence({ name: "body cam", outDir: dir });
|
||||
const passing = screenshotArtifact("passing");
|
||||
const failing = screenshotArtifact("failing");
|
||||
testEvidence.recordScreenshot(passing);
|
||||
testEvidence.recordScreenshot(failing);
|
||||
testEvidence.recordScreenshot(screenshotArtifact("unvalidated"));
|
||||
testEvidence.recordVisualValidation(passing.hash, seen("Passing screenshot", true));
|
||||
testEvidence.recordVisualValidation(failing.hash, seen("Failing screenshot", false));
|
||||
testEvidence.recordAssertionEvidence("API returned success", "HTTP 200", true);
|
||||
await testEvidence.close();
|
||||
|
||||
const testRun = await payload(dir);
|
||||
assert.deepEqual(testRun.summary, {
|
||||
ok: false,
|
||||
totalArtifacts: 4,
|
||||
passedArtifacts: 2,
|
||||
failedArtifacts: 1,
|
||||
unvalidatedArtifacts: 1,
|
||||
pendingArtifacts: 0,
|
||||
passedExpectations: 2,
|
||||
failedExpectations: 1,
|
||||
pendingJudgments: 0,
|
||||
});
|
||||
assert.ok(Array.isArray(testRun.artifacts));
|
||||
assert.deepEqual(
|
||||
testRun.artifacts.map((artifact) => isRecord(artifact) ? artifact.caption : null),
|
||||
["Passing screenshot", "Failing screenshot", "API returned success", "body cam artifact 3"],
|
||||
);
|
||||
const failedArtifact = testRun.artifacts[1];
|
||||
const assertionArtifact = testRun.artifacts[2];
|
||||
assert.ok(isRecord(failedArtifact));
|
||||
assert.equal(failedArtifact.ok, false);
|
||||
assert.ok(isRecord(assertionArtifact));
|
||||
assert.equal(assertionArtifact.fileName, "");
|
||||
await stat(join(dir, "01-passing-screenshot.png"));
|
||||
await stat(join(dir, "02-failing-screenshot.png"));
|
||||
await stat(join(dir, "03-body-cam-artifact-3.png"));
|
||||
|
||||
const index = await readFile(join(dir, "index.html"), "utf8");
|
||||
assert.match(index, /API returned success/);
|
||||
assert.match(index, /unvalidated artifacts \(1\)/);
|
||||
assert.doesNotMatch(index, /<img src=""/);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("test evidence writes a JSON artifact and lists it in the test run", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-json-"));
|
||||
try {
|
||||
const testEvidence = createTestEvidence({ name: "world evidence", outDir: dir });
|
||||
testEvidence.recordJsonArtifact("world-snapshot primary", { version: 1, name: "primary" });
|
||||
await testEvidence.close();
|
||||
|
||||
assert.deepEqual(JSON.parse(await readFile(join(dir, "01-world-snapshot-primary.json"), "utf8")), {
|
||||
version: 1,
|
||||
name: "primary",
|
||||
});
|
||||
const testRun = await payload(dir);
|
||||
assert.deepEqual(testRun.artifacts, [{
|
||||
kind: "json",
|
||||
label: "world-snapshot primary",
|
||||
fileName: "01-world-snapshot-primary.json",
|
||||
}]);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("test evidence accepts unchanged screenshots and only lets one validation use their pixel hash", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-retake-"));
|
||||
try {
|
||||
const testEvidence = createTestEvidence({ name: "retakes", outDir: dir });
|
||||
const duplicate = screenshotArtifact("same pixels");
|
||||
testEvidence.recordScreenshot(duplicate);
|
||||
testEvidence.recordScreenshot(duplicate);
|
||||
testEvidence.recordVisualValidation(duplicate.hash, seen("Same validation", false, "first judgment"));
|
||||
testEvidence.recordVisualValidation(duplicate.hash, seen("Same validation", true, "replacement judgment"));
|
||||
assert.throws(
|
||||
() => testEvidence.recordVisualValidation(duplicate.hash, seen("Different validation", true)),
|
||||
/Different validation.*different visual validation.*Same validation/i,
|
||||
);
|
||||
await testEvidence.close();
|
||||
|
||||
const testRun = await payload(dir);
|
||||
assert.deepEqual(testRun.summary, {
|
||||
ok: false,
|
||||
totalArtifacts: 2,
|
||||
passedArtifacts: 1,
|
||||
failedArtifacts: 0,
|
||||
unvalidatedArtifacts: 1,
|
||||
pendingArtifacts: 0,
|
||||
passedExpectations: 1,
|
||||
failedExpectations: 0,
|
||||
pendingJudgments: 0,
|
||||
});
|
||||
assert.ok(Array.isArray(testRun.artifacts));
|
||||
const validated = testRun.artifacts[0];
|
||||
assert.ok(isRecord(validated));
|
||||
assert.equal(validated.ok, true);
|
||||
assert.ok(Array.isArray(validated.results));
|
||||
assert.deepEqual(validated.results[0], {
|
||||
expectation: "Same validation",
|
||||
passed: true,
|
||||
evidence: "replacement judgment",
|
||||
});
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("screenshot automatically records an artifact in ambient test evidence", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-test-evidence-screenshot-"));
|
||||
try {
|
||||
const png = Buffer.from("ambient screenshot pixels");
|
||||
const client: CdpClient = {
|
||||
close() {},
|
||||
async send(method) {
|
||||
if (method === "Page.captureScreenshot") return { data: png.toString("base64") };
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { route: "#/ambient", visibleText: "Ambient screenshot" } } };
|
||||
}
|
||||
throw new Error(`Unexpected CDP method: ${method}`);
|
||||
},
|
||||
};
|
||||
const app: Surface = {
|
||||
handle: { name: "fake", kind: "chrome", hostKind: "test", cdpUrl: "http://127.0.0.1" },
|
||||
client,
|
||||
};
|
||||
const testEvidence = createTestEvidence({ name: "ambient screenshot", outDir: dir });
|
||||
const captured = await withTestEvidence(testEvidence, () => screenshot(app));
|
||||
assert.equal(captured.route, "#/ambient");
|
||||
await testEvidence.close();
|
||||
|
||||
const testRun = await payload(dir);
|
||||
assert.deepEqual(testRun.summary, {
|
||||
ok: false,
|
||||
totalArtifacts: 1,
|
||||
passedArtifacts: 0,
|
||||
failedArtifacts: 0,
|
||||
unvalidatedArtifacts: 1,
|
||||
pendingArtifacts: 0,
|
||||
passedExpectations: 0,
|
||||
failedExpectations: 0,
|
||||
pendingJudgments: 0,
|
||||
});
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,182 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
import { withTestEvidence } from "../src/ambient.ts";
|
||||
import { createTestEvidence, judgeTestRun } from "../src/test-evidence.ts";
|
||||
import { validate } from "../src/validate.ts";
|
||||
import type { VisionRequest } from "../src/validate.ts";
|
||||
import type { ScreenshotArtifact } from "../src/screenshot.ts";
|
||||
|
||||
function testScreenshot(hash: string): ScreenshotArtifact {
|
||||
return {
|
||||
png: Buffer.from("canned png"),
|
||||
hash,
|
||||
route: "#/workspace/ws_test/session",
|
||||
visibleText: "OpenWork composer",
|
||||
at: "2026-07-29T12:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
test("validate describes, matches, computes failures, and caches", async () => {
|
||||
const unique = randomUUID();
|
||||
const expectations = [`Composer is visible ${unique}`, `No error is visible ${unique}`];
|
||||
const requests: VisionRequest[] = [];
|
||||
const ask = async (request: VisionRequest): Promise<string> => {
|
||||
requests.push(request);
|
||||
if (requests.length === 1) return JSON.stringify({ description: "A composer and an error banner are visible." });
|
||||
return JSON.stringify({
|
||||
results: [
|
||||
{ expectation: expectations[0], passed: true, evidence: "The composer is centered on screen." },
|
||||
{ expectation: expectations[1], passed: false, evidence: "An error banner is visible." },
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
const first = await validate(testScreenshot(unique), expectations, { ask });
|
||||
assert.equal(first.cached, false);
|
||||
assert.equal(first.ok, false);
|
||||
assert.equal(first.description, "A composer and an error banner are visible.");
|
||||
assert.equal(first.results[0]?.passed, true);
|
||||
assert.equal(first.results[1]?.passed, false);
|
||||
assert.match(first.why, /No error is visible/);
|
||||
assert.match(first.why, /An error banner is visible/);
|
||||
assert.equal(requests.length, 2);
|
||||
assert.equal(requests[0]?.prompt.includes(unique), false, "description request must not know the expectations");
|
||||
assert.equal(requests[1]?.prompt.includes(unique), true);
|
||||
|
||||
const second = await validate(testScreenshot(unique), expectations, { ask });
|
||||
assert.equal(second.cached, true);
|
||||
assert.equal(second.ok, false);
|
||||
assert.equal(requests.length, 2, "cache hit must not call the transport");
|
||||
});
|
||||
|
||||
test("validate rejects malformed model verdicts clearly", async () => {
|
||||
let calls = 0;
|
||||
const ask = async (): Promise<string> => {
|
||||
calls += 1;
|
||||
return calls === 1 ? JSON.stringify({ description: "A visible app window." }) : "not-json";
|
||||
};
|
||||
await assert.rejects(
|
||||
() => validate(testScreenshot(randomUUID()), [`Visible app ${randomUUID()}`], { ask }),
|
||||
/Vision model verdict response was not valid JSON/,
|
||||
);
|
||||
});
|
||||
|
||||
test("defer mode judges a caller-provided deterministic witness inline", async () => {
|
||||
const previousMode = process.env.OPENWORK_EVAL_VISION;
|
||||
try {
|
||||
process.env.OPENWORK_EVAL_VISION = "defer";
|
||||
const expectation = `Synthetic screenshot is visible ${randomUUID()}`;
|
||||
let calls = 0;
|
||||
const visualEvidence = await validate(testScreenshot(randomUUID()), [expectation], {
|
||||
bypassCache: true,
|
||||
ask: async () => {
|
||||
calls += 1;
|
||||
return calls === 1
|
||||
? JSON.stringify({ description: "A synthetic screenshot is visible." })
|
||||
: JSON.stringify({ results: [{ expectation, passed: false, evidence: "The expected detail is absent." }] });
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(visualEvidence.ok, false);
|
||||
assert.equal(visualEvidence.results[0]?.passed, false);
|
||||
assert.equal(visualEvidence.deferred, undefined);
|
||||
assert.equal(calls, 2);
|
||||
} finally {
|
||||
if (previousMode === undefined) delete process.env.OPENWORK_EVAL_VISION;
|
||||
else process.env.OPENWORK_EVAL_VISION = previousMode;
|
||||
}
|
||||
});
|
||||
|
||||
test("deferred validation records pending expectations that the judge resolves", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-deferred-vision-"));
|
||||
const previousMode = process.env.OPENWORK_EVAL_VISION;
|
||||
try {
|
||||
const expectation = `Synthetic screenshot is visible ${randomUUID()}`;
|
||||
const screenshot = testScreenshot(randomUUID());
|
||||
const testEvidence = createTestEvidence({ name: "deferred vision", outDir: dir });
|
||||
testEvidence.recordScreenshot(screenshot);
|
||||
process.env.OPENWORK_EVAL_VISION = "defer";
|
||||
const deferred = await withTestEvidence(testEvidence, () => validate(screenshot, [expectation], { bypassCache: true }));
|
||||
assert.equal(deferred.ok, true);
|
||||
assert.equal(deferred.deferred, true);
|
||||
assert.equal(deferred.why, "vision judgment deferred");
|
||||
await testEvidence.close();
|
||||
|
||||
const pendingTestRun = await readFile(join(dir, "test-run.json"), "utf8");
|
||||
assert.match(pendingTestRun, /"state": "pending"/);
|
||||
assert.match(pendingTestRun, /"pendingArtifacts": 1/);
|
||||
assert.match(pendingTestRun, /"pendingJudgments": 1/);
|
||||
|
||||
const providerError = await judgeTestRun(dir, {
|
||||
ask: async () => {
|
||||
throw new Error("stub provider unavailable");
|
||||
},
|
||||
});
|
||||
assert.equal(providerError.judgedValidations, 0);
|
||||
assert.equal(providerError.pendingValidations, 1);
|
||||
assert.match(providerError.errors[0], /stub provider unavailable/);
|
||||
|
||||
const requests: VisionRequest[] = [];
|
||||
const judged = await judgeTestRun(dir, {
|
||||
ask: async (request) => {
|
||||
requests.push(request);
|
||||
return requests.length === 1
|
||||
? JSON.stringify({ description: "A synthetic screenshot is visible." })
|
||||
: JSON.stringify({ results: [{ expectation, passed: true, evidence: "The synthetic screenshot is present." }] });
|
||||
},
|
||||
});
|
||||
assert.equal(judged.judgedValidations, 1);
|
||||
assert.equal(judged.failedValidations, 0);
|
||||
assert.equal(judged.pendingValidations, 0);
|
||||
assert.equal(requests.length, 2);
|
||||
|
||||
const resolvedTestRun = await readFile(join(dir, "test-run.json"), "utf8");
|
||||
assert.match(resolvedTestRun, /"state": "passed"/);
|
||||
assert.match(resolvedTestRun, /"reasoning": "The synthetic screenshot is present\."/);
|
||||
assert.match(resolvedTestRun, /"pendingJudgments": 0/);
|
||||
|
||||
let idempotentCalls = 0;
|
||||
const unchanged = await judgeTestRun(dir, {
|
||||
ask: async () => {
|
||||
idempotentCalls += 1;
|
||||
throw new Error("resolved validations must not be re-judged");
|
||||
},
|
||||
});
|
||||
assert.equal(unchanged.judgedValidations, 0);
|
||||
assert.equal(unchanged.failedValidations, 0);
|
||||
assert.equal(idempotentCalls, 0);
|
||||
|
||||
const cli = spawnSync(process.execPath, [
|
||||
fileURLToPath(new URL("../bin/test-evidence-judge.mjs", import.meta.url)),
|
||||
"--test-run",
|
||||
dir,
|
||||
], { encoding: "utf8" });
|
||||
assert.equal(cli.status, 0, cli.stderr);
|
||||
assert.match(cli.stdout, /Judged 0 visual validation\(s\): 0 failed, 0 pending\./);
|
||||
|
||||
let forcedCalls = 0;
|
||||
const forced = await judgeTestRun(dir, {
|
||||
force: true,
|
||||
ask: async (request) => {
|
||||
forcedCalls += 1;
|
||||
return forcedCalls === 1
|
||||
? JSON.stringify({ description: "A synthetic screenshot is visible." })
|
||||
: JSON.stringify({ results: [{ expectation, passed: false, evidence: "The expected detail is absent." }] });
|
||||
},
|
||||
});
|
||||
assert.equal(forced.judgedValidations, 1);
|
||||
assert.equal(forced.failedValidations, 1);
|
||||
assert.equal(forced.pendingValidations, 0);
|
||||
assert.equal(forcedCalls, 2, "--force must bypass the cached judgment");
|
||||
} finally {
|
||||
if (previousMode === undefined) delete process.env.OPENWORK_EVAL_VISION;
|
||||
else process.env.OPENWORK_EVAL_VISION = previousMode;
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { createVisualEvidence } from "../src/visual-evidence.ts";
|
||||
import type { ScreenshotArtifact } from "../src/screenshot.ts";
|
||||
import type { VisualEvidenceResult } from "../src/validate.ts";
|
||||
|
||||
function screenshotArtifact(contents: string): ScreenshotArtifact {
|
||||
const png = Buffer.from(contents);
|
||||
return {
|
||||
png,
|
||||
hash: createHash("sha256").update(png).digest("hex"),
|
||||
route: `#/workspace/${contents}/session`,
|
||||
visibleText: contents,
|
||||
at: "2026-07-29T12:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
function seen(caption: string, passed: boolean): VisualEvidenceResult {
|
||||
return {
|
||||
ok: passed,
|
||||
description: `Description for ${caption}`,
|
||||
results: [{ expectation: caption, passed, evidence: `${caption} evidence` }],
|
||||
why: passed ? "" : `${caption} failed`,
|
||||
model: "injected-vision-model",
|
||||
cached: false,
|
||||
};
|
||||
}
|
||||
|
||||
test("visual evidence writes distinct screenshots, verdicts, and idempotent indexes", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-visual-evidence-"));
|
||||
try {
|
||||
const visualEvidence = createVisualEvidence("unit test run", { outDir: dir });
|
||||
const firstPath = await visualEvidence.recordScreenshot(screenshotArtifact("first"), seen("First caption", true));
|
||||
const secondPath = await visualEvidence.recordScreenshot(screenshotArtifact("second"), seen("Second caption", false));
|
||||
|
||||
const firstClose = await visualEvidence.close();
|
||||
const secondClose = await visualEvidence.close();
|
||||
await stat(firstPath);
|
||||
await stat(secondPath);
|
||||
assert.equal(firstClose, join(dir, "index.html"));
|
||||
assert.equal(secondClose, firstClose);
|
||||
|
||||
const index = await readFile(join(dir, "index.html"), "utf8");
|
||||
const json = await readFile(join(dir, "test-run.json"), "utf8");
|
||||
for (const expected of ["First caption", "Second caption", "PASS", "FAIL"]) {
|
||||
assert.match(index, new RegExp(expected));
|
||||
}
|
||||
for (const expected of ["First caption", "Second caption", '"passed": true', '"passed": false']) {
|
||||
assert.match(json, new RegExp(expected));
|
||||
}
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("visual evidence rejects duplicate pixels assigned to different validations", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "openwork-visual-evidence-duplicate-"));
|
||||
try {
|
||||
const visualEvidence = createVisualEvidence("duplicate test run", { outDir: dir });
|
||||
const duplicate = screenshotArtifact("same pixels");
|
||||
await visualEvidence.recordScreenshot(duplicate, seen("Original screenshot", true));
|
||||
await assert.rejects(
|
||||
() => visualEvidence.recordScreenshot(duplicate, seen("Copied screenshot", true)),
|
||||
/Copied screenshot.*Original screenshot/,
|
||||
);
|
||||
await visualEvidence.close();
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user