import { execFileSync } from "node:child_process" import { appendFileSync, readFileSync } from "node:fs" import { resolve } from "node:path" import { fileURLToPath } from "node:url" const repoRoot = fileURLToPath(new URL("../..", import.meta.url)) const defaultSnapshot = resolve(repoRoot, "evals/specs/contracts.snapshot.json") function matches(pathname, pattern) { if (pattern.endsWith("/**")) return pathname.startsWith(pattern.slice(0, -3)) return pathname === pattern } function strings(value, label) { if (!Array.isArray(value) || value.length === 0 || value.some((entry) => typeof entry !== "string" || entry.length === 0)) { throw new Error(`${label} must be a non-empty string array`) } return value } export function validateSnapshot(value) { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("snapshot must be an object") if (Reflect.get(value, "version") !== 1) throw new Error("snapshot version must be 1") const rawContracts = Reflect.get(value, "contracts") if (!Array.isArray(rawContracts) || rawContracts.length === 0) throw new Error("snapshot contracts must be a non-empty array") const ids = new Set() return rawContracts.map((contract, index) => { if (typeof contract !== "object" || contract === null || Array.isArray(contract)) { throw new Error(`contracts[${index}] must be an object`) } const id = Reflect.get(contract, "id") const description = Reflect.get(contract, "description") if (typeof id !== "string" || id.length === 0) throw new Error(`contracts[${index}].id must be a string`) if (ids.has(id)) throw new Error(`duplicate contract id: ${id}`) ids.add(id) if (typeof description !== "string" || description.length === 0) { throw new Error(`contracts[${index}].description must be a string`) } return { id, description, implementation: strings(Reflect.get(contract, "implementation"), `${id}.implementation`), specs: strings(Reflect.get(contract, "specs"), `${id}.specs`), } }) } export function analyzeImpact(contracts, changedFiles) { const changed = [...new Set(changedFiles.map((entry) => entry.trim()).filter(Boolean))].sort() const changedE2eTests = changed.filter((pathname) => pathname.startsWith("evals/specs/") && pathname.endsWith(".e2e.test.ts")) const impacted = contracts.flatMap((contract) => { const implementation = changed.filter((pathname) => contract.implementation.some((pattern) => matches(pathname, pattern))) if (implementation.length === 0) return [] const specs = changed.filter((pathname) => contract.specs.includes(pathname)) return [{ ...contract, changedImplementation: implementation, changedSpecs: specs, covered: specs.length > 0 }] }) return { changed, changedE2eTests, impacted, attention: impacted.filter((contract) => !contract.covered), matchedTests: [...new Set([...impacted.flatMap((contract) => contract.specs), ...changedE2eTests])].sort(), } } function markdown(result) { const lines = ["### Soft spec-impact snapshot", ""] if (result.impacted.length === 0) { lines.push( "No mapped implementation contract changed.", "Warden suggestion: add or update an `evals/specs/.e2e.test.ts` test for this change if it affects app behavior.", "", ) return lines.join("\n") } lines.push("| Contract | Result | Changed implementation | Mapped E2E tests |", "| --- | --- | --- | --- |") for (const contract of result.impacted) { const status = contract.covered ? "Covered by a changed E2E test" : "Needs attention" const implementation = contract.changedImplementation.map((entry) => `\`${entry}\``).join("
") const specs = contract.specs.map((entry) => `\`${entry}\``).join("
") lines.push(`| \`${contract.id}\` | ${status} | ${implementation} | ${specs} |`) } lines.push("") if (result.attention.length > 0) { lines.push("This check is advisory: confirm the existing mapped E2E test still proves the contract, or update/add an E2E test.", "") } lines.push(`Matched E2E tests: ${result.matchedTests.map((test) => `\`${test}\``).join(", ") || "none"}`, "") return lines.join("\n") } function annotation(value) { return value.replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A") } function parseArgs(args) { const options = { changedFiles: [] } for (let index = 0; index < args.length; index += 1) { const argument = args[index] if (argument === "--strict") options.strict = true else if (argument === "--json") options.json = true else if (argument === "--matched-tests") options.matchedTests = true else if (["--base", "--head", "--snapshot", "--summary", "--changed-file"].includes(argument)) { const value = args[index + 1] if (!value) throw new Error(`${argument} requires a value`) index += 1 if (argument === "--changed-file") options.changedFiles.push(value) else options[argument.slice(2)] = value } else throw new Error(`unknown argument: ${argument}`) } return options } function gitChangedFiles(base, head) { if (!base || !head) throw new Error("pass both --base and --head, or one or more --changed-file values") return execFileSync("git", ["diff", "--name-only", "--diff-filter=ACMR", `${base}...${head}`], { cwd: repoRoot, encoding: "utf8", }).split("\n").filter(Boolean) } function main() { const options = parseArgs(process.argv.slice(2)) const snapshotPath = resolve(repoRoot, options.snapshot ?? defaultSnapshot) const snapshot = JSON.parse(readFileSync(snapshotPath, "utf8")) const contracts = validateSnapshot(snapshot) const changedFiles = options.changedFiles.length > 0 ? options.changedFiles : gitChangedFiles(options.base, options.head) const result = analyzeImpact(contracts, changedFiles) const report = markdown(result) if (options.summary) appendFileSync(options.summary, report) if (options.matchedTests) { process.stdout.write(`${JSON.stringify(result.matchedTests)}\n`) return } if (options.json) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`) else process.stdout.write(report) for (const contract of result.attention) { process.stdout.write(`::warning title=Spec impact snapshot::${annotation(`${contract.id} changed without a mapped E2E test change`)}\n`) } if (options.strict && result.attention.length > 0) process.exitCode = 2 } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main()