/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import { execFileSync } from 'node:child_process'; import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from 'node:fs'; import { basename, join, relative, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; const PROCESS_TIMEOUT_MS = 10 * 60 * 1000; const MAX_OUTPUT_BYTES = 2 * 1024 * 1024; const MAX_DIAGNOSTIC_TEXT = 4_000; export function findEvalReleaseTarball( releaseDirectory, platform = process.platform, arch = process.arch, ) { if (platform !== 'linux' || arch !== 'x64') { throw new Error('The real Eval release validation requires Linux x64'); } return findReleaseTarball(releaseDirectory); } export function findReleaseTarball(releaseDirectory) { const tarballs = readdirSync(releaseDirectory) .filter((name) => /^maka-agent-[^/]+\.tgz$/u.test(name)) .map((name) => join(releaseDirectory, name)); if (tarballs.length !== 1) { throw new Error( `Expected one release tarball in ${releaseDirectory}, found ${tarballs.length}`, ); } return tarballs[0]; } export function isolatedEnvironment(home) { mkdirSync(home, { recursive: true, mode: 0o700 }); const environment = { ...process.env, HOME: home, NODE_PATH: '', XDG_CACHE_HOME: join(home, '.cache'), XDG_CONFIG_HOME: join(home, '.config'), XDG_DATA_HOME: join(home, '.local/share'), }; delete environment.PYTHONHOME; delete environment.PYTHONPATH; for (const name of Object.keys(environment)) { if (name.startsWith('MAKA_EVAL_') || name.startsWith('GIT_')) delete environment[name]; } environment.GIT_CONFIG_NOSYSTEM = '1'; for (const name of [ 'ANTHROPIC_API_KEY', 'DEEPSEEK_API_KEY', 'OPENAI_API_KEY', 'OPENROUTER_API_KEY', ]) { delete environment[name]; } return environment; } export function createTaskFixture(root, environment, taskImage) { const repositoryRoot = join(root, 'repository'); const taskRoot = join(repositoryRoot, 'task'); mkdirSync(join(taskRoot, 'environment'), { recursive: true, mode: 0o700 }); mkdirSync(join(taskRoot, 'tests'), { recursive: true, mode: 0o700 }); writeFileSync( join(taskRoot, 'task.toml'), [ 'version = "1.0"', '', '[metadata]', '', '[verifier]', 'timeout_sec = 60.0', '', '[agent]', 'timeout_sec = 60.0', '', '[environment]', 'build_timeout_sec = 120.0', '', ].join('\n'), { mode: 0o600 }, ); writeFileSync( join(taskRoot, 'instruction.md'), 'Exit successfully without modifying the task.\n', { mode: 0o600 }, ); writeFileSync(join(taskRoot, 'environment/Dockerfile'), `FROM ${taskImage}\nWORKDIR /app\n`, { mode: 0o600, }); const testPath = join(taskRoot, 'tests/test.sh'); writeFileSync(testPath, '#!/bin/sh\nset -eu\nprintf "1\\n" > /logs/verifier/reward.txt\n', { mode: 0o700, }); chmodSync(testPath, 0o700); run('git', ['init', '--initial-branch=main'], environment, repositoryRoot); run('git', ['config', 'user.name', 'Maka Release Validation'], environment, repositoryRoot); run( 'git', ['config', 'user.email', 'release-validation@maka.invalid'], environment, repositoryRoot, ); run('git', ['add', '.'], environment, repositoryRoot); run( 'git', ['commit', '--message', 'Add deterministic release validation task'], environment, repositoryRoot, ); const commit = run('git', ['rev-parse', 'HEAD'], environment, repositoryRoot).trim(); return { commit, repository: pathToFileURL(repositoryRoot).href, tasksRoot: repositoryRoot, }; } export function readFrameworkOutputs({ framework, invocation, outputRoot, trialsRoot }) { if (invocation.status !== 0) { throw frameworkOutputError( framework, `exited with status ${invocation.status}`, invocation, [], findJsonDiagnostics(trialsRoot), ); } const attempts = findAttemptFiles(join(outputRoot, 'attempts')); const attemptNames = attempts.map((path) => relative(outputRoot, path).split(sep).join('/')); if (attempts.length !== 1) { throw frameworkOutputError( framework, `produced ${attempts.length} attempt files instead of one`, invocation, attemptNames, findJsonDiagnostics(trialsRoot), ); } const summary = decodeJsonRecord(invocation.stdout); if (!summary.ok) { throw frameworkOutputError( framework, 'produced invalid summary JSON', invocation, attemptNames, findJsonDiagnostics(trialsRoot), { summary: summary.evidence }, ); } const attempt = decodeJsonRecord(readFileSync(attempts[0], 'utf8')); if (!attempt.ok) { throw frameworkOutputError( framework, 'produced invalid attempt JSON', invocation, attemptNames, findJsonDiagnostics(trialsRoot), { attempt: attempt.evidence }, ); } return { summary: summary.value, attempt: attempt.value }; } export function findJsonDiagnostics(root, current = root, diagnostics = {}) { if (!existsSync(current)) return diagnostics; let entries; try { entries = readdirSync(current, { withFileTypes: true }); } catch (error) { const name = relative(root, current).split(sep).join('/') || '.'; diagnostics[name] = diagnosticIoError('list', error); return diagnostics; } for (const entry of entries) { const path = join(current, entry.name); if (entry.isDirectory()) { findJsonDiagnostics(root, path, diagnostics); continue; } if (!['preparation-error.json', 'result.json', 'trial.log'].includes(entry.name)) continue; const name = relative(root, path).split(sep).join('/'); let content; try { content = readFileSync(path, 'utf8').trim(); } catch (error) { diagnostics[name] = diagnosticIoError('read', error); continue; } if (entry.name === 'trial.log') { diagnostics[name] = tail(content); continue; } const decoded = decodeJson(content); if (!decoded.ok) { diagnostics[name] = decoded.evidence; continue; } const parsed = decoded.value; diagnostics[name] = entry.name === 'result.json' && isRecord(parsed) ? { exceptionInfo: isRecord(parsed.exception_info) ? { type: parsed.exception_info.exception_type, message: tail(String(parsed.exception_info.exception_message ?? '')), } : null, verifierResult: parsed.verifier_result ?? null, } : parsed; } return diagnostics; } export function run(command, args, environment, cwd) { try { return execFileSync(command, args, { cwd, env: environment, encoding: 'utf8', timeout: PROCESS_TIMEOUT_MS, maxBuffer: MAX_OUTPUT_BYTES, stdio: ['ignore', 'pipe', 'pipe'], }); } catch (error) { const stdout = String(error.stdout ?? '').trim(); const stderr = String(error.stderr ?? '').trim(); throw new Error( `${command} ${args.join(' ')} failed${stdout ? `\nstdout:\n${stdout}` : ''}${stderr ? `\nstderr:\n${stderr}` : ''}`, { cause: error }, ); } } export function runAllowingFailure(command, args, environment, cwd) { try { return { status: 0, stdout: run(command, args, environment, cwd), stderr: '' }; } catch (error) { const cause = error.cause; if (typeof cause?.status !== 'number') throw error; return { status: cause.status, stdout: String(cause.stdout ?? ''), stderr: String(cause.stderr ?? ''), }; } } function findAttemptFiles(root) { if (!existsSync(root)) return []; const files = []; for (const entry of readdirSync(root, { withFileTypes: true })) { const path = join(root, entry.name); if (entry.isDirectory()) files.push(...findAttemptFiles(path)); else if (/^\d{6}\.json$/u.test(entry.name)) files.push(path); } return files.sort(); } function frameworkOutputError( framework, reason, invocation, attempts, frameworkDiagnostics, extra = {}, ) { return new Error( `${framework} ${reason}: ${JSON.stringify({ invocation: { status: invocation.status, stdout: tail(invocation.stdout), stderr: tail(invocation.stderr), }, attempts, ...extra, frameworkDiagnostics, })}`, ); } function decodeJsonRecord(content) { const decoded = decodeJson(content); if (!decoded.ok) return decoded; if (!isRecord(decoded.value)) { return { ok: false, evidence: { parseError: 'expected a JSON object', raw: tail(String(content).trim()) }, }; } return decoded; } function decodeJson(content) { const raw = String(content).trim(); try { return { ok: true, value: JSON.parse(raw) }; } catch (error) { return { ok: false, evidence: { parseError: error instanceof Error ? error.message : String(error), raw: tail(raw), }, }; } } function isRecord(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } function diagnosticIoError(operation, error) { const code = typeof error?.code === 'string' ? error.code : 'UNKNOWN'; return { diagnosticError: { operation, code } }; } function tail(value) { return String(value).slice(-MAX_DIAGNOSTIC_TEXT); }