源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)

scripts/release-cli-eval-package.mjs

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join, resolve } from 'node:path';
+import {
+  createTaskFixture,
+  findEvalReleaseTarball,
+  findJsonDiagnostics,
+  isolatedEnvironment,
+  readFrameworkOutputs,
+  run,
+  runAllowingFailure,
+} from './release-cli-eval-support.mjs';
+
+const HARBOR_VERSION = '0.20.0';
+const PIER_VERSION = '0.3.0';
+const TASK_IMAGE =
+  'python:3.12-slim@sha256:dd29372629eeba2dd003fd9e9d35a5b8236c44727875a0364254b5127af88e65';
+const releaseDirectory = resolve('packages/cli/release');
+const tarballPath = findEvalReleaseTarball(releaseDirectory);
+
+const root = mkdtempSync(join(tmpdir(), 'maka-cli-eval-validation-'));
+let primaryError;
+try {
+  validateChecksum();
+  const prefix = join(root, 'prefix');
+  const environment = isolatedEnvironment(join(root, 'home'));
+  logStep('installing the immutable tarball with an empty offline npm cache');
+  run(
+    'npm',
+    [
+      'install',
+      '--global',
+      '--ignore-scripts',
+      '--offline',
+      '--no-audit',
+      '--no-fund',
+      '--cache',
+      join(root, 'npm-cache'),
+      '--prefix',
+      prefix,
+      tarballPath,
+    ],
+    environment,
+    root,
+  );
+
+  const maka = join(prefix, 'bin/maka');
+  const packageRoot = join(prefix, 'lib/node_modules/maka-agent');
+  if (
+    !existsSync(maka) ||
+    !existsSync(join(packageRoot, 'node_modules/@maka/eval/harbor/run_trial.py'))
+  ) {
+    throw new Error('The installed candidate is missing its CLI or bundled Eval runtime');
+  }
+
+  const fixture = createTaskFixture(join(root, 'fixture'), environment, TASK_IMAGE);
+  for (const framework of ['harbor', 'pier']) {
+    logStep(`running a real ${framework} Docker cell from the installed candidate`);
+    validateFramework({
+      environment,
+      fixture,
+      framework,
+      maka,
+      root: join(root, framework),
+    });
+  }
+  logStep(`OK — installed ${basename(tarballPath)} completed real Harbor and Pier cells`);
+} catch (error) {
+  primaryError = error;
+} finally {
+  let cleanupError;
+  try {
+    rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
+  } catch (error) {
+    cleanupError = error;
+  }
+  if (primaryError && cleanupError) {
+    throw new AggregateError([primaryError, cleanupError], 'Eval validation and cleanup failed');
+  }
+  if (primaryError) throw primaryError;
+  if (cleanupError) throw cleanupError;
+}
+
+function validateFramework({ environment, fixture, framework, maka, root: frameworkRoot }) {
+  mkdirSync(frameworkRoot, { recursive: true, mode: 0o700 });
+  const outputRoot = join(frameworkRoot, 'output');
+  const trialsRoot = join(frameworkRoot, 'trials');
+  const pythonVariable =
+    framework === 'harbor' ? 'MAKA_RELEASE_HARBOR_PYTHON' : 'MAKA_RELEASE_PIER_PYTHON';
+  const pythonPath = process.env[pythonVariable];
+  if (!pythonPath) throw new Error(`${pythonVariable} is required`);
+  const pythonPathVariable = `MAKA_RELEASE_${framework.toUpperCase()}_PYTHON_PATH`;
+  const trialsVariable = `MAKA_RELEASE_${framework.toUpperCase()}_TRIALS`;
+  const tasksVariable = 'MAKA_RELEASE_PIER_TASKS';
+  const specPath = join(frameworkRoot, 'experiment.json');
+  const spec = experimentSpec({
+    fixture,
+    framework,
+    pythonPathVariable,
+    tasksVariable,
+    trialsVariable,
+  });
+  writeFileSync(specPath, `${JSON.stringify(spec)}\n`, { mode: 0o600 });
+  const childEnvironment = {
+    ...environment,
+    [pythonPathVariable]: pythonPath,
+    [trialsVariable]: trialsRoot,
+    ...(framework === 'pier' ? { [tasksVariable]: fixture.tasksRoot } : {}),
+  };
+  const invocation = runAllowingFailure(
+    maka,
+    ['eval', 'run', specPath, '--out', outputRoot],
+    childEnvironment,
+    frameworkRoot,
+  );
+  const { summary, attempt } = readFrameworkOutputs({
+    framework,
+    invocation,
+    outputRoot,
+    trialsRoot,
+  });
+  const result = attempt.result;
+  if (
+    summary.experimentId !== `release-${framework}` ||
+    summary.cells !== 1 ||
+    summary.incomplete !== 0 ||
+    attempt.cellId !== 'task::1::subject' ||
+    attempt.sequence !== 1 ||
+    result?.status !== 'completed' ||
+    result?.score !== 1 ||
+    result?.usage !== null ||
+    result?.costUsd !== null ||
+    result?.failureReason !== null
+  ) {
+    throw new Error(
+      `${framework} produced an invalid completed attempt: ${JSON.stringify({
+        invocation,
+        summary,
+        attempt,
+        frameworkDiagnostics: findJsonDiagnostics(trialsRoot),
+      })}`,
+    );
+  }
+  const trialArtifact = result.artifacts.find(
+    (artifact) => artifact.kind === 'trial' && artifact.framework === framework,
+  );
+  const processArtifact = result.artifacts.find(
+    (artifact) => artifact.kind === 'external_process' && artifact.exitCode === 0,
+  );
+  const collected = result.artifacts.filter((artifact) => artifact.kind === 'collected-artifact');
+  if (!trialArtifact || !processArtifact || collected.length < 2) {
+    throw new Error(`${framework} did not preserve the expected trial and subject artifacts`);
+  }
+  const containers = run(
+    'docker',
+    ['ps', '--all', '--quiet', '--filter', `name=${trialArtifact.trialName}`],
+    childEnvironment,
+    frameworkRoot,
+  ).trim();
+  if (containers) throw new Error(`${framework} left trial containers behind: ${containers}`);
+}
+
+function experimentSpec({ fixture, framework, pythonPathVariable, tasksVariable, trialsVariable }) {
+  return {
+    schemaVersion: 'maka.eval.v1',
+    id: `release-${framework}`,
+    benchmark: {
+      id: 'release-validation',
+      version: fixture.commit,
+      config: { repository: fixture.repository },
+    },
+    executor: {
+      kind: framework,
+      config: {
+        frameworkVersion: framework === 'harbor' ? HARBOR_VERSION : PIER_VERSION,
+        pythonPathEnv: pythonPathVariable,
+        trialsRootEnv: trialsVariable,
+        ...(framework === 'pier' ? { tasksRootEnv: tasksVariable } : {}),
+        environment: { type: 'docker', delete: true },
+        preparationEnvironment: [],
+        mounts: [],
+      },
+    },
+    subjects: [
+      {
+        id: 'subject',
+        kind: 'external',
+        credentials: [],
+        config: { command: '/bin/true', args: [], result: 'exit-code' },
+      },
+    ],
+    tasks: [
+      {
+        id: 'task',
+        input: 'Exit successfully without modifying the task.',
+        config: framework === 'harbor' ? { harbor: { path: 'task' } } : { pier: { path: 'task' } },
+      },
+    ],
+    repetitions: 1,
+    budget: { timeoutMultiplier: 1 },
+    verifier: { reward: 'reward' },
+  };
+}
+
+function validateChecksum() {
+  const checksumPath = `${tarballPath}.sha256`;
+  const [expected, name, extra] = readFileSync(checksumPath, 'utf8').trim().split(/\s+/u);
+  if (!expected || name !== basename(tarballPath) || extra !== undefined) {
+    throw new Error(`Invalid checksum file ${checksumPath}`);
+  }
+  const actual = createHash('sha256').update(readFileSync(tarballPath)).digest('hex');
+  if (actual !== expected) throw new Error(`Checksum mismatch for ${tarballPath}`);
+}
+
+function logStep(message) {
+  console.log(`[release-cli-eval-validation] ${message}`);
+}