源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

.github/workflows/nightly-flake-report.yml

@@ -0,0 +1,119 @@
+# Quantifies test reliability: runs the hermetic pr lane several times,
+# aggregates per-spec pass rates with evals/scripts/flake-report.mjs, publishes
+# the report to the run summary and artifacts, and fails when any spec is
+# flaky or below the pass-rate threshold so the failure-alert workflow files
+# a developer-facing issue.
+name: Nightly Flake Report
+
+on:
+  schedule:
+    - cron: "17 4 * * *"
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: "How many times to run the pr lane"
+        required: false
+        default: "3"
+        type: string
+      threshold:
+        description: "Quarantine pass-rate threshold (0-1]"
+        required: false
+        default: "0.95"
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nightly-flake-report
+  cancel-in-progress: false
+
+jobs:
+  flake-report:
+    if: github.repository == 'different-ai/openwork'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    timeout-minutes: 120
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+
+      - name: Setup Node
+        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
+        with:
+          node-version: 24
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
+        with:
+          version: 11.4.0
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          set -euo pipefail
+          pnpm install --frozen-lockfile
+          pnpm --dir evals install --frozen-lockfile
+
+      - name: Collect pr-lane runs
+        shell: bash
+        working-directory: evals
+        env:
+          FLAKE_RUNS: ${{ inputs.runs || '3' }}
+        run: |
+          set -euo pipefail
+          for index in $(seq 1 "$FLAKE_RUNS"); do
+            echo "::group::flake:collect run $index of $FLAKE_RUNS"
+            node scripts/flake-report.mjs collect --project pr
+            echo "::endgroup::"
+          done
+
+      - name: Aggregate flake report
+        shell: bash
+        working-directory: evals
+        env:
+          FLAKE_THRESHOLD: ${{ inputs.threshold || '0.95' }}
+        run: |
+          set -euo pipefail
+          node scripts/flake-report.mjs report --project pr --threshold "$FLAKE_THRESHOLD"
+          cat results/flake/pr/flake-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Publish scenario coverage matrix
+        shell: bash
+        working-directory: evals
+        run: |
+          set -euo pipefail
+          node scripts/spec-coverage-matrix.mjs --out results/flake/spec-coverage-matrix.md
+          {
+            echo ""
+            head -3 results/flake/spec-coverage-matrix.md
+            echo ""
+            echo "Full matrix in the run artifacts."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload flake report artifacts
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: nightly-flake-report
+          path: evals/results/flake/
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Fail on flaky or quarantined specs
+        shell: bash
+        working-directory: evals
+        run: |
+          set -euo pipefail
+          flaky="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).flakySpecs")"
+          quarantine="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).quarantineSpecs")"
+          if [ "$flaky" != "0" ] || [ "$quarantine" != "0" ]; then
+            echo "::error::Flake report found $flaky flaky and $quarantine below-threshold specs. See the run summary."
+            exit 1
+          fi
+          echo "All specs stable across the collected runs."