# Quantifies test reliability: runs the hermetic pr lane several times,
# aggregates per-spec pass rates with evals/scripts/flake-report.mjs, publishes
# the report to the run summary and artifacts, and fails when any spec is
# flaky or below the pass-rate threshold so the failure-alert workflow files
# a developer-facing issue.
name: Nightly Flake Report

on:
  schedule:
    - cron: "17 4 * * *"
  workflow_dispatch:
    inputs:
      runs:
        description: "How many times to run the pr lane"
        required: false
        default: "3"
        type: string
      threshold:
        description: "Quarantine pass-rate threshold (0-1]"
        required: false
        default: "0.95"
        type: string

permissions:
  contents: read

concurrency:
  group: nightly-flake-report
  cancel-in-progress: false

jobs:
  flake-report:
    if: github.repository == 'different-ai/openwork'
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 120

    steps:
      - name: Checkout
        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0

      - name: Setup Node
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
        with:
          node-version: 24

      - name: Setup pnpm
        uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
        with:
          version: 11.4.0

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
        with:
          bun-version: 1.3.14

      - name: Install dependencies
        shell: bash
        run: |
          set -euo pipefail
          pnpm install --frozen-lockfile
          pnpm --dir evals install --frozen-lockfile

      - name: Collect pr-lane runs
        shell: bash
        working-directory: evals
        env:
          FLAKE_RUNS: ${{ inputs.runs || '3' }}
        run: |
          set -euo pipefail
          for index in $(seq 1 "$FLAKE_RUNS"); do
            echo "::group::flake:collect run $index of $FLAKE_RUNS"
            node scripts/flake-report.mjs collect --project pr
            echo "::endgroup::"
          done

      - name: Aggregate flake report
        shell: bash
        working-directory: evals
        env:
          FLAKE_THRESHOLD: ${{ inputs.threshold || '0.95' }}
        run: |
          set -euo pipefail
          node scripts/flake-report.mjs report --project pr --threshold "$FLAKE_THRESHOLD"
          cat results/flake/pr/flake-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Publish scenario coverage matrix
        shell: bash
        working-directory: evals
        run: |
          set -euo pipefail
          node scripts/spec-coverage-matrix.mjs --out results/flake/spec-coverage-matrix.md
          {
            echo ""
            head -3 results/flake/spec-coverage-matrix.md
            echo ""
            echo "Full matrix in the run artifacts."
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload flake report artifacts
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: nightly-flake-report
          path: evals/results/flake/
          retention-days: 30
          if-no-files-found: warn

      - name: Fail on flaky or quarantined specs
        shell: bash
        working-directory: evals
        run: |
          set -euo pipefail
          flaky="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).flakySpecs")"
          quarantine="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).quarantineSpecs")"
          if [ "$flaky" != "0" ] || [ "$quarantine" != "0" ]; then
            echo "::error::Flake report found $flaky flaky and $quarantine below-threshold specs. See the run summary."
            exit 1
          fi
          echo "All specs stable across the collected runs."