120 lines
3.8 KiB
YAML
120 lines
3.8 KiB
YAML
# Quantifies test reliability: runs the hermetic pr lane several times,
|
|
# aggregates per-spec pass rates with evals/scripts/flake-report.mjs, publishes
|
|
# the report to the run summary and artifacts, and fails when any spec is
|
|
# flaky or below the pass-rate threshold so the failure-alert workflow files
|
|
# a developer-facing issue.
|
|
name: Nightly Flake Report
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "17 4 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
runs:
|
|
description: "How many times to run the pr lane"
|
|
required: false
|
|
default: "3"
|
|
type: string
|
|
threshold:
|
|
description: "Quarantine pass-rate threshold (0-1]"
|
|
required: false
|
|
default: "0.95"
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: nightly-flake-report
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
flake-report:
|
|
if: github.repository == 'different-ai/openwork'
|
|
runs-on: blacksmith-4vcpu-ubuntu-2404
|
|
timeout-minutes: 120
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
|
|
|
- name: Setup Node
|
|
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
|
|
with:
|
|
version: 11.4.0
|
|
|
|
- name: Setup Bun
|
|
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.3.14
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
pnpm install --frozen-lockfile
|
|
pnpm --dir evals install --frozen-lockfile
|
|
|
|
- name: Collect pr-lane runs
|
|
shell: bash
|
|
working-directory: evals
|
|
env:
|
|
FLAKE_RUNS: ${{ inputs.runs || '3' }}
|
|
run: |
|
|
set -euo pipefail
|
|
for index in $(seq 1 "$FLAKE_RUNS"); do
|
|
echo "::group::flake:collect run $index of $FLAKE_RUNS"
|
|
node scripts/flake-report.mjs collect --project pr
|
|
echo "::endgroup::"
|
|
done
|
|
|
|
- name: Aggregate flake report
|
|
shell: bash
|
|
working-directory: evals
|
|
env:
|
|
FLAKE_THRESHOLD: ${{ inputs.threshold || '0.95' }}
|
|
run: |
|
|
set -euo pipefail
|
|
node scripts/flake-report.mjs report --project pr --threshold "$FLAKE_THRESHOLD"
|
|
cat results/flake/pr/flake-report.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Publish scenario coverage matrix
|
|
shell: bash
|
|
working-directory: evals
|
|
run: |
|
|
set -euo pipefail
|
|
node scripts/spec-coverage-matrix.mjs --out results/flake/spec-coverage-matrix.md
|
|
{
|
|
echo ""
|
|
head -3 results/flake/spec-coverage-matrix.md
|
|
echo ""
|
|
echo "Full matrix in the run artifacts."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload flake report artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: nightly-flake-report
|
|
path: evals/results/flake/
|
|
retention-days: 30
|
|
if-no-files-found: warn
|
|
|
|
- name: Fail on flaky or quarantined specs
|
|
shell: bash
|
|
working-directory: evals
|
|
run: |
|
|
set -euo pipefail
|
|
flaky="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).flakySpecs")"
|
|
quarantine="$(node -p "JSON.parse(require('node:fs').readFileSync('results/flake/pr/flake-report.json', 'utf8')).quarantineSpecs")"
|
|
if [ "$flaky" != "0" ] || [ "$quarantine" != "0" ]; then
|
|
echo "::error::Flake report found $flaky flaky and $quarantine below-threshold specs. See the run summary."
|
|
exit 1
|
|
fi
|
|
echo "All specs stable across the collected runs."
|