# Runs the eligible E2E tests in three balanced Daytona batches. Each batch keeps
# one prewarmed Den + desktop pair and reuses it across its assigned tests.
# The checked-in lane profile removes only topology/dependency skips before the
# matrix is built; compatible product regressions remain in the matrix.
# The critical-path journey remains owned by its dedicated nightly workflow.
# Tests that drive raw desktop() from @openwork/hosts are excluded dynamically:
# they spawn Electron on the caller's machine (correct inside a VNC sandbox or
# on a dev workstation, impossible on a headless runner) and need a follow-up
# in-sandbox vitest lane. The plan job lists every exclusion in its summary.
name: Daytona E2E Regression Suite

on:
  workflow_run:
    workflows: [Warden]
    types: [completed]
  workflow_dispatch:
    inputs:
      only:
        description: "Substring filter on E2E test filenames; empty runs all eligible tests"
        required: false
        default: ""
        type: string
permissions:
  actions: read
  contents: read
  pull-requests: read

concurrency:
  group: daytona-e2e-regression-suite-${{ github.event.workflow_run.pull_requests[0].number || github.run_id }}
  cancel-in-progress: true

jobs:
  plan:
    if: >-
      github.repository == 'different-ai/openwork' &&
      (github.event_name == 'workflow_dispatch' ||
      (github.event.workflow_run.event == 'pull_request' &&
      github.event.workflow_run.conclusion == 'success' &&
      github.event.workflow_run.head_repository.full_name == github.repository))
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 10
    outputs:
      enabled: ${{ steps.config.outputs.enabled }}
      batches: ${{ steps.matrix.outputs.batches }}
      has_tests: ${{ steps.matrix.outputs.has_tests }}

    steps:
      - name: Authorize Warden-cleared pull request
        id: authorize
        if: github.event_name == 'workflow_run'
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.event.workflow_run.id }}
          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
          PR: ${{ github.event.workflow_run.pull_requests[0].number }}
        run: |
          set -euo pipefail

          mkdir -p /tmp/warden-summary
          gh run download "$RUN_ID" --repo "$REPO" --name warden-summary --dir /tmp/warden-summary
          summary_sha="$(jq -r '.head_sha' /tmp/warden-summary/warden-summary.json)"
          blocking="$(jq -r '.blocking_count // .findings_count' /tmp/warden-summary/warden-summary.json)"
          current_head="$(gh api "repos/$REPO/pulls/$PR" --jq '.head.sha')"

          if [ "$summary_sha" != "$HEAD_SHA" ] || [ "$current_head" != "$HEAD_SHA" ]; then
            echo "::error::Warden summary or pull request head does not match $HEAD_SHA."
            exit 1
          fi
          if [ "$blocking" != "0" ]; then
            echo "::notice::E2E regression withheld: Warden reported $blocking blocking finding(s)."
            echo "authorized=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          guarded="$(gh api --paginate "repos/$REPO/pulls/$PR/files" --jq '.[].filename' \
            | grep -E '^(\.github/|warden\.toml$|\.warden/|\.agents/skills/|\.claude/skills/)' || true)"
          if [ -n "$guarded" ]; then
            echo "::notice::E2E regression withheld: PR changes trusted review machinery."
            echo "authorized=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          echo "authorized=true" >> "$GITHUB_OUTPUT"

      - name: Authorize manual E2E regression
        id: authorize-manual
        if: github.event_name == 'workflow_dispatch'
        run: echo "authorized=true" >> "$GITHUB_OUTPUT"

      - name: Check E2E regression configuration
        id: config
        if: steps.authorize.outputs.authorized == 'true' || steps.authorize-manual.outputs.authorized == 'true'
        shell: bash
        env:
          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -euo pipefail

          missing=""
          [ -z "${DAYTONA_API_KEY:-}" ] && missing="$missing DAYTONA_API_KEY"
          [ -z "${OPENAI_API_KEY:-}" ] && missing="$missing OPENAI_API_KEY"
          if [ -n "$missing" ]; then
            echo "::notice::Skipping Daytona E2E regression suite; missing GitHub configuration:$missing."
            echo "enabled=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          echo "enabled=true" >> "$GITHUB_OUTPUT"

      - name: Checkout
        if: steps.config.outputs.enabled == 'true'
        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
        with:
          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Build E2E test matrix
        id: matrix
        if: steps.config.outputs.enabled == 'true'
        shell: bash
        env:
          ONLY_FILTER: ${{ inputs.only }}
          EVENT_NAME: ${{ github.event_name }}
          BASE_SHA: ${{ github.event.workflow_run.pull_requests[0].base.sha }}
          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
        run: |
          set -euo pipefail

          # Raw desktop() tests spawn Electron on the Vitest host itself; on a
          # headless runner that dies before CDP. Exclude them until the
          # in-sandbox vitest lane exists, and surface the list in the summary.
          raw_desktop="$(node evals/scripts/list-daytona-raw-desktop-tests.mjs)"
          # The checked-in profile is the explicit contract for this lane.
          # Raw desktop() detection remains as a guard for newly added tests
          # that have not been classified yet.
          profile='evals/specs/daytona-e2e-regression-profile.json'
          profile_excluded_json="$(jq -c '[.excluded[].test]' "$profile")"
          raw_desktop_json="$(printf '%s\n' "$raw_desktop" | jq -Rsc 'split("\n") | map(select(length > 0))')"
          dedicated_workflow_json='["org-team-lifecycle-critical-path.e2e.test.ts"]'
          excluded="$(
            printf '%s\n' "$raw_desktop" "$(echo "$profile_excluded_json" | jq -r '.[]')" \
              "$(echo "$dedicated_workflow_json" | jq -r '.[]')" \
              | sed '/^$/d' \
              | sort -u
          )"
          excluded_json="$(printf '%s\n' "$excluded" | jq -Rsc 'split("\n") | map(select(length > 0))')"

          all_tests="$(
            printf '%s\n' evals/specs/*.e2e.test.ts \
              | sed 's#^evals/specs/##' \
              | jq -Rsc 'split("\n") | map(select(length > 0))'
          )"
          eligible_tests="$(
            echo "$all_tests" \
              | jq -c --argjson excluded "$excluded_json" '
                  map(select(. as $test | $excluded | index($test) | not))
                '
          )"

          if [ "$EVENT_NAME" = "workflow_run" ]; then
            matched_tests="$(node evals/scripts/spec-impact.mjs --base "$BASE_SHA" --head "$HEAD_SHA" --matched-tests)"
            only_filter=""
          else
            matched_tests='[]'
            only_filter="$ONLY_FILTER"
          fi

          tests="$(
            echo "$eligible_tests" \
              | jq -c --arg only "$only_filter" --argjson matched "$matched_tests" '
                  map(select(($matched | length == 0) or (. as $test | ($matched | index($test)) != null)))
                  | map(select($only == "" or contains($only)))
                '
          )"
          batches="$(
            echo "$tests" | jq -c '
              . as $tests
              | [
                  range(0; 3) as $index
                  | {
                      name: "batch-\($index + 1)",
                      tests: [
                        $tests
                        | to_entries[]
                        | select((.key % 3) == $index)
                        | .value
                      ]
                    }
                ]
              | map(select(.tests | length > 0))
            '
          )"

          {
            echo "### Daytona E2E regression plan"
            echo
            echo "Inventory: $(echo "$all_tests" | jq 'length') total; $(echo "$excluded_json" | jq 'length') excluded; $(echo "$eligible_tests" | jq 'length') eligible for the shared topology"
            echo "- Profile exclusions: $(echo "$profile_excluded_json" | jq 'length')"
            echo "- Raw desktop() exclusions: $(echo "$raw_desktop_json" | jq 'length')"
            echo "- Dedicated-workflow exclusions: $(echo "$dedicated_workflow_json" | jq 'length')"
            echo
            echo "Matrix tests: $(echo "$tests" | jq 'length') across $(echo "$batches" | jq 'length') batches"
            if [ "$EVENT_NAME" = "workflow_run" ] && [ "$(echo "$tests" | jq 'length')" -eq 0 ]; then
              echo "Warden suggestion: add or update an evals/specs/<feature>.e2e.test.ts test for this PR."
            fi
            echo "$batches" | jq -r '.[] | "- `\(.name)`: \(.tests | length) E2E tests"'
            echo
            echo "Excluded (profile-incompatible, raw desktop(), or dedicated workflow): $(echo "$excluded_json" | jq 'length')"
            echo "$excluded_json" | jq -r --slurpfile profile "$profile" --argjson dedicated "$dedicated_workflow_json" '
              .[] as $test
              | (($profile[0].excluded | map(select(.test == $test)) | first | .reason)
                // (if $dedicated | index($test) then "owned by the dedicated nightly critical-path workflow" else "raw desktop() topology" end)) as $reason
              | "- `\($test)`: \($reason)"
            '
          } >> "$GITHUB_STEP_SUMMARY"

          echo "batches=$batches" >> "$GITHUB_OUTPUT"
          if [ "$(echo "$tests" | jq 'length')" -gt 0 ]; then
            echo "has_tests=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_tests=false" >> "$GITHUB_OUTPUT"
          fi
          echo "Batch matrix: $batches"

  e2e:
    needs: plan
    if: needs.plan.outputs.enabled == 'true' && needs.plan.outputs.has_tests == 'true'
    name: E2E (${{ matrix.batch.name }})
    # Existing GitHub environment name retained for approval compatibility.
    environment: pr-slow-specs
    strategy:
      fail-fast: false
      max-parallel: 3
      matrix:
        batch: ${{ fromJSON(needs.plan.outputs.batches) }}
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 45

    steps:
      - name: Checkout
        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
        with:
          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
          persist-credentials: false

      - name: Setup Node
        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
        with:
          node-version: 24

      - name: Setup pnpm
        uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
        with:
          version: 11.4.0

      - name: Install dependencies
        shell: bash
        run: |
          set -euo pipefail
          pnpm install --frozen-lockfile
          pnpm --dir evals install --frozen-lockfile --ignore-scripts

      # The opencode MCP OAuth spec (opencode-mcp-agent-oauth.e2e.test.ts)
      # drives the real opencode CLI as an OAuth MCP client against the Den
      # agent endpoint. Install the exact validated version without lifecycle
      # scripts before Daytona credentials are provisioned on the runner, then
      # run the package's one known postinstall deliberately — the bin shim
      # refuses to start without it, and --ignore-scripts must not silently
      # skip it for every transitive dependency's sake.
      - name: Install opencode CLI
        shell: bash
        run: |
          set -euo pipefail
          export PNPM_HOME="$RUNNER_TEMP/pnpm-global"
          export PATH="$PNPM_HOME/bin:$PNPM_HOME:$PATH"
          mkdir -p "$PNPM_HOME/bin"
          pnpm add --global --ignore-scripts opencode-ai@1.18.18
          package_dir=""
          while IFS= read -r candidate; do
            package_dir="$candidate"
          done < <(pnpm list --global --parseable opencode-ai)
          [ -n "$package_dir" ]
          (cd "$package_dir" && node postinstall.mjs)
          opencode --version
          echo "$PNPM_HOME/bin" >> "$GITHUB_PATH"
          echo "$PNPM_HOME" >> "$GITHUB_PATH"

      # The eval harness shells out to the daytona CLI (testkit daytonaAvailable
      # gate and @openwork/hosts exec). The binary is pinned to a release tag
      # and checksum-verified before it can run with CI secrets. When bumping,
      # take the digest from GitHub's authoritative asset metadata, never from
      # a local download (a truncating proxy once poisoned the pin):
      #   gh api repos/daytona/clients/releases/tags/<tag> --jq '.assets[] | select(.name == "daytona-linux-amd64") | .digest'
      - name: Install Daytona CLI
        shell: bash
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
          DAYTONA_CLI_TAG: v0.204.0
          DAYTONA_CLI_SHA256: 76701138c2ce93c14b30e7b10f51aa5ea2a9e156777bab933a4272237d81f14d
        run: |
          set -euo pipefail
          gh release download "$DAYTONA_CLI_TAG" --repo daytona/clients --pattern daytona-linux-amd64 --output /tmp/daytona --clobber
          echo "$DAYTONA_CLI_SHA256  /tmp/daytona" | sha256sum -c -
          sudo install -m 0755 /tmp/daytona /usr/local/bin/daytona
          daytona --version
          # The CLI requires a stored profile; the env var alone is not enough.
          daytona login --api-key "$DAYTONA_API_KEY"

      - name: Run batch
        id: run
        shell: bash
        env:
          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          NO_COLOR: "1"
          BATCH_NAME: ${{ matrix.batch.name }}
          TESTS_JSON: ${{ toJSON(matrix.batch.tests) }}
        run: |
          set -euo pipefail

          mapfile -t tests < <(jq -r '.[] | "specs/" + .' <<< "$TESTS_JSON")
          if [ "${#tests[@]}" -eq 0 ]; then
            echo "No E2E tests were assigned to $BATCH_NAME." >&2
            exit 1
          fi

          set +e
          OPENWORK_EVAL_DAYTONA=1 OPENWORK_EVAL_E2E_TESTS=1 \
          OPENWORK_EVAL_MAX_WORKERS=1 \
          OPENWORK_EVAL_AUTOMATIONS_E2E_TEST=1 OPENWORK_EVAL_CONNECTOR_E2E_TEST=1 \
          OPENWORK_EVAL_LOCAL_MANAGED_MCP=1 OPENWORK_EVAL_GENERATED_ARTIFACT_VIEWS_E2E_TEST=1 \
          OPENWORK_EVAL_SAVED_SCRIPT_AUTOMATIONS_E2E_TEST=1 OPENWORK_EVAL_ENGINE_ROLLOVER_E2E_TEST=1 \
          OPENWORK_EVAL_MODEL=big-pickle OPENWORK_EVAL_VISION=defer \
          pnpm --dir evals exec vitest run --config vitest.config.ts --project e2e "${tests[@]}" \
            2>&1 | tee /tmp/e2e-run.log
          vitest_status="${PIPESTATUS[0]}"
          set -e

          sed -E 's/\x1B\[[0-9;]*m//g' /tmp/e2e-run.log > /tmp/e2e-run-plain.log
          detail="${#tests[@]} E2E tests"
          if [ "$vitest_status" -ne 0 ]; then
            result="failed"
          elif grep -Eq "Tests[[:space:]].*skipped" /tmp/e2e-run-plain.log; then
            result="skipped"
            needs_reasons="$(grep -o '\[needs:[^]]*\]' /tmp/e2e-run-plain.log | sort -u | paste -sd ';' - || true)"
            if [ -n "$needs_reasons" ]; then
              detail="$detail; $needs_reasons"
            fi
          else
            result="passed"
          fi

          echo "result=$result" >> "$GITHUB_OUTPUT"
          {
            echo "### $BATCH_NAME"
            echo
            echo "| Batch | Result | Detail |"
            echo "| --- | --- | --- |"
            echo "| $BATCH_NAME | $result | $detail |"
          } >> "$GITHUB_STEP_SUMMARY"
          echo "Vitest exit code: $vitest_status"

          if [ "$result" != "passed" ]; then
            exit 1
          fi

      - name: Judge deferred vision claims
        if: steps.run.outputs.result == 'passed'
        shell: bash
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -euo pipefail
          while IFS= read -r -d '' test_run; do
            pnpm --dir evals evidence:judge -- --test-run "$test_run"
          done < <(find "$GITHUB_WORKSPACE/evals/results/test-runs" -mindepth 1 -maxdepth 1 -type d -print0)

      - name: Upload E2E test artifacts
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: e2e-test-artifacts-${{ strategy.job-index }}
          path: evals/results/test-runs/
          retention-days: 14
          if-no-files-found: ignore

  verdict:
    needs:
      - plan
      - e2e
    if: always() && needs.plan.outputs.enabled == 'true'
    runs-on: blacksmith-4vcpu-ubuntu-2404
    timeout-minutes: 10

    steps:
      - name: Report E2E regression verdict
        shell: bash
        env:
          E2E_RESULT: ${{ needs.e2e.result }}
        run: |
          set -euo pipefail

          {
            echo "## Daytona E2E Regression Suite"
            echo
            echo "E2E matrix result: **$E2E_RESULT**."
            echo "Only a successful, non-skipped matrix passes; an empty or skipped matrix is Incomplete and fails this required check."
          } >> "$GITHUB_STEP_SUMMARY"

          if [ "$E2E_RESULT" != "success" ]; then
            exit 1
          fi