415 lines
17 KiB
YAML
415 lines
17 KiB
YAML
# Runs the eligible E2E tests in three balanced Daytona batches. Each batch keeps
|
|
# one prewarmed Den + desktop pair and reuses it across its assigned tests.
|
|
# The checked-in lane profile removes only topology/dependency skips before the
|
|
# matrix is built; compatible product regressions remain in the matrix.
|
|
# The critical-path journey remains owned by its dedicated nightly workflow.
|
|
# Tests that drive raw desktop() from @openwork/hosts are excluded dynamically:
|
|
# they spawn Electron on the caller's machine (correct inside a VNC sandbox or
|
|
# on a dev workstation, impossible on a headless runner) and need a follow-up
|
|
# in-sandbox vitest lane. The plan job lists every exclusion in its summary.
|
|
name: Daytona E2E Regression Suite
|
|
|
|
on:
|
|
workflow_run:
|
|
workflows: [Warden]
|
|
types: [completed]
|
|
workflow_dispatch:
|
|
inputs:
|
|
only:
|
|
description: "Substring filter on E2E test filenames; empty runs all eligible tests"
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
pull-requests: read
|
|
|
|
concurrency:
|
|
group: daytona-e2e-regression-suite-${{ github.event.workflow_run.pull_requests[0].number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
plan:
|
|
if: >-
|
|
github.repository == 'different-ai/openwork' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(github.event.workflow_run.event == 'pull_request' &&
|
|
github.event.workflow_run.conclusion == 'success' &&
|
|
github.event.workflow_run.head_repository.full_name == github.repository))
|
|
runs-on: blacksmith-4vcpu-ubuntu-2404
|
|
timeout-minutes: 10
|
|
outputs:
|
|
enabled: ${{ steps.config.outputs.enabled }}
|
|
batches: ${{ steps.matrix.outputs.batches }}
|
|
has_tests: ${{ steps.matrix.outputs.has_tests }}
|
|
|
|
steps:
|
|
- name: Authorize Warden-cleared pull request
|
|
id: authorize
|
|
if: github.event_name == 'workflow_run'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
RUN_ID: ${{ github.event.workflow_run.id }}
|
|
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
PR: ${{ github.event.workflow_run.pull_requests[0].number }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
mkdir -p /tmp/warden-summary
|
|
gh run download "$RUN_ID" --repo "$REPO" --name warden-summary --dir /tmp/warden-summary
|
|
summary_sha="$(jq -r '.head_sha' /tmp/warden-summary/warden-summary.json)"
|
|
blocking="$(jq -r '.blocking_count // .findings_count' /tmp/warden-summary/warden-summary.json)"
|
|
current_head="$(gh api "repos/$REPO/pulls/$PR" --jq '.head.sha')"
|
|
|
|
if [ "$summary_sha" != "$HEAD_SHA" ] || [ "$current_head" != "$HEAD_SHA" ]; then
|
|
echo "::error::Warden summary or pull request head does not match $HEAD_SHA."
|
|
exit 1
|
|
fi
|
|
if [ "$blocking" != "0" ]; then
|
|
echo "::notice::E2E regression withheld: Warden reported $blocking blocking finding(s)."
|
|
echo "authorized=false" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
guarded="$(gh api --paginate "repos/$REPO/pulls/$PR/files" --jq '.[].filename' \
|
|
| grep -E '^(\.github/|warden\.toml$|\.warden/|\.agents/skills/|\.claude/skills/)' || true)"
|
|
if [ -n "$guarded" ]; then
|
|
echo "::notice::E2E regression withheld: PR changes trusted review machinery."
|
|
echo "authorized=false" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
echo "authorized=true" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Authorize manual E2E regression
|
|
id: authorize-manual
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: echo "authorized=true" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Check E2E regression configuration
|
|
id: config
|
|
if: steps.authorize.outputs.authorized == 'true' || steps.authorize-manual.outputs.authorized == 'true'
|
|
shell: bash
|
|
env:
|
|
DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
missing=""
|
|
[ -z "${DAYTONA_API_KEY:-}" ] && missing="$missing DAYTONA_API_KEY"
|
|
[ -z "${OPENAI_API_KEY:-}" ] && missing="$missing OPENAI_API_KEY"
|
|
if [ -n "$missing" ]; then
|
|
echo "::notice::Skipping Daytona E2E regression suite; missing GitHub configuration:$missing."
|
|
echo "enabled=false" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
echo "enabled=true" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Checkout
|
|
if: steps.config.outputs.enabled == 'true'
|
|
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
|
with:
|
|
ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Build E2E test matrix
|
|
id: matrix
|
|
if: steps.config.outputs.enabled == 'true'
|
|
shell: bash
|
|
env:
|
|
ONLY_FILTER: ${{ inputs.only }}
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
BASE_SHA: ${{ github.event.workflow_run.pull_requests[0].base.sha }}
|
|
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# Raw desktop() tests spawn Electron on the Vitest host itself; on a
|
|
# headless runner that dies before CDP. Exclude them until the
|
|
# in-sandbox vitest lane exists, and surface the list in the summary.
|
|
raw_desktop="$(node evals/scripts/list-daytona-raw-desktop-tests.mjs)"
|
|
# The checked-in profile is the explicit contract for this lane.
|
|
# Raw desktop() detection remains as a guard for newly added tests
|
|
# that have not been classified yet.
|
|
profile='evals/specs/daytona-e2e-regression-profile.json'
|
|
profile_excluded_json="$(jq -c '[.excluded[].test]' "$profile")"
|
|
raw_desktop_json="$(printf '%s\n' "$raw_desktop" | jq -Rsc 'split("\n") | map(select(length > 0))')"
|
|
dedicated_workflow_json='["org-team-lifecycle-critical-path.e2e.test.ts"]'
|
|
excluded="$(
|
|
printf '%s\n' "$raw_desktop" "$(echo "$profile_excluded_json" | jq -r '.[]')" \
|
|
"$(echo "$dedicated_workflow_json" | jq -r '.[]')" \
|
|
| sed '/^$/d' \
|
|
| sort -u
|
|
)"
|
|
excluded_json="$(printf '%s\n' "$excluded" | jq -Rsc 'split("\n") | map(select(length > 0))')"
|
|
|
|
all_tests="$(
|
|
printf '%s\n' evals/specs/*.e2e.test.ts \
|
|
| sed 's#^evals/specs/##' \
|
|
| jq -Rsc 'split("\n") | map(select(length > 0))'
|
|
)"
|
|
eligible_tests="$(
|
|
echo "$all_tests" \
|
|
| jq -c --argjson excluded "$excluded_json" '
|
|
map(select(. as $test | $excluded | index($test) | not))
|
|
'
|
|
)"
|
|
|
|
if [ "$EVENT_NAME" = "workflow_run" ]; then
|
|
matched_tests="$(node evals/scripts/spec-impact.mjs --base "$BASE_SHA" --head "$HEAD_SHA" --matched-tests)"
|
|
only_filter=""
|
|
else
|
|
matched_tests='[]'
|
|
only_filter="$ONLY_FILTER"
|
|
fi
|
|
|
|
tests="$(
|
|
echo "$eligible_tests" \
|
|
| jq -c --arg only "$only_filter" --argjson matched "$matched_tests" '
|
|
map(select(($matched | length == 0) or (. as $test | ($matched | index($test)) != null)))
|
|
| map(select($only == "" or contains($only)))
|
|
'
|
|
)"
|
|
batches="$(
|
|
echo "$tests" | jq -c '
|
|
. as $tests
|
|
| [
|
|
range(0; 3) as $index
|
|
| {
|
|
name: "batch-\($index + 1)",
|
|
tests: [
|
|
$tests
|
|
| to_entries[]
|
|
| select((.key % 3) == $index)
|
|
| .value
|
|
]
|
|
}
|
|
]
|
|
| map(select(.tests | length > 0))
|
|
'
|
|
)"
|
|
|
|
{
|
|
echo "### Daytona E2E regression plan"
|
|
echo
|
|
echo "Inventory: $(echo "$all_tests" | jq 'length') total; $(echo "$excluded_json" | jq 'length') excluded; $(echo "$eligible_tests" | jq 'length') eligible for the shared topology"
|
|
echo "- Profile exclusions: $(echo "$profile_excluded_json" | jq 'length')"
|
|
echo "- Raw desktop() exclusions: $(echo "$raw_desktop_json" | jq 'length')"
|
|
echo "- Dedicated-workflow exclusions: $(echo "$dedicated_workflow_json" | jq 'length')"
|
|
echo
|
|
echo "Matrix tests: $(echo "$tests" | jq 'length') across $(echo "$batches" | jq 'length') batches"
|
|
if [ "$EVENT_NAME" = "workflow_run" ] && [ "$(echo "$tests" | jq 'length')" -eq 0 ]; then
|
|
echo "Warden suggestion: add or update an evals/specs/<feature>.e2e.test.ts test for this PR."
|
|
fi
|
|
echo "$batches" | jq -r '.[] | "- `\(.name)`: \(.tests | length) E2E tests"'
|
|
echo
|
|
echo "Excluded (profile-incompatible, raw desktop(), or dedicated workflow): $(echo "$excluded_json" | jq 'length')"
|
|
echo "$excluded_json" | jq -r --slurpfile profile "$profile" --argjson dedicated "$dedicated_workflow_json" '
|
|
.[] as $test
|
|
| (($profile[0].excluded | map(select(.test == $test)) | first | .reason)
|
|
// (if $dedicated | index($test) then "owned by the dedicated nightly critical-path workflow" else "raw desktop() topology" end)) as $reason
|
|
| "- `\($test)`: \($reason)"
|
|
'
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
echo "batches=$batches" >> "$GITHUB_OUTPUT"
|
|
if [ "$(echo "$tests" | jq 'length')" -gt 0 ]; then
|
|
echo "has_tests=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_tests=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
echo "Batch matrix: $batches"
|
|
|
|
e2e:
|
|
needs: plan
|
|
if: needs.plan.outputs.enabled == 'true' && needs.plan.outputs.has_tests == 'true'
|
|
name: E2E (${{ matrix.batch.name }})
|
|
# Existing GitHub environment name retained for approval compatibility.
|
|
environment: pr-slow-specs
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 3
|
|
matrix:
|
|
batch: ${{ fromJSON(needs.plan.outputs.batches) }}
|
|
runs-on: blacksmith-4vcpu-ubuntu-2404
|
|
timeout-minutes: 45
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
|
with:
|
|
ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
|
|
persist-credentials: false
|
|
|
|
- name: Setup Node
|
|
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
|
|
with:
|
|
version: 11.4.0
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
pnpm install --frozen-lockfile
|
|
pnpm --dir evals install --frozen-lockfile --ignore-scripts
|
|
|
|
# The opencode MCP OAuth spec (opencode-mcp-agent-oauth.e2e.test.ts)
|
|
# drives the real opencode CLI as an OAuth MCP client against the Den
|
|
# agent endpoint. Install the exact validated version without lifecycle
|
|
# scripts before Daytona credentials are provisioned on the runner, then
|
|
# run the package's one known postinstall deliberately — the bin shim
|
|
# refuses to start without it, and --ignore-scripts must not silently
|
|
# skip it for every transitive dependency's sake.
|
|
- name: Install opencode CLI
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
export PNPM_HOME="$RUNNER_TEMP/pnpm-global"
|
|
export PATH="$PNPM_HOME/bin:$PNPM_HOME:$PATH"
|
|
mkdir -p "$PNPM_HOME/bin"
|
|
pnpm add --global --ignore-scripts opencode-ai@1.18.18
|
|
package_dir=""
|
|
while IFS= read -r candidate; do
|
|
package_dir="$candidate"
|
|
done < <(pnpm list --global --parseable opencode-ai)
|
|
[ -n "$package_dir" ]
|
|
(cd "$package_dir" && node postinstall.mjs)
|
|
opencode --version
|
|
echo "$PNPM_HOME/bin" >> "$GITHUB_PATH"
|
|
echo "$PNPM_HOME" >> "$GITHUB_PATH"
|
|
|
|
# The eval harness shells out to the daytona CLI (testkit daytonaAvailable
|
|
# gate and @openwork/hosts exec). The binary is pinned to a release tag
|
|
# and checksum-verified before it can run with CI secrets. When bumping,
|
|
# take the digest from GitHub's authoritative asset metadata, never from
|
|
# a local download (a truncating proxy once poisoned the pin):
|
|
# gh api repos/daytona/clients/releases/tags/<tag> --jq '.assets[] | select(.name == "daytona-linux-amd64") | .digest'
|
|
- name: Install Daytona CLI
|
|
shell: bash
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
|
|
DAYTONA_CLI_TAG: v0.204.0
|
|
DAYTONA_CLI_SHA256: 76701138c2ce93c14b30e7b10f51aa5ea2a9e156777bab933a4272237d81f14d
|
|
run: |
|
|
set -euo pipefail
|
|
gh release download "$DAYTONA_CLI_TAG" --repo daytona/clients --pattern daytona-linux-amd64 --output /tmp/daytona --clobber
|
|
echo "$DAYTONA_CLI_SHA256 /tmp/daytona" | sha256sum -c -
|
|
sudo install -m 0755 /tmp/daytona /usr/local/bin/daytona
|
|
daytona --version
|
|
# The CLI requires a stored profile; the env var alone is not enough.
|
|
daytona login --api-key "$DAYTONA_API_KEY"
|
|
|
|
- name: Run batch
|
|
id: run
|
|
shell: bash
|
|
env:
|
|
DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
NO_COLOR: "1"
|
|
BATCH_NAME: ${{ matrix.batch.name }}
|
|
TESTS_JSON: ${{ toJSON(matrix.batch.tests) }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
mapfile -t tests < <(jq -r '.[] | "specs/" + .' <<< "$TESTS_JSON")
|
|
if [ "${#tests[@]}" -eq 0 ]; then
|
|
echo "No E2E tests were assigned to $BATCH_NAME." >&2
|
|
exit 1
|
|
fi
|
|
|
|
set +e
|
|
OPENWORK_EVAL_DAYTONA=1 OPENWORK_EVAL_E2E_TESTS=1 \
|
|
OPENWORK_EVAL_MAX_WORKERS=1 \
|
|
OPENWORK_EVAL_AUTOMATIONS_E2E_TEST=1 OPENWORK_EVAL_CONNECTOR_E2E_TEST=1 \
|
|
OPENWORK_EVAL_LOCAL_MANAGED_MCP=1 OPENWORK_EVAL_GENERATED_ARTIFACT_VIEWS_E2E_TEST=1 \
|
|
OPENWORK_EVAL_SAVED_SCRIPT_AUTOMATIONS_E2E_TEST=1 OPENWORK_EVAL_ENGINE_ROLLOVER_E2E_TEST=1 \
|
|
OPENWORK_EVAL_MODEL=big-pickle OPENWORK_EVAL_VISION=defer \
|
|
pnpm --dir evals exec vitest run --config vitest.config.ts --project e2e "${tests[@]}" \
|
|
2>&1 | tee /tmp/e2e-run.log
|
|
vitest_status="${PIPESTATUS[0]}"
|
|
set -e
|
|
|
|
sed -E 's/\x1B\[[0-9;]*m//g' /tmp/e2e-run.log > /tmp/e2e-run-plain.log
|
|
detail="${#tests[@]} E2E tests"
|
|
if [ "$vitest_status" -ne 0 ]; then
|
|
result="failed"
|
|
elif grep -Eq "Tests[[:space:]].*skipped" /tmp/e2e-run-plain.log; then
|
|
result="skipped"
|
|
needs_reasons="$(grep -o '\[needs:[^]]*\]' /tmp/e2e-run-plain.log | sort -u | paste -sd ';' - || true)"
|
|
if [ -n "$needs_reasons" ]; then
|
|
detail="$detail; $needs_reasons"
|
|
fi
|
|
else
|
|
result="passed"
|
|
fi
|
|
|
|
echo "result=$result" >> "$GITHUB_OUTPUT"
|
|
{
|
|
echo "### $BATCH_NAME"
|
|
echo
|
|
echo "| Batch | Result | Detail |"
|
|
echo "| --- | --- | --- |"
|
|
echo "| $BATCH_NAME | $result | $detail |"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
echo "Vitest exit code: $vitest_status"
|
|
|
|
if [ "$result" != "passed" ]; then
|
|
exit 1
|
|
fi
|
|
|
|
- name: Judge deferred vision claims
|
|
if: steps.run.outputs.result == 'passed'
|
|
shell: bash
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
while IFS= read -r -d '' test_run; do
|
|
pnpm --dir evals evidence:judge -- --test-run "$test_run"
|
|
done < <(find "$GITHUB_WORKSPACE/evals/results/test-runs" -mindepth 1 -maxdepth 1 -type d -print0)
|
|
|
|
- name: Upload E2E test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: e2e-test-artifacts-${{ strategy.job-index }}
|
|
path: evals/results/test-runs/
|
|
retention-days: 14
|
|
if-no-files-found: ignore
|
|
|
|
verdict:
|
|
needs:
|
|
- plan
|
|
- e2e
|
|
if: always() && needs.plan.outputs.enabled == 'true'
|
|
runs-on: blacksmith-4vcpu-ubuntu-2404
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Report E2E regression verdict
|
|
shell: bash
|
|
env:
|
|
E2E_RESULT: ${{ needs.e2e.result }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
{
|
|
echo "## Daytona E2E Regression Suite"
|
|
echo
|
|
echo "E2E matrix result: **$E2E_RESULT**."
|
|
echo "Only a successful, non-skipped matrix passes; an empty or skipped matrix is Incomplete and fails this required check."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
if [ "$E2E_RESULT" != "success" ]; then
|
|
exit 1
|
|
fi
|