// Live E2E test evidence for the script-installable OpenWork bootstrap CLI. // // This proof installs the CLI into a temp bin directory, runs doctor, then uses // the installed command to drive real Den API onboarding over HTTP: sign up, // create org, invite teammate, and create a skill. // // Required: // DEN_API_E2E_BASE_URL=http://127.0.0.1:18990 // // Run from packages/openwork-bootstrap: // DEN_API_E2E_BASE_URL=... pnpm exec node evals/agent-test-evidence-openwork-bootstrap.mjs import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { dirname, join, resolve } from "node:path" import { spawnSync } from "node:child_process" import { fileURLToPath } from "node:url" const baseUrl = process.env.DEN_API_E2E_BASE_URL?.replace(/\/$/, "") if (!baseUrl) { throw new Error("DEN_API_E2E_BASE_URL is required for live E2E test evidence") } const here = dirname(fileURLToPath(import.meta.url)) const packageRoot = resolve(here, "..") const repoRoot = resolve(packageRoot, "..", "..") const cli = join(packageRoot, "bin", "openwork.mjs") const temp = mkdtempSync(join(tmpdir(), "openwork-bootstrap-e2e-")) const installDir = join(temp, "install") const binDir = join(temp, "bin") const installedOpenwork = join(binDir, process.platform === "win32" ? "openwork-bootstrap.cmd" : "openwork-bootstrap") const outDir = join(repoRoot, "evals", "results", "openwork-bootstrap-cli") mkdirSync(outDir, { recursive: true }) const frames = [] const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}` function parseJson(text) { if (!text) return null try { return JSON.parse(text) } catch { return text } } function run(command, args, options = {}) { const result = spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 10_000, ...options }) const stdout = result.stdout?.trim() ?? "" const stderr = result.stderr?.trim() ?? "" return { command: [command, ...args].join(" "), status: result.status, signal: result.signal, error: result.error ? String(result.error) : null, stdout, stderr, json: parseJson(stdout), } } function prove(claim, { action, assert, evidence }, ok) { frames.push({ claim, action, assert, evidence, ok }) if (!ok) throw new Error(`Frame failed: ${claim}`) } function randomPassword() { return `A${crypto.randomUUID()}!9zZ` } try { const health = await fetch(`${baseUrl}/health`).then(async (response) => ({ status: response.status, body: await response.json().catch(() => null), })) prove("Den API is running as the live e2e target", { action: `GET ${baseUrl}/health`, assert: "HTTP 200 { ok: true }", evidence: health, }, health.status === 200 && health.body?.ok === true) const install = run(process.execPath, [cli, "install", "--install-dir", installDir, "--bin-dir", binDir, "--json"]) prove("A bootstrap script can install the openwork CLI", { action: "node bin/openwork.mjs install --install-dir --bin-dir /bin --json", assert: "exit 0 and installed executable path returned", evidence: { status: install.status, body: install.json }, }, install.status === 0 && install.json?.ok === true && install.json?.install?.executable === installedOpenwork) const doctor = run(installedOpenwork, ["doctor", "--install-dir", installDir, "--bin-dir", binDir, "--base-url", baseUrl, "--json"]) prove("The installed CLI can doctor itself and the Den API", { action: "openwork doctor --base-url --json", assert: "exit 0, local install checks pass, and Den API health passes", evidence: { status: doctor.status, body: doctor.json }, }, doctor.status === 0 && doctor.json?.ok === true && doctor.json?.checks?.some((check) => check.name === "denApiHealth" && check.ok === true)) const ownerEmail = `owner-${runId}@example.com` const inviteEmail = `teammate-${runId}@example.com` const orgName = `Bootstrap CLI Org ${runId}` const skillName = `Bootstrap Skill ${runId}` const onboard = run(installedOpenwork, [ "cloud", "onboard", "--base-url", baseUrl, "--owner-email", ownerEmail, "--org-name", orgName, "--invite-email", inviteEmail, "--skill-name", skillName, "--json", ], { env: { ...process.env, OPENWORK_OWNER_PASSWORD: randomPassword() }, timeout: 30_000 }) prove("The installed CLI can onboard a user, org, invite, and skill end-to-end", { action: "openwork cloud onboard --base-url --owner-email ... --org-name ... --invite-email ... --skill-name ... --json", assert: "exit 0 with user, organization, invitation, and skill ids from live API responses", evidence: { status: onboard.status, body: onboard.json, stderr: onboard.stderr }, }, onboard.status === 0 && onboard.json?.ok === true && onboard.json?.organization?.id && onboard.json?.invitation?.invitationId && onboard.json?.skill?.id && onboard.json?.skill?.title === skillName && onboard.json?.skillRun?.triggered === true && onboard.json?.skillRun?.output === "OPENWORK_BOOTSTRAP_SKILL_TRIGGERED") } finally { rmSync(temp, { recursive: true, force: true }) } const esc = (value) => String(value).replace(/&/g, "&").replace(//g, ">") const escAttr = (value) => esc(value).replace(/"/g, """) const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60) const frameFiles = frames.map((frame, index) => { const name = `${String(index + 1).padStart(2, "0")}-${slug(frame.claim)}.html` const html = ` ${esc(frame.claim)}

${esc(frame.claim)} — ${frame.ok ? "PASS" : "FAIL"}

action: ${esc(frame.action)}

assert: ${esc(frame.assert)}

evidence:

${esc(JSON.stringify(frame.evidence, null, 2))}
` writeFileSync(join(outDir, name), html) return { name, frame } }) const allOk = frames.every((frame) => frame.ok) // Keep fraimz.html only because the standalone legacy report contract consumes this filename. writeFileSync(join(outDir, "fraimz.html"), ` OpenWork Bootstrap CLI — live E2E test evidence

OpenWork Bootstrap CLI — live E2E test evidence

Result: ${allOk ? "passed" : "failed"} · Live Den API: ${esc(baseUrl)} · Frames: ${frames.length}
${frameFiles.map((entry) => `

${esc(entry.frame.claim)}

Open frame

`).join("\n")}
`) writeFileSync(join(outDir, "report.json"), JSON.stringify({ flow: "openwork-bootstrap-cli", result: allOk ? "passed" : "failed", liveBaseUrl: baseUrl, frames: frames.map((frame) => ({ claim: frame.claim, action: frame.action, assert: frame.assert, ok: frame.ok })), }, null, 2)) console.log(`test evidence: ${join(outDir, "fraimz.html")}`) console.log(`result: ${allOk ? "passed" : "failed"} (${frames.length} frames)`) process.exit(allOk ? 0 : 1)