源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
@@ -0,0 +1,199 @@
|
||||
// Linux E2E test evidence for OpenWork bootstrap without shipping to production.
|
||||
//
|
||||
// This runs inside a real Linux container and proves:
|
||||
// - the bootstrap CLI installs on Linux,
|
||||
// - `openwork install app` installs a Linux tar.gz app artifact from a manifest,
|
||||
// - `openwork doctor --app` verifies the installed app,
|
||||
// - `openwork cloud onboard` can hit a live Den API from inside Linux.
|
||||
//
|
||||
// Required host setup:
|
||||
// DEN_API_E2E_BASE_URL=http://host.docker.internal:<den-api-port>
|
||||
//
|
||||
// Run from packages/openwork-bootstrap:
|
||||
// DEN_API_E2E_BASE_URL=http://host.docker.internal:18995 pnpm exec node evals/agent-test-evidence-linux-container.mjs
|
||||
|
||||
import { createHash } from "node:crypto"
|
||||
import { execFileSync, spawnSync } from "node:child_process"
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { dirname, join, resolve } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const denBaseUrl = process.env.DEN_API_E2E_BASE_URL?.replace(/\/$/, "")
|
||||
if (!denBaseUrl) {
|
||||
throw new Error("DEN_API_E2E_BASE_URL is required")
|
||||
}
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const packageRoot = resolve(here, "..")
|
||||
const repoRoot = resolve(packageRoot, "..", "..")
|
||||
const outDir = join(repoRoot, "evals", "results", "openwork-linux-container")
|
||||
const temp = join(outDir, `_tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`)
|
||||
const fixtureDir = join(temp, "fixture")
|
||||
const artifactRoot = join(temp, "artifact-root")
|
||||
const artifactPath = join(fixtureDir, "openwork-linux.tar.gz")
|
||||
const manifestPath = join(fixtureDir, "install-manifest.json")
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
mkdirSync(fixtureDir, { recursive: true })
|
||||
mkdirSync(artifactRoot, { recursive: true })
|
||||
|
||||
const frames = []
|
||||
const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
|
||||
function parseJson(text) {
|
||||
if (!text) return null
|
||||
try {
|
||||
return JSON.parse(text)
|
||||
} catch {
|
||||
return text
|
||||
}
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const result = spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 20_000, ...options })
|
||||
const stdout = result.stdout?.trim() ?? ""
|
||||
const stderr = result.stderr?.trim() ?? ""
|
||||
return {
|
||||
command: redactSecrets([command, ...args].join(" ")),
|
||||
status: result.status,
|
||||
signal: result.signal,
|
||||
error: result.error ? String(result.error) : null,
|
||||
stdout,
|
||||
stderr,
|
||||
json: parseJson(stdout),
|
||||
}
|
||||
}
|
||||
|
||||
function redactSecrets(value) {
|
||||
return value.replace(/OPENWORK_OWNER_PASSWORD=[^\s]+/g, "OPENWORK_OWNER_PASSWORD=[redacted]")
|
||||
}
|
||||
|
||||
function prove(claim, { action, assert, evidence }, ok) {
|
||||
// Record the screenshot regardless of outcome so the test-evidence report still renders.
|
||||
// when a frame fails. The process exit code below reflects pass/fail.
|
||||
frames.push({ claim, action, assert, evidence, ok: Boolean(ok) })
|
||||
}
|
||||
|
||||
function sha256(path) {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex")
|
||||
}
|
||||
|
||||
function dockerRun(script, options = {}) {
|
||||
return run("docker", [
|
||||
"run",
|
||||
"--rm",
|
||||
// Linux Docker Engine does not auto-map host.docker.internal; map it to the
|
||||
// host gateway so the Den API is reachable from inside the container.
|
||||
"--add-host=host.docker.internal:host-gateway",
|
||||
"-v",
|
||||
`${packageRoot}:/package:ro`,
|
||||
"-v",
|
||||
`${fixtureDir}:/fixture:ro`,
|
||||
"-e",
|
||||
`DEN_API_E2E_BASE_URL=${denBaseUrl}`,
|
||||
...(options.env ?? []),
|
||||
"node:20-bookworm-slim",
|
||||
"bash",
|
||||
"-lc",
|
||||
script,
|
||||
], { timeout: options.timeout ?? 60_000 })
|
||||
}
|
||||
|
||||
function randomPassword() {
|
||||
return `A${crypto.randomUUID()}!9zZ`
|
||||
}
|
||||
|
||||
try {
|
||||
const appBinary = join(artifactRoot, "openwork")
|
||||
writeFileSync(appBinary, "#!/usr/bin/env sh\necho OpenWork Linux fixture\n")
|
||||
chmodSync(appBinary, 0o755)
|
||||
execFileSync("tar", ["-czf", artifactPath, "-C", artifactRoot, "openwork"])
|
||||
const digest = sha256(artifactPath)
|
||||
writeFileSync(manifestPath, JSON.stringify({
|
||||
version: "0.0.0-linux-fixture",
|
||||
artifacts: {
|
||||
linux: {
|
||||
x64: { type: "tar.gz", url: "file:///fixture/openwork-linux.tar.gz", sha256: digest, appName: "openwork" },
|
||||
arm64: { type: "tar.gz", url: "file:///fixture/openwork-linux.tar.gz", sha256: digest, appName: "openwork" },
|
||||
},
|
||||
},
|
||||
}, null, 2))
|
||||
prove("A Linux app artifact manifest is available without shipping to production", {
|
||||
action: "Create local openwork tar.gz artifact and manifest with SHA-256",
|
||||
assert: "manifest and tar.gz exist on the host and are mounted into Linux container",
|
||||
evidence: { manifestPath, artifactPath, sha256: digest },
|
||||
}, existsSync(manifestPath) && existsSync(artifactPath) && digest.length === 64)
|
||||
|
||||
const linuxIdentity = dockerRun("node -p 'JSON.stringify({platform:process.platform,arch:process.arch,node:process.version})'")
|
||||
prove("The e2e runs inside Linux", {
|
||||
action: "docker run node:20-bookworm-slim node -p process.platform",
|
||||
assert: "container reports platform linux",
|
||||
evidence: linuxIdentity,
|
||||
}, linuxIdentity.status === 0 && linuxIdentity.json?.platform === "linux")
|
||||
|
||||
const installAndDoctor = dockerRun([
|
||||
"set -euo pipefail",
|
||||
"node /package/bin/openwork.mjs install --install-dir /tmp/openwork/install --bin-dir /tmp/openwork/bin --json >/tmp/install.json",
|
||||
"/tmp/openwork/bin/openwork-bootstrap install app --manifest /fixture/install-manifest.json --app-dir /tmp/openwork/apps --json >/tmp/app-install.json",
|
||||
"/tmp/openwork/bin/openwork-bootstrap doctor --install-dir /tmp/openwork/install --bin-dir /tmp/openwork/bin --app --app-dir /tmp/openwork/apps --json",
|
||||
].join(" && "), { timeout: 60_000 })
|
||||
prove("Linux can install the CLI and app artifact, then doctor the app", {
|
||||
action: "Run openwork install, openwork install app, and openwork doctor --app inside Linux",
|
||||
assert: "all commands exit 0; final doctor JSON returns ok true",
|
||||
evidence: installAndDoctor,
|
||||
}, installAndDoctor.status === 0 && installAndDoctor.json?.ok === true)
|
||||
|
||||
const ownerEmail = `linux-owner-${runId}@example.com`
|
||||
const inviteEmail = `linux-teammate-${runId}@example.com`
|
||||
const skillName = `Linux Bootstrap Skill ${runId}`
|
||||
const onboard = dockerRun([
|
||||
"set -euo pipefail",
|
||||
"node /package/bin/openwork.mjs install --install-dir /tmp/openwork/install --bin-dir /tmp/openwork/bin --json >/tmp/install.json",
|
||||
`/tmp/openwork/bin/openwork-bootstrap cloud onboard --base-url "$DEN_API_E2E_BASE_URL" --owner-email ${JSON.stringify(ownerEmail)} --org-name ${JSON.stringify(`Linux CLI Org ${runId}`)} --invite-email ${JSON.stringify(inviteEmail)} --skill-name ${JSON.stringify(skillName)} --prepare-desktop --desktop-bootstrap-path /tmp/openwork/desktop-bootstrap.json --skills-dir /tmp/openwork/skills --json >/tmp/onboard.json`,
|
||||
"/tmp/openwork/bin/openwork-bootstrap doctor --install-dir /tmp/openwork/install --bin-dir /tmp/openwork/bin --desktop-bootstrap --desktop-bootstrap-path /tmp/openwork/desktop-bootstrap.json --json >/tmp/doctor.json",
|
||||
"node -e \"const fs=require('fs'); console.log(JSON.stringify({onboard:JSON.parse(fs.readFileSync('/tmp/onboard.json','utf8')),doctor:JSON.parse(fs.readFileSync('/tmp/doctor.json','utf8'))}, null, 2))\"",
|
||||
].join(" && "), {
|
||||
env: ["-e", `OPENWORK_OWNER_PASSWORD=${randomPassword()}`],
|
||||
timeout: 90_000,
|
||||
})
|
||||
prove("Linux installed CLI can complete live cloud onboarding", {
|
||||
action: "Run openwork cloud onboard from inside Linux against live Den API",
|
||||
assert: "exit 0 and returns prepared desktop bootstrap with user, organization, invitation, skill, and triggered skill output from live API",
|
||||
evidence: onboard,
|
||||
}, onboard.status === 0 && onboard.json?.onboard?.ok === true && onboard.json?.onboard?.organization?.id && onboard.json?.onboard?.invitation?.invitationId && onboard.json?.onboard?.skill?.id && onboard.json?.onboard?.skill?.title === skillName && onboard.json?.onboard?.skillRun?.triggered === true && onboard.json?.onboard?.skillRun?.output === "OPENWORK_BOOTSTRAP_SKILL_TRIGGERED" && onboard.json?.onboard?.desktop?.prepared === true && onboard.json?.onboard?.desktop?.bootstrapPath && onboard.json?.onboard?.desktop?.skillPath && onboard.json?.doctor?.ok === true && onboard.json?.doctor?.checks?.some((check) => check.name === "desktopBootstrap" && check.ok === true) && onboard.json?.doctor?.checks?.some((check) => check.name === "desktopBootstrapHandoff" && check.ok === true))
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const esc = (value) => String(value).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
|
||||
const escAttr = (value) => esc(value).replace(/"/g, """)
|
||||
const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60)
|
||||
const frameFiles = frames.map((frame, index) => {
|
||||
const name = `${String(index + 1).padStart(2, "0")}-${slug(frame.claim)}.html`
|
||||
const html = `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>${esc(frame.claim)}</title>
|
||||
<style>body{margin:0;background:#f8fafc;color:#111827;font-family:ui-monospace,Menlo,Consolas,monospace}main{max-width:980px;margin:0 auto;padding:32px}h1{font-family:system-ui,sans-serif;margin-top:0;font-size:18px}.k{color:#6b7280;font-family:system-ui,sans-serif}.ok{color:#047857;font-weight:600}pre{white-space:pre-wrap;word-break:break-word;padding:16px;border:1px solid #d1d5db;border-radius:14px;background:white}</style>
|
||||
</head><body><main><h1>${esc(frame.claim)} — <span class="ok">${frame.ok ? "PASS" : "FAIL"}</span></h1>
|
||||
<p><span class="k">action:</span> ${esc(frame.action)}</p><p><span class="k">assert:</span> ${esc(frame.assert)}</p><p class="k">evidence:</p>
|
||||
<pre>${esc(JSON.stringify(frame.evidence, null, 2))}</pre></main></body></html>`
|
||||
writeFileSync(join(outDir, name), html)
|
||||
return { name, frame }
|
||||
})
|
||||
|
||||
const allOk = frames.every((frame) => frame.ok)
|
||||
// Keep fraimz.html only because the standalone legacy report contract consumes this filename.
|
||||
writeFileSync(join(outDir, "fraimz.html"), `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>OpenWork Linux Bootstrap — test evidence</title>
|
||||
<style>body{margin:0;background:#f3f4f6;color:#111827;font-family:system-ui,sans-serif}main{max-width:1180px;margin:0 auto;padding:32px}.meta{color:#4b5563;margin-bottom:24px}section{margin:20px 0;padding:16px;border:1px solid #d1d5db;border-radius:16px;background:white}iframe{width:100%;min-height:360px;border:1px solid #e5e7eb;border-radius:12px;background:white}code{background:#e5e7eb;padding:2px 5px;border-radius:5px}</style>
|
||||
</head><body><main><h1>OpenWork Linux Bootstrap — test evidence</h1><div class="meta">Result: <code>${allOk ? "passed" : "failed"}</code> · Live Den API: <code>${esc(denBaseUrl)}</code> · Screenshots: ${frames.length}</div>
|
||||
${frameFiles.map((entry) => `<section><h2>${esc(entry.frame.claim)}</h2><iframe src="${escAttr(entry.name)}" title="${escAttr(entry.frame.claim)}"></iframe><p><a href="${escAttr(entry.name)}">Open frame</a></p></section>`).join("\n")}
|
||||
</main></body></html>`)
|
||||
writeFileSync(join(outDir, "report.json"), JSON.stringify({
|
||||
flow: "openwork-linux-container",
|
||||
result: allOk ? "passed" : "failed",
|
||||
liveBaseUrl: denBaseUrl,
|
||||
frames: frames.map((frame) => ({ claim: frame.claim, action: frame.action, assert: frame.assert, ok: frame.ok })),
|
||||
}, null, 2))
|
||||
|
||||
console.log(`test evidence: ${join(outDir, "fraimz.html")}`)
|
||||
console.log(`result: ${allOk ? "passed" : "failed"} (${frames.length} frames)`)
|
||||
process.exit(allOk ? 0 : 1)
|
||||
@@ -0,0 +1,163 @@
|
||||
// Live local E2E test evidence for `openwork install app` using a real macOS DMG.
|
||||
//
|
||||
// The flow creates a tiny OpenWork.app fixture, packages it as a .dmg, writes an
|
||||
// install manifest with a SHA-256 digest, installs the bootstrap CLI into a temp
|
||||
// bin dir, then runs the installed `openwork install app` command against the
|
||||
// manifest and verifies it with `openwork doctor --app`.
|
||||
|
||||
import { createHash } from "node:crypto"
|
||||
import { execFileSync, spawnSync } from "node:child_process"
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { dirname, join, resolve } from "node:path"
|
||||
import { fileURLToPath, pathToFileURL } from "node:url"
|
||||
|
||||
if (process.platform !== "darwin") {
|
||||
throw new Error("openwork install app DMG test evidence requires macOS")
|
||||
}
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const packageRoot = resolve(here, "..")
|
||||
const repoRoot = resolve(packageRoot, "..", "..")
|
||||
const cli = join(packageRoot, "bin", "openwork.mjs")
|
||||
const outDir = join(repoRoot, "evals", "results", "openwork-app-install-dmg")
|
||||
const temp = join(tmpdir(), `openwork-app-install-dmg-${Date.now()}-${Math.random().toString(36).slice(2)}`)
|
||||
const sourceDir = join(temp, "source")
|
||||
const appFixture = join(sourceDir, "OpenWork.app")
|
||||
const installDir = join(temp, "install")
|
||||
const binDir = join(temp, "bin")
|
||||
const appDir = join(temp, "Applications")
|
||||
const dmgPath = join(temp, "OpenWork.dmg")
|
||||
const manifestPath = join(temp, "openwork-install-manifest.json")
|
||||
const installedOpenwork = join(binDir, "openwork-bootstrap")
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
|
||||
const frames = []
|
||||
|
||||
function parseJson(text) {
|
||||
if (!text) return null
|
||||
try {
|
||||
return JSON.parse(text)
|
||||
} catch {
|
||||
return text
|
||||
}
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const result = spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 10_000, ...options })
|
||||
const stdout = result.stdout?.trim() ?? ""
|
||||
const stderr = result.stderr?.trim() ?? ""
|
||||
return {
|
||||
command: [command, ...args].join(" "),
|
||||
status: result.status,
|
||||
signal: result.signal,
|
||||
error: result.error ? String(result.error) : null,
|
||||
stdout,
|
||||
stderr,
|
||||
json: parseJson(stdout),
|
||||
}
|
||||
}
|
||||
|
||||
function prove(claim, { action, assert, evidence }, ok) {
|
||||
frames.push({ claim, action, assert, evidence, ok })
|
||||
if (!ok) throw new Error(`Frame failed: ${claim}`)
|
||||
}
|
||||
|
||||
function sha256(path) {
|
||||
return createHash("sha256").update(readFileSync(path)).digest("hex")
|
||||
}
|
||||
|
||||
try {
|
||||
mkdirSync(join(appFixture, "Contents", "MacOS"), { recursive: true })
|
||||
writeFileSync(join(appFixture, "Contents", "Info.plist"), `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0"><dict>
|
||||
<key>CFBundleExecutable</key><string>OpenWork</string>
|
||||
<key>CFBundleIdentifier</key><string>com.openwork.fixture</string>
|
||||
<key>CFBundleName</key><string>OpenWork</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>0.0.0-fixture</string>
|
||||
</dict></plist>
|
||||
`)
|
||||
writeFileSync(join(appFixture, "Contents", "MacOS", "OpenWork"), "#!/usr/bin/env sh\necho OpenWork fixture\n")
|
||||
chmodSync(join(appFixture, "Contents", "MacOS", "OpenWork"), 0o755)
|
||||
|
||||
execFileSync("hdiutil", ["create", "-quiet", "-volname", "OpenWork", "-srcfolder", sourceDir, "-ov", "-format", "UDZO", dmgPath])
|
||||
const digest = sha256(dmgPath)
|
||||
writeFileSync(manifestPath, JSON.stringify({
|
||||
version: "0.0.0-fixture",
|
||||
appName: "OpenWork.app",
|
||||
artifacts: {
|
||||
darwin: {
|
||||
[process.arch]: {
|
||||
type: "dmg",
|
||||
url: pathToFileURL(dmgPath).toString(),
|
||||
sha256: digest,
|
||||
appName: "OpenWork.app",
|
||||
},
|
||||
},
|
||||
},
|
||||
}, null, 2))
|
||||
prove("A real macOS DMG install manifest is available", {
|
||||
action: "Create OpenWork.app fixture, package it with hdiutil, and write manifest JSON",
|
||||
assert: "manifest references a .dmg with SHA-256 for this macOS architecture",
|
||||
evidence: { manifestPath, dmgPath, sha256: digest, arch: process.arch },
|
||||
}, existsSync(dmgPath) && existsSync(manifestPath) && digest.length === 64)
|
||||
|
||||
const installCli = run(process.execPath, [cli, "install", "--install-dir", installDir, "--bin-dir", binDir, "--json"])
|
||||
prove("The bootstrap CLI can be installed from a script", {
|
||||
action: "node bin/openwork.mjs install --install-dir <tmp> --bin-dir <tmp>/bin --json",
|
||||
assert: "exit 0 and an openwork executable exists",
|
||||
evidence: { status: installCli.status, body: installCli.json },
|
||||
}, installCli.status === 0 && existsSync(installedOpenwork))
|
||||
|
||||
const installApp = run(installedOpenwork, ["install", "app", "--manifest", manifestPath, "--app-dir", appDir, "--json"], { timeout: 30_000 })
|
||||
const appPath = join(appDir, "OpenWork.app")
|
||||
prove("The installed CLI can download, verify, mount, and install OpenWork.app from a DMG", {
|
||||
action: "openwork install app --manifest <fixture-manifest> --app-dir <tmp>/Applications --json",
|
||||
assert: "exit 0, checksum recorded, and OpenWork.app copied into app dir",
|
||||
evidence: { status: installApp.status, body: installApp.json, appExists: existsSync(appPath), stderr: installApp.stderr },
|
||||
}, installApp.status === 0 && installApp.json?.ok === true && installApp.json?.install?.artifact?.sha256 === digest && existsSync(appPath))
|
||||
|
||||
const doctor = run(installedOpenwork, ["doctor", "--install-dir", installDir, "--bin-dir", binDir, "--app", "--app-dir", appDir, "--json"])
|
||||
prove("doctor verifies the installed desktop app", {
|
||||
action: "openwork doctor --app --app-dir <tmp>/Applications --json",
|
||||
assert: "exit 0 with openworkApp and appInstallManifest checks passing",
|
||||
evidence: { status: doctor.status, body: doctor.json, stderr: doctor.stderr },
|
||||
}, doctor.status === 0 && doctor.json?.ok === true && doctor.json?.checks?.some((check) => check.name === "openworkApp" && check.ok) && doctor.json?.checks?.some((check) => check.name === "appInstallManifest" && check.ok))
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const esc = (value) => String(value).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
|
||||
const escAttr = (value) => esc(value).replace(/"/g, """)
|
||||
const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60)
|
||||
const frameFiles = frames.map((frame, index) => {
|
||||
const name = `${String(index + 1).padStart(2, "0")}-${slug(frame.claim)}.html`
|
||||
const html = `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>${esc(frame.claim)}</title>
|
||||
<style>body{margin:0;background:#f8fafc;color:#111827;font-family:ui-monospace,Menlo,Consolas,monospace}main{max-width:980px;margin:0 auto;padding:32px}h1{font-family:system-ui,sans-serif;margin-top:0;font-size:18px}.k{color:#6b7280;font-family:system-ui,sans-serif}.ok{color:#047857;font-weight:600}pre{white-space:pre-wrap;word-break:break-word;padding:16px;border:1px solid #d1d5db;border-radius:14px;background:white}</style>
|
||||
</head><body><main><h1>${esc(frame.claim)} — <span class="ok">${frame.ok ? "PASS" : "FAIL"}</span></h1>
|
||||
<p><span class="k">action:</span> ${esc(frame.action)}</p><p><span class="k">assert:</span> ${esc(frame.assert)}</p><p class="k">evidence:</p>
|
||||
<pre>${esc(JSON.stringify(frame.evidence, null, 2))}</pre></main></body></html>`
|
||||
writeFileSync(join(outDir, name), html)
|
||||
return { name, frame }
|
||||
})
|
||||
|
||||
const allOk = frames.every((frame) => frame.ok)
|
||||
// Keep fraimz.html only because the standalone legacy report contract consumes this filename.
|
||||
writeFileSync(join(outDir, "fraimz.html"), `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>OpenWork App DMG Install — test evidence</title>
|
||||
<style>body{margin:0;background:#f3f4f6;color:#111827;font-family:system-ui,sans-serif}main{max-width:1180px;margin:0 auto;padding:32px}.meta{color:#4b5563;margin-bottom:24px}section{margin:20px 0;padding:16px;border:1px solid #d1d5db;border-radius:16px;background:white}iframe{width:100%;min-height:360px;border:1px solid #e5e7eb;border-radius:12px;background:white}code{background:#e5e7eb;padding:2px 5px;border-radius:5px}</style>
|
||||
</head><body><main><h1>OpenWork App DMG Install — test evidence</h1><div class="meta">Result: <code>${allOk ? "passed" : "failed"}</code> · Screenshots: ${frames.length}</div>
|
||||
${frameFiles.map((entry) => `<section><h2>${esc(entry.frame.claim)}</h2><iframe src="${escAttr(entry.name)}" title="${escAttr(entry.frame.claim)}"></iframe><p><a href="${escAttr(entry.name)}">Open frame</a></p></section>`).join("\n")}
|
||||
</main></body></html>`)
|
||||
writeFileSync(join(outDir, "report.json"), JSON.stringify({
|
||||
flow: "openwork-app-install-dmg",
|
||||
result: allOk ? "passed" : "failed",
|
||||
frames: frames.map((frame) => ({ claim: frame.claim, action: frame.action, assert: frame.assert, ok: frame.ok })),
|
||||
}, null, 2))
|
||||
|
||||
console.log(`test evidence: ${join(outDir, "fraimz.html")}`)
|
||||
console.log(`result: ${allOk ? "passed" : "failed"} (${frames.length} frames)`)
|
||||
process.exit(allOk ? 0 : 1)
|
||||
@@ -0,0 +1,162 @@
|
||||
// Live E2E test evidence for the script-installable OpenWork bootstrap CLI.
|
||||
//
|
||||
// This proof installs the CLI into a temp bin directory, runs doctor, then uses
|
||||
// the installed command to drive real Den API onboarding over HTTP: sign up,
|
||||
// create org, invite teammate, and create a skill.
|
||||
//
|
||||
// Required:
|
||||
// DEN_API_E2E_BASE_URL=http://127.0.0.1:18990
|
||||
//
|
||||
// Run from packages/openwork-bootstrap:
|
||||
// DEN_API_E2E_BASE_URL=... pnpm exec node evals/agent-test-evidence-openwork-bootstrap.mjs
|
||||
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { dirname, join, resolve } from "node:path"
|
||||
import { spawnSync } from "node:child_process"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const baseUrl = process.env.DEN_API_E2E_BASE_URL?.replace(/\/$/, "")
|
||||
if (!baseUrl) {
|
||||
throw new Error("DEN_API_E2E_BASE_URL is required for live E2E test evidence")
|
||||
}
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const packageRoot = resolve(here, "..")
|
||||
const repoRoot = resolve(packageRoot, "..", "..")
|
||||
const cli = join(packageRoot, "bin", "openwork.mjs")
|
||||
const temp = mkdtempSync(join(tmpdir(), "openwork-bootstrap-e2e-"))
|
||||
const installDir = join(temp, "install")
|
||||
const binDir = join(temp, "bin")
|
||||
const installedOpenwork = join(binDir, process.platform === "win32" ? "openwork-bootstrap.cmd" : "openwork-bootstrap")
|
||||
const outDir = join(repoRoot, "evals", "results", "openwork-bootstrap-cli")
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
|
||||
const frames = []
|
||||
const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
|
||||
function parseJson(text) {
|
||||
if (!text) return null
|
||||
try {
|
||||
return JSON.parse(text)
|
||||
} catch {
|
||||
return text
|
||||
}
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const result = spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 10_000, ...options })
|
||||
const stdout = result.stdout?.trim() ?? ""
|
||||
const stderr = result.stderr?.trim() ?? ""
|
||||
return {
|
||||
command: [command, ...args].join(" "),
|
||||
status: result.status,
|
||||
signal: result.signal,
|
||||
error: result.error ? String(result.error) : null,
|
||||
stdout,
|
||||
stderr,
|
||||
json: parseJson(stdout),
|
||||
}
|
||||
}
|
||||
|
||||
function prove(claim, { action, assert, evidence }, ok) {
|
||||
frames.push({ claim, action, assert, evidence, ok })
|
||||
if (!ok) throw new Error(`Frame failed: ${claim}`)
|
||||
}
|
||||
|
||||
function randomPassword() {
|
||||
return `A${crypto.randomUUID()}!9zZ`
|
||||
}
|
||||
|
||||
try {
|
||||
const health = await fetch(`${baseUrl}/health`).then(async (response) => ({
|
||||
status: response.status,
|
||||
body: await response.json().catch(() => null),
|
||||
}))
|
||||
prove("Den API is running as the live e2e target", {
|
||||
action: `GET ${baseUrl}/health`,
|
||||
assert: "HTTP 200 { ok: true }",
|
||||
evidence: health,
|
||||
}, health.status === 200 && health.body?.ok === true)
|
||||
|
||||
const install = run(process.execPath, [cli, "install", "--install-dir", installDir, "--bin-dir", binDir, "--json"])
|
||||
prove("A bootstrap script can install the openwork CLI", {
|
||||
action: "node bin/openwork.mjs install --install-dir <tmp> --bin-dir <tmp>/bin --json",
|
||||
assert: "exit 0 and installed executable path returned",
|
||||
evidence: { status: install.status, body: install.json },
|
||||
}, install.status === 0 && install.json?.ok === true && install.json?.install?.executable === installedOpenwork)
|
||||
|
||||
const doctor = run(installedOpenwork, ["doctor", "--install-dir", installDir, "--bin-dir", binDir, "--base-url", baseUrl, "--json"])
|
||||
prove("The installed CLI can doctor itself and the Den API", {
|
||||
action: "openwork doctor --base-url <live-den-api> --json",
|
||||
assert: "exit 0, local install checks pass, and Den API health passes",
|
||||
evidence: { status: doctor.status, body: doctor.json },
|
||||
}, doctor.status === 0 && doctor.json?.ok === true && doctor.json?.checks?.some((check) => check.name === "denApiHealth" && check.ok === true))
|
||||
|
||||
const ownerEmail = `owner-${runId}@example.com`
|
||||
const inviteEmail = `teammate-${runId}@example.com`
|
||||
const orgName = `Bootstrap CLI Org ${runId}`
|
||||
const skillName = `Bootstrap Skill ${runId}`
|
||||
const onboard = run(installedOpenwork, [
|
||||
"cloud",
|
||||
"onboard",
|
||||
"--base-url",
|
||||
baseUrl,
|
||||
"--owner-email",
|
||||
ownerEmail,
|
||||
"--org-name",
|
||||
orgName,
|
||||
"--invite-email",
|
||||
inviteEmail,
|
||||
"--skill-name",
|
||||
skillName,
|
||||
"--json",
|
||||
], { env: { ...process.env, OPENWORK_OWNER_PASSWORD: randomPassword() }, timeout: 30_000 })
|
||||
prove("The installed CLI can onboard a user, org, invite, and skill end-to-end", {
|
||||
action: "openwork cloud onboard --base-url <live-den-api> --owner-email ... --org-name ... --invite-email ... --skill-name ... --json",
|
||||
assert: "exit 0 with user, organization, invitation, and skill ids from live API responses",
|
||||
evidence: { status: onboard.status, body: onboard.json, stderr: onboard.stderr },
|
||||
}, onboard.status === 0 && onboard.json?.ok === true && onboard.json?.organization?.id && onboard.json?.invitation?.invitationId && onboard.json?.skill?.id && onboard.json?.skill?.title === skillName && onboard.json?.skillRun?.triggered === true && onboard.json?.skillRun?.output === "OPENWORK_BOOTSTRAP_SKILL_TRIGGERED")
|
||||
} finally {
|
||||
rmSync(temp, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const esc = (value) => String(value).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
|
||||
const escAttr = (value) => esc(value).replace(/"/g, """)
|
||||
const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60)
|
||||
const frameFiles = frames.map((frame, index) => {
|
||||
const name = `${String(index + 1).padStart(2, "0")}-${slug(frame.claim)}.html`
|
||||
const html = `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>${esc(frame.claim)}</title>
|
||||
<style>body{margin:0;background:#f8fafc;color:#111827;font-family:ui-monospace,Menlo,Consolas,monospace}main{max-width:980px;margin:0 auto;padding:32px}h1{font-family:system-ui,sans-serif;margin-top:0;font-size:18px}.k{color:#6b7280;font-family:system-ui,sans-serif}.ok{color:#047857;font-weight:600}pre{white-space:pre-wrap;word-break:break-word;padding:16px;border:1px solid #d1d5db;border-radius:14px;background:white}</style>
|
||||
</head><body><main>
|
||||
<h1>${esc(frame.claim)} — <span class="ok">${frame.ok ? "PASS" : "FAIL"}</span></h1>
|
||||
<p><span class="k">action:</span> ${esc(frame.action)}</p>
|
||||
<p><span class="k">assert:</span> ${esc(frame.assert)}</p>
|
||||
<p class="k">evidence:</p>
|
||||
<pre>${esc(JSON.stringify(frame.evidence, null, 2))}</pre>
|
||||
</main></body></html>`
|
||||
writeFileSync(join(outDir, name), html)
|
||||
return { name, frame }
|
||||
})
|
||||
|
||||
const allOk = frames.every((frame) => frame.ok)
|
||||
// Keep fraimz.html only because the standalone legacy report contract consumes this filename.
|
||||
writeFileSync(join(outDir, "fraimz.html"), `<!doctype html><html lang="en"><head><meta charset="utf-8" />
|
||||
<title>OpenWork Bootstrap CLI — live E2E test evidence</title>
|
||||
<style>body{margin:0;background:#f3f4f6;color:#111827;font-family:system-ui,sans-serif}main{max-width:1180px;margin:0 auto;padding:32px}h1{margin-bottom:4px}.meta{color:#4b5563;margin-bottom:24px}section{margin:20px 0;padding:16px;border:1px solid #d1d5db;border-radius:16px;background:white}iframe{width:100%;min-height:360px;border:1px solid #e5e7eb;border-radius:12px;background:white}code{background:#e5e7eb;padding:2px 5px;border-radius:5px}</style>
|
||||
</head><body><main>
|
||||
<h1>OpenWork Bootstrap CLI — live E2E test evidence</h1>
|
||||
<div class="meta">Result: <code>${allOk ? "passed" : "failed"}</code> · Live Den API: <code>${esc(baseUrl)}</code> · Frames: ${frames.length}</div>
|
||||
${frameFiles.map((entry) => `<section><h2>${esc(entry.frame.claim)}</h2><iframe src="${escAttr(entry.name)}" title="${escAttr(entry.frame.claim)}"></iframe><p><a href="${escAttr(entry.name)}">Open frame</a></p></section>`).join("\n")}
|
||||
</main></body></html>`)
|
||||
writeFileSync(join(outDir, "report.json"), JSON.stringify({
|
||||
flow: "openwork-bootstrap-cli",
|
||||
result: allOk ? "passed" : "failed",
|
||||
liveBaseUrl: baseUrl,
|
||||
frames: frames.map((frame) => ({ claim: frame.claim, action: frame.action, assert: frame.assert, ok: frame.ok })),
|
||||
}, null, 2))
|
||||
|
||||
console.log(`test evidence: ${join(outDir, "fraimz.html")}`)
|
||||
console.log(`result: ${allOk ? "passed" : "failed"} (${frames.length} frames)`)
|
||||
process.exit(allOk ? 0 : 1)
|
||||
Reference in New Issue
Block a user