import { expect, onTestFinished } from "vitest"; import { createOrgConnection, deleteConnection, denFetch, evalIn, go } from "@openwork/behaviors"; import type { DenSession } from "@openwork/behaviors"; import { screenshot, validate } from "@openwork/test-evidence"; import type { Surface } from "@openwork/cdp"; import { app, eventually, localMysqlIsRunning, mcpMock, needs, readConnectState, server, test, } from "@openwork/testkit"; const e2eTestsEnabled = process.env.OPENWORK_EVAL_E2E_TESTS === "1"; const remotePlacement = process.env.OPENWORK_EVAL_DAYTONA === "1" || Boolean(process.env.OPENWORK_EVAL_DEN_API_URL?.trim()); const mysqlOpen = remotePlacement || await localMysqlIsRunning(); const title = !e2eTestsEnabled ? "preseeded Connect readiness skipped — needs: set OPENWORK_EVAL_E2E_TESTS=1" : !mysqlOpen ? "preseeded Connect readiness skipped — needs: MySQL on 127.0.0.1:3306 for local placement" : "bundled engine connects to preseeded organization skills and connections"; let requestId = 0; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function requireRecord(value: unknown, label: string): Record { if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`); return value; } function toolText(result: unknown): string { const record = requireRecord(result, "MCP tool result"); const first = Array.isArray(record.content) ? record.content[0] : null; if (!isRecord(first) || typeof first.text !== "string") { throw new Error(`MCP tool result had no text content: ${JSON.stringify(result).slice(0, 500)}`); } return first.text; } function toolJson(result: unknown): unknown { return JSON.parse(toolText(result)); } function searchMatches(result: unknown): Record[] { const payload = requireRecord(toolJson(result), "search_capabilities payload"); return Array.isArray(payload.matches) ? payload.matches.filter(isRecord) : []; } async function organizationId(session: DenSession): Promise { const result = await denFetch(session, "/v1/me/orgs", { headers: { authorization: `Bearer ${session.token}` }, }); const orgs = isRecord(result.body) && Array.isArray(result.body.orgs) ? result.body.orgs.filter(isRecord) : []; const id = orgs[0] && typeof orgs[0].id === "string" ? orgs[0].id : ""; if (!result.response.ok || !id) { throw new Error(`Finding the provisioned organization failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`); } return id; } async function mintMcpToken(session: DenSession, orgId: string): Promise { const result = await denFetch(session, "/v1/mcp/token", { method: "POST", headers: { authorization: `Bearer ${session.token}`, "x-openwork-org-id": orgId }, body: JSON.stringify({}), }); const mcpToken = isRecord(result.body) && typeof result.body.token === "string" ? result.body.token : ""; if (!result.response.ok || !mcpToken.startsWith("ow_mcp_at_")) { throw new Error(`Minting the member Connect token failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`); } return mcpToken; } async function callTool( apiUrl: string, mcpToken: string, name: "search_capabilities" | "execute_capability", args: Record, ): Promise { const response = await fetch(`${apiUrl}/mcp/agent`, { method: "POST", headers: { authorization: `Bearer ${mcpToken}`, "content-type": "application/json", accept: "application/json, text/event-stream", }, body: JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method: "tools/call", params: { name, arguments: args }, }), }); const raw = await response.text(); if (!response.ok) throw new Error(`MCP tools/call failed: HTTP ${response.status} ${raw.slice(0, 500)}`); const dataLine = raw.split("\n").find((line) => line.startsWith("data:")); if (!dataLine) throw new Error(`MCP tools/call returned no SSE data frame: ${raw.slice(0, 500)}`); const payload: unknown = JSON.parse(dataLine.slice(5)); const record = requireRecord(payload, "MCP JSON-RPC payload"); if (record.error) throw new Error(`MCP tools/call returned JSON-RPC error: ${JSON.stringify(record.error)}`); return record.result; } async function readCloudMcpHealth(surface: Surface, workspaceId: string): Promise> { const raw = await evalIn(surface, `(async () => { const port = localStorage.getItem("openwork.server.port"); const token = localStorage.getItem("openwork.server.token"); if (!port || !token) return JSON.stringify({ error: "missing local server credentials" }); const response = await fetch( "http://127.0.0.1:" + port + "/workspace/" + encodeURIComponent(${JSON.stringify(workspaceId)}) + "/mcp/openwork-cloud/health?probe=1", { headers: { Authorization: "Bearer " + token } }, ); const text = await response.text(); if (!response.ok) return JSON.stringify({ error: "HTTP " + response.status, body: text.slice(0, 500) }); return text; })()`, { awaitPromise: true, timeoutMs: 30_000 }); if (typeof raw !== "string") throw new Error(`Cloud MCP health response was not text: ${JSON.stringify(raw)}`); return requireRecord(JSON.parse(raw), "Cloud MCP health response"); } function healthIsReady(health: Record): boolean { const engine = isRecord(health.engine) ? health.engine : null; const tools = isRecord(health.tools) ? health.tools : null; const direct = tools && isRecord(tools.direct) ? tools.direct : null; return health.phase === "ready" && health.usable === true && engine?.status === "connected" && Array.isArray(tools?.present) && tools.present.includes("openwork-cloud_search_capabilities") && tools.present.includes("openwork-cloud_execute_capability") && Array.isArray(direct?.present) && direct.present.includes("search_capabilities") && direct.present.includes("execute_capability"); } test.skipIf(!e2eTestsEnabled || !mysqlOpen)(title, async ({ evidence, place }) => { needs({ optIn: ["OPENWORK_EVAL_E2E_TESTS"] }); const run = Date.now(); const skillName = `pr3806-connect-proof-${run}`; const connectionName = `PR3806 conn ${String(run).slice(-6)}`; const nonsenseName = `no-such-capability-${run}`; const rawSourceText = `---\nname: ${skillName}\ndescription: Proves preseeded Connect skill discovery on OpenCode 1.18.18.\n---\n\nReturn the PR 3806 Connect proof phrase.`; await using den = await server({ place, org: { name: `PR 3806 Connect Readiness ${run}`, admin: { email: `pr3806-connect-admin-${run}@openwork.test`, name: "PR 3806 Connect Admin", password: "OpenWorkEval123!", }, }, mocks: { connector: mcpMock() }, }); const orgId = await organizationId(den.admin); const createdSkill = await denFetch(den.admin, "/v1/plugins", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}`, "x-openwork-org-id": orgId }, body: JSON.stringify({ name: skillName, orgWide: true, components: [{ type: "skill", input: { rawSourceText } }], }), }); const skillItem = isRecord(createdSkill.body) && isRecord(createdSkill.body.item) ? createdSkill.body.item : null; const pluginId = skillItem && typeof skillItem.id === "string" ? skillItem.id : ""; if (!createdSkill.response.ok || !pluginId) { throw new Error(`Creating the org-wide proof skill failed: HTTP ${createdSkill.response.status} ${createdSkill.text.slice(0, 500)}`); } onTestFinished(async () => { await denFetch(den.admin, `/v1/plugins/${encodeURIComponent(pluginId)}/archive`, { method: "POST", headers: { authorization: `Bearer ${den.admin.token}`, "x-openwork-org-id": orgId }, }).catch(() => undefined); }); const connectionInput = { name: connectionName, url: den.mocks.connector.mcpUrl, authType: "oauth", credentialMode: "per_member", access: { orgWide: true }, }; const connection = await createOrgConnection(den.admin, connectionInput); onTestFinished(async () => deleteConnection(den.admin, connection.id).catch(() => undefined)); evidence.recordAssertionEvidence( "The organization was seeded before desktop boot", `Skill plugin ${pluginId} used components=[{type:"skill",input:{rawSourceText}}] with orgWide=true; connection input: ${JSON.stringify(connectionInput)}.`, createdSkill.response.status === 201 && connection.name === connectionName, ); await using desktopApp = await app({ den, as: "admin", place, beforeSignIn: async (surface) => { const signedOutState = await eventually(() => readConnectState(surface), { within: 15_000, label: "signed-out fresh-profile Connect state", until: (state) => state.ok && state.status === "missing", }); const signedOutNotAvailable = signedOutState.status !== "available"; expect(signedOutState.status).toBe("missing"); expect(signedOutState.connectEnabled).toBe(false); expect(signedOutNotAvailable).toBe(true); evidence.recordAssertionEvidence( "A signed-out fresh profile has no Connect capability access", `Observed Connect state before sign-in: ${JSON.stringify(signedOutState)}.`, signedOutState.status === "missing" && signedOutState.connectEnabled === false && signedOutNotAvailable, ); const shot = await screenshot(surface); const seen = await validate(shot, [ "The OpenWork desktop is visible before organization sign-in", "No crash or error dialog is visible", ]); expect(seen.ok, seen.why).toBe(true); }, }); const signedInState = await eventually(() => readConnectState(desktopApp), { within: 90_000, label: "signed-in available Connect state", until: (state) => state.ok && state.status === "available" && state.connectEnabled === true, }); const signedInNotMissing = signedInState.status !== "missing"; expect(signedInState.status).toBe("available"); expect(signedInState.connectEnabled).toBe(true); expect(signedInNotMissing).toBe(true); evidence.recordAssertionEvidence( "Organization sign-in enables Connect", `Observed Connect state after sign-in: ${JSON.stringify(signedInState)}.`, signedInState.status === "available" && signedInState.connectEnabled === true && signedInNotMissing, ); const health = await eventually(() => readCloudMcpHealth(desktopApp, desktopApp.workspaceId), { within: 180_000, label: "OpenCode 1.18.18 openwork-cloud engine and agent-tool readiness", until: healthIsReady, }); const engine = requireRecord(health.engine, "Cloud MCP engine health"); const tools = requireRecord(health.tools, "Cloud MCP tools health"); expect(health.phase).toBe("ready"); expect(health.usable).toBe(true); expect(engine.status).toBe("connected"); expect(engine.status).not.toBe("needs_auth"); expect(engine.status).not.toBe("failed"); expect(engine.status).not.toBe("needs_client_registration"); expect(tools.present).toEqual(expect.arrayContaining([ "openwork-cloud_search_capabilities", "openwork-cloud_execute_capability", ])); evidence.recordAssertionEvidence( "OpenCode 1.18.18 connects openwork-cloud with both agent tools", `Health payload: ${JSON.stringify({ phase: health.phase, usable: health.usable, engine, tools: tools.present })}.`, healthIsReady(health) && engine.status !== "needs_auth" && engine.status !== "failed" && engine.status !== "needs_client_registration", ); const mcpToken = await mintMcpToken(den.admin, orgId); const skillSearch = await callTool(den.ref.apiUrl, mcpToken, "search_capabilities", { query: skillName, limit: 20, type: "skills", }); const skillMatch = searchMatches(skillSearch).find((match) => ( match.kind === "skill" && typeof match.name === "string" && match.name.startsWith(`plugin:${pluginId}:`) )); if (!skillMatch || typeof skillMatch.name !== "string") throw new Error(`Connect did not discover the exact skill ${skillName}.`); const capabilityName = skillMatch.name; expect(skillMatch.kind).toBe("skill"); expect(capabilityName.startsWith(`plugin:${pluginId}:`)).toBe(true); const execution = await callTool(den.ref.apiUrl, mcpToken, "execute_capability", { name: capabilityName }); expect(isRecord(execution) && execution.isError === true).toBe(false); expect(toolJson(execution)).toMatchObject({ kind: "skill", content: rawSourceText }); evidence.recordAssertionEvidence( "The member's Connect token discovers and executes the preseeded skill", `Search query ${skillName} returned ${JSON.stringify(skillMatch)}; execution returned the exact ${rawSourceText.length}-character source.`, skillMatch.kind === "skill" && capabilityName.startsWith(`plugin:${pluginId}:`) && JSON.stringify(toolJson(execution)).includes(rawSourceText), ); const nonsenseSearch = await callTool(den.ref.apiUrl, mcpToken, "search_capabilities", { query: nonsenseName, limit: 20, type: "skills", }); const nonsenseSkillMatches = searchMatches(nonsenseSearch).filter((match) => ( match.kind === "skill" && JSON.stringify(match).includes(nonsenseName) )); expect(nonsenseSkillMatches).toEqual([]); evidence.recordAssertionEvidence( "Connect does not invent a nonexistent skill", `Search query ${nonsenseName} returned no skill match naming that capability.`, nonsenseSkillMatches.length === 0, ); const connectionSearch = await callTool(den.ref.apiUrl, mcpToken, "search_capabilities", { query: connectionName, limit: 20, type: "mcp", }); const connectionMatch = searchMatches(connectionSearch).find((match) => { const status = isRecord(match.connectionStatus) ? match.connectionStatus : null; return status?.connectionName === connectionName || JSON.stringify(match).includes(connectionName); }); if (!connectionMatch) throw new Error(`Connect did not discover the preseeded connection ${connectionName}.`); const connectionStatus = isRecord(connectionMatch.connectionStatus) ? connectionMatch.connectionStatus : null; const readiness = connectionStatus?.state === "needs_connection" && connectionStatus.actor === "member" ? "needs_signin" : connectionStatus?.actor === "organization_admin" ? "needs_admin_setup" : "ready"; expect(["ready", "needs_signin", "needs_admin_setup"]).toContain(readiness); evidence.recordAssertionEvidence( "The preseeded organization connection is discoverable with truthful readiness", `Connection match: ${JSON.stringify(connectionMatch)}; normalized readiness=${readiness}.`, Boolean(connectionMatch) && ["ready", "needs_signin", "needs_admin_setup"].includes(readiness), ); const connectionsDeadline = Date.now() + 30_000; while (Date.now() < connectionsDeadline) { const settled = await evalIn( desktopApp, `document.body.innerText.includes(${JSON.stringify(connectionName)})`, { timeoutMs: 5_000 }, ).catch(() => false); if (settled === true) break; await go(desktopApp, "/workspace/" + desktopApp.workspaceId + "/settings/extensions/connections").catch(() => undefined); await new Promise((resolve) => setTimeout(resolve, 1_500)); } expect(await evalIn(desktopApp, `document.body.innerText.includes(${JSON.stringify(connectionName)})`)).toBe(true); const signedInShot = await screenshot(desktopApp); const signedInSeen = await validate(signedInShot, [ `A Library view of skills, connections, and tools lists an organization connection card named '${connectionName}'`, "No crash or error dialog is visible", ]); expect(signedInSeen.ok, signedInSeen.why).toBe(true); });