import { expect } from "vitest" import { createOrgConnection, denFetch } from "@openwork/behaviors" import { mcpMock, needs, server, test } from "@openwork/testkit" const requirements = { optIn: ["OPENWORK_EVAL_E2E_TESTS", "OPENWORK_EVAL_SAVED_SCRIPT_AUTOMATIONS_E2E_TEST"], } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value) } function requireRecord(value: unknown, label: string): Record { if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`) return value } function records(value: unknown): Record[] { return Array.isArray(value) ? value.filter(isRecord) : [] } async function eventually( read: () => Promise, accepted: (value: T) => boolean, label: string, timeoutMs = 180_000, ): Promise { const deadline = Date.now() + timeoutMs let latest: T | undefined while (Date.now() < deadline) { latest = await read() if (accepted(latest)) return latest await new Promise((resolve) => setTimeout(resolve, 2_000)) } throw new Error(`Timed out waiting for ${label}: ${JSON.stringify(latest).slice(0, 1_000)}`) } let mcpRequestId = 0 async function agentRpc( apiUrl: string, token: string, method: string, params: Record, ): Promise> { const response = await fetch(`${apiUrl}/mcp/agent`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json, text/event-stream", }, body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpRequestId, method, params }), signal: AbortSignal.timeout(180_000), }) const raw = await response.text() if (!response.ok) throw new Error(`MCP ${method} failed: HTTP ${response.status} ${raw.slice(0, 500)}`) const dataLine = raw.split("\n").find((line) => line.startsWith("data:")) if (!dataLine) throw new Error(`MCP ${method} returned no SSE data frame: ${raw.slice(0, 500)}`) const message = requireRecord(JSON.parse(dataLine.slice(5)), "MCP response") if (message.error) throw new Error(`MCP ${method} returned an error: ${JSON.stringify(message.error)}`) return requireRecord(message.result, `MCP ${method} result`) } test("a Code Mode result becomes a cloud Automation and a durable artifact result", { timeout: 1_200_000 }, async ({ evidence, place }) => { needs(requirements) await using den = await server({ place, org: { name: `Workflow Automation ${Date.now()}`, admin: { name: "Sarah" } }, mocks: { reports: mcpMock({ allowUnauthenticatedMcp: true }) }, }) const orgs = await denFetch(den.admin, "/v1/me/orgs", { headers: { authorization: `Bearer ${den.admin.token}` }, }) const orgRows = isRecord(orgs.body) ? records(orgs.body.orgs) : [] const organizationId = String(orgRows[0]?.id ?? "") expect(organizationId).not.toBe("") const connection = await createOrgConnection(den.admin, { name: "Report source", url: den.mocks.reports.mcpUrl, authType: "none", credentialMode: "shared", access: { orgWide: true }, }) const catalog = await denFetch(den.admin, `/v1/mcp-connections/${connection.id}/tools`, { headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(catalog.response.ok, catalog.text).toBe(true) const catalogTools = isRecord(catalog.body) ? records(catalog.body.tools) : [] expect(catalogTools.some((tool) => tool.name === "mock_echo")).toBe(true) const tokenResponse = await denFetch(den.admin, "/v1/mcp/token", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}`, "x-openwork-org-id": organizationId, }, body: JSON.stringify({ scopes: ["mcp:read", "mcp:write"] }), }) expect(tokenResponse.response.ok, tokenResponse.text).toBe(true) const mcpToken = isRecord(tokenResponse.body) && typeof tokenResponse.body.token === "string" ? tokenResponse.body.token : "" expect(mcpToken).toMatch(/^ow_mcp_at_/) const stamp = Date.now() const scriptName = `Launch briefing ${stamp}` const firstMarker = `launch-now-${stamp}` const scheduledMarker = `launch-scheduled-${stamp}` const code = [ "const result = await tools.den.getWorkers({})", "return { briefing: { topic: input.topic, workerCount: result.workers.length } }", ].join("\n") const inputSchema = { type: "object", properties: { topic: { type: "string" } }, required: ["topic"], additionalProperties: false, } const outputSchema = { type: "object", properties: { briefing: {} }, required: ["briefing"], additionalProperties: false, } const executed = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", { name: "execute_capability_script", arguments: { code, input: { topic: firstMarker } }, }) expect(executed.isError).not.toBe(true) expect(JSON.stringify(executed.content)).toContain(firstMarker) const savedResponse = await denFetch(den.admin, "/v1/workflows", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ name: scriptName, description: "Builds a reusable launch briefing from the organization's worker roster.", code, currentInput: { topic: firstMarker }, inputSchema, outputSchema, }), }) expect(savedResponse.response.status, savedResponse.text).toBe(201) const saved = requireRecord(savedResponse.body, "saved Workflow") const pluginId = typeof saved.pluginId === "string" ? saved.pluginId : "" const configObjectId = typeof saved.configObjectId === "string" ? saved.configObjectId : "" const configObjectVersionId = typeof saved.configObjectVersionId === "string" ? saved.configObjectVersionId : "" expect(pluginId).not.toBe("") expect(configObjectId).not.toBe("") expect(configObjectVersionId).not.toBe("") evidence.recordAssertionEvidence( "A successful ad-hoc Code Mode result is promotable without retyping its procedure", "The exact successful code was saved as an immutable Workflow version using its recent receipt.", true, ) const manualRun = await denFetch(den.admin, `/v1/workflows/${configObjectId}/run`, { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ pluginId, configObjectVersionId, input: { topic: firstMarker } }), }) expect(manualRun.response.ok, manualRun.text).toBe(true) const manualResult = requireRecord(manualRun.body, "manual Script result") expect(manualResult.status).toBe("succeeded") expect(JSON.stringify(manualResult.value)).toContain(firstMarker) expect(String(manualResult.receiptId ?? "")).not.toBe("") evidence.recordAssertionEvidence( "The Workflow produces a validated artifact-ready result", "A direct run of the immutable version returned a schema-valid result and durable receipt.", true, ) const scheduledAfter = new Date().toISOString() const automationResponse = await denFetch(den.admin, "/v1/cloud-automations", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ name: `${scriptName} once`, schedule: { kind: "once", timezone: "UTC", at: Date.now() + 30_000 }, action: { kind: "saved_script", script: { pluginId, configObjectId, configObjectVersionId }, input: { topic: scheduledMarker }, }, }), }) expect(automationResponse.response.status, automationResponse.text).toBe(201) const automationDetail = requireRecord(automationResponse.body, "Automation") const automation = requireRecord(automationDetail.automation, "Automation identity") const automationId = typeof automation.id === "string" ? automation.id : "" expect(automationId).not.toBe("") const scheduledRun = await eventually(async () => { const response = await denFetch(den.admin, `/v1/automations/${automationId}/runs`, { headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(response.response.ok, response.text).toBe(true) return isRecord(response.body) ? records(response.body.items).find((run) => run.trigger === "scheduled") : undefined }, (run) => run?.status === "succeeded", "scheduled Workflow Automation to succeed", 5 * 60_000) const scheduledRunId = typeof scheduledRun?.id === "string" ? scheduledRun.id : "" expect(scheduledRunId).not.toBe("") const scheduledExternalCalls = await den.mocks.reports.toolCalls({ name: "mock_echo", sinceIso: scheduledAfter, }) expect(scheduledExternalCalls).toHaveLength(0) const scheduledReceiptResponse = await denFetch(den.admin, `/v1/automation-runs/${scheduledRunId}`, { headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(scheduledReceiptResponse.response.ok, scheduledReceiptResponse.text).toBe(true) const scheduledReceipt = requireRecord(scheduledReceiptResponse.body, "scheduled Automation receipt") const scheduledReceiptRun = requireRecord(scheduledReceipt.run, "scheduled Automation run") const scheduledReceiptAutomation = requireRecord(scheduledReceipt.automation, "scheduled Automation identity") const scheduledReceiptRevision = requireRecord(scheduledReceipt.revision, "scheduled Automation revision") const scheduledExecutionThread = requireRecord(scheduledReceiptRun.executionThread, "scheduled Automation execution thread") expect(JSON.stringify(scheduledReceipt)).toContain(scheduledMarker) expect(scheduledReceiptAutomation.id).toBe(automationId) expect(scheduledReceiptRevision.id).toBe(scheduledRun?.revisionId) expect(Array.isArray(scheduledReceipt.events)).toBe(true) expect(scheduledReceipt.events).toEqual([]) expect(String(scheduledExecutionThread.id ?? "")).not.toBe("") expect(scheduledExecutionThread).toMatchObject({ threadKind: "automation", executionLocation: "cloud", automationId, automationRunId: scheduledRunId, engineKind: "openwork-cloud-codemode-v1", }) const toolList = await agentRpc(den.ref.apiUrl, mcpToken, "tools/list", {}) const tools = records(toolList.tools) const renderTool = tools.find((candidate) => candidate.name === "render_workflow_artifact") const renderToolMeta = isRecord(renderTool?._meta) ? renderTool._meta : {} const modernUi = isRecord(renderToolMeta.ui) ? renderToolMeta.ui : {} expect(modernUi.resourceUri).toBe("ui://openwork/workflow-artifact/v1/view.html") expect(renderToolMeta["ui/resourceUri"]).toBe("ui://openwork/workflow-artifact/v1/view.html") const resourceList = await agentRpc(den.ref.apiUrl, mcpToken, "resources/list", {}) const resources = records(resourceList.resources) const appResource = resources.find((candidate) => candidate.uri === "ui://openwork/workflow-artifact/v1/view.html") expect(appResource?.mimeType).toBe("text/html;profile=mcp-app") const resourceRead = await agentRpc(den.ref.apiUrl, mcpToken, "resources/read", { uri: "ui://openwork/workflow-artifact/v1/view.html", }) const resourceContents = records(resourceRead.contents) expect(resourceContents[0]?.mimeType).toBe("text/html;profile=mcp-app") expect(String(resourceContents[0]?.text ?? "")).toContain("ui/initialize") expect(String(resourceContents[0]?.text ?? "")).not.toContain("fetch(") const rendered = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", { name: "render_workflow_artifact", arguments: { configObjectId }, }) expect(rendered.isError).not.toBe(true) const structured = requireRecord(rendered.structuredContent, "Workflow Artifact structuredContent") const artifact = requireRecord(structured.artifact, "Workflow Artifact lineage") const fallback = records(rendered.content) expect(structured.schemaVersion).toBe("1") expect(artifact.configObjectId).toBe(configObjectId) expect(artifact.source).toBe("scheduled") expect(String(artifact.receiptId ?? "")).not.toBe("") expect(JSON.stringify(structured.data)).toContain(scheduledMarker) expect(String(fallback[0]?.text ?? "")).toContain(scheduledMarker) evidence.recordAssertionEvidence( "The latest Automation snapshot is portable as a standards-based MCP App", "The agent endpoint returns the scheduled result as versioned structuredContent and a Markdown fallback linked to a self-contained ui:// resource.", true, ) const externalMarker = `launch-external-${stamp}` const externalCode = "return { briefing: await tools.report_source.mock_echo({ text: input.topic }) }" const externalRunStartedAt = new Date().toISOString() const externalExecuted = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", { name: "execute_capability_script", arguments: { code: externalCode, input: { topic: externalMarker } }, }) expect(externalExecuted.isError).not.toBe(true) expect(JSON.stringify(externalExecuted.content)).toContain(externalMarker) const interactiveExternalCalls = await den.mocks.reports.toolCalls({ name: "mock_echo", atLeast: 1, sinceIso: externalRunStartedAt, timeoutMs: 60_000, }) expect(interactiveExternalCalls.filter((call) => call.args.text === externalMarker)).toHaveLength(1) const externalSavedResponse = await denFetch(den.admin, "/v1/workflows", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ name: `${scriptName} external`, description: "Checks the unattended Cloud boundary for external MCP tools.", code: externalCode, currentInput: { topic: externalMarker }, inputSchema, outputSchema, }), }) expect(externalSavedResponse.response.status, externalSavedResponse.text).toBe(201) const externalSaved = requireRecord(externalSavedResponse.body, "external saved Workflow") const externalPluginId = typeof externalSaved.pluginId === "string" ? externalSaved.pluginId : "" const externalConfigObjectId = typeof externalSaved.configObjectId === "string" ? externalSaved.configObjectId : "" const externalConfigObjectVersionId = typeof externalSaved.configObjectVersionId === "string" ? externalSaved.configObjectVersionId : "" expect(externalPluginId).not.toBe("") expect(externalConfigObjectId).not.toBe("") expect(externalConfigObjectVersionId).not.toBe("") const externalAutomation = await denFetch(den.admin, `/v1/automations/${automationId}`, { method: "PATCH", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ action: { kind: "saved_script", script: { pluginId: externalPluginId, configObjectId: externalConfigObjectId, configObjectVersionId: externalConfigObjectVersionId, }, input: { topic: externalMarker }, }, }), }) expect(externalAutomation.response.ok, externalAutomation.text).toBe(true) const unattendedRunStartedAt = new Date().toISOString() const failedRunResponse = await denFetch(den.admin, `/v1/automations/${automationId}/run`, { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(failedRunResponse.response.status, failedRunResponse.text).toBe(202) const queued = isRecord(failedRunResponse.body) ? requireRecord(failedRunResponse.body.run, "queued Automation run") : {} const failedRunId = typeof queued.id === "string" ? queued.id : "" expect(failedRunId).not.toBe("") const failedReceipt = await eventually(async () => { const response = await denFetch(den.admin, `/v1/automation-runs/${failedRunId}`, { headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(response.response.ok, response.text).toBe(true) return requireRecord(response.body, "failed Automation receipt") }, (receipt) => isRecord(receipt.run) && ["failed", "skipped", "cancelled"].includes(String(receipt.run.status)), "external-capability run to finish") const failedReceiptRun = requireRecord(failedReceipt.run, "failed Automation run") expect(failedReceiptRun.status).toBe("failed") const failedRunError = requireRecord(failedReceiptRun.error, "failed Automation error") expect(String(failedRunError.message ?? "")).toContain("must be read-only and explicitly approved") const afterBoundaryRejection = await eventually(async () => { const response = await denFetch(den.admin, `/v1/automations/${automationId}`, { headers: { authorization: `Bearer ${den.admin.token}` }, }) expect(response.response.ok, response.text).toBe(true) return requireRecord(response.body, "Automation after unattended boundary rejection") }, (detail) => isRecord(detail.automation) && detail.automation.state === "needs_attention", "Automation to need attention") expect(JSON.stringify(afterBoundaryRejection)).toContain(scheduledMarker) let unattendedExternalCalls = 0 try { unattendedExternalCalls = (await den.mocks.reports.toolCalls({ name: "mock_echo", atLeast: 1, sinceIso: unattendedRunStartedAt, timeoutMs: 5_000, })).length } catch { unattendedExternalCalls = 0 } expect(unattendedExternalCalls).toBe(0) evidence.recordAssertionEvidence( "Unattended Cloud rejects external MCP capability access before provider I/O and preserves the last good result", `Provider calls from the unattended run: ${unattendedExternalCalls}; the previous ${scheduledMarker} result remains durable.`, unattendedExternalCalls === 0 && JSON.stringify(afterBoundaryRejection).includes(scheduledMarker), ) })