源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
@@ -0,0 +1,383 @@
|
||||
import { expect } from "vitest"
|
||||
import { createOrgConnection, denFetch } from "@openwork/behaviors"
|
||||
import { mcpMock, needs, server, test } from "@openwork/testkit"
|
||||
|
||||
const requirements = {
|
||||
optIn: ["OPENWORK_EVAL_E2E_TESTS", "OPENWORK_EVAL_SAVED_SCRIPT_AUTOMATIONS_E2E_TEST"],
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function requireRecord(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter(isRecord) : []
|
||||
}
|
||||
|
||||
async function eventually<T>(
|
||||
read: () => Promise<T>,
|
||||
accepted: (value: T) => boolean,
|
||||
label: string,
|
||||
timeoutMs = 180_000,
|
||||
): Promise<T> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
let latest: T | undefined
|
||||
while (Date.now() < deadline) {
|
||||
latest = await read()
|
||||
if (accepted(latest)) return latest
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_000))
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${label}: ${JSON.stringify(latest).slice(0, 1_000)}`)
|
||||
}
|
||||
|
||||
let mcpRequestId = 0
|
||||
|
||||
async function agentRpc(
|
||||
apiUrl: string,
|
||||
token: string,
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const response = await fetch(`${apiUrl}/mcp/agent`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
"content-type": "application/json",
|
||||
accept: "application/json, text/event-stream",
|
||||
},
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpRequestId, method, params }),
|
||||
signal: AbortSignal.timeout(180_000),
|
||||
})
|
||||
const raw = await response.text()
|
||||
if (!response.ok) throw new Error(`MCP ${method} failed: HTTP ${response.status} ${raw.slice(0, 500)}`)
|
||||
const dataLine = raw.split("\n").find((line) => line.startsWith("data:"))
|
||||
if (!dataLine) throw new Error(`MCP ${method} returned no SSE data frame: ${raw.slice(0, 500)}`)
|
||||
const message = requireRecord(JSON.parse(dataLine.slice(5)), "MCP response")
|
||||
if (message.error) throw new Error(`MCP ${method} returned an error: ${JSON.stringify(message.error)}`)
|
||||
return requireRecord(message.result, `MCP ${method} result`)
|
||||
}
|
||||
|
||||
test("a Code Mode result becomes a cloud Automation and a durable artifact result", { timeout: 1_200_000 }, async ({ evidence, place }) => {
|
||||
needs(requirements)
|
||||
await using den = await server({
|
||||
place,
|
||||
org: { name: `Workflow Automation ${Date.now()}`, admin: { name: "Sarah" } },
|
||||
mocks: { reports: mcpMock({ allowUnauthenticatedMcp: true }) },
|
||||
})
|
||||
const orgs = await denFetch(den.admin, "/v1/me/orgs", {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
const orgRows = isRecord(orgs.body) ? records(orgs.body.orgs) : []
|
||||
const organizationId = String(orgRows[0]?.id ?? "")
|
||||
expect(organizationId).not.toBe("")
|
||||
|
||||
const connection = await createOrgConnection(den.admin, {
|
||||
name: "Report source",
|
||||
url: den.mocks.reports.mcpUrl,
|
||||
authType: "none",
|
||||
credentialMode: "shared",
|
||||
access: { orgWide: true },
|
||||
})
|
||||
const catalog = await denFetch(den.admin, `/v1/mcp-connections/${connection.id}/tools`, {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(catalog.response.ok, catalog.text).toBe(true)
|
||||
const catalogTools = isRecord(catalog.body) ? records(catalog.body.tools) : []
|
||||
expect(catalogTools.some((tool) => tool.name === "mock_echo")).toBe(true)
|
||||
|
||||
const tokenResponse = await denFetch(den.admin, "/v1/mcp/token", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${den.admin.token}`,
|
||||
"x-openwork-org-id": organizationId,
|
||||
},
|
||||
body: JSON.stringify({ scopes: ["mcp:read", "mcp:write"] }),
|
||||
})
|
||||
expect(tokenResponse.response.ok, tokenResponse.text).toBe(true)
|
||||
const mcpToken = isRecord(tokenResponse.body) && typeof tokenResponse.body.token === "string"
|
||||
? tokenResponse.body.token
|
||||
: ""
|
||||
expect(mcpToken).toMatch(/^ow_mcp_at_/)
|
||||
|
||||
const stamp = Date.now()
|
||||
const scriptName = `Launch briefing ${stamp}`
|
||||
const firstMarker = `launch-now-${stamp}`
|
||||
const scheduledMarker = `launch-scheduled-${stamp}`
|
||||
const code = [
|
||||
"const result = await tools.den.getWorkers({})",
|
||||
"return { briefing: { topic: input.topic, workerCount: result.workers.length } }",
|
||||
].join("\n")
|
||||
const inputSchema = {
|
||||
type: "object",
|
||||
properties: { topic: { type: "string" } },
|
||||
required: ["topic"],
|
||||
additionalProperties: false,
|
||||
}
|
||||
const outputSchema = {
|
||||
type: "object",
|
||||
properties: { briefing: {} },
|
||||
required: ["briefing"],
|
||||
additionalProperties: false,
|
||||
}
|
||||
|
||||
const executed = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", {
|
||||
name: "execute_capability_script",
|
||||
arguments: { code, input: { topic: firstMarker } },
|
||||
})
|
||||
expect(executed.isError).not.toBe(true)
|
||||
expect(JSON.stringify(executed.content)).toContain(firstMarker)
|
||||
|
||||
const savedResponse = await denFetch(den.admin, "/v1/workflows", {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
body: JSON.stringify({
|
||||
name: scriptName,
|
||||
description: "Builds a reusable launch briefing from the organization's worker roster.",
|
||||
code,
|
||||
currentInput: { topic: firstMarker },
|
||||
inputSchema,
|
||||
outputSchema,
|
||||
}),
|
||||
})
|
||||
expect(savedResponse.response.status, savedResponse.text).toBe(201)
|
||||
const saved = requireRecord(savedResponse.body, "saved Workflow")
|
||||
const pluginId = typeof saved.pluginId === "string" ? saved.pluginId : ""
|
||||
const configObjectId = typeof saved.configObjectId === "string" ? saved.configObjectId : ""
|
||||
const configObjectVersionId = typeof saved.configObjectVersionId === "string" ? saved.configObjectVersionId : ""
|
||||
expect(pluginId).not.toBe("")
|
||||
expect(configObjectId).not.toBe("")
|
||||
expect(configObjectVersionId).not.toBe("")
|
||||
evidence.recordAssertionEvidence(
|
||||
"A successful ad-hoc Code Mode result is promotable without retyping its procedure",
|
||||
"The exact successful code was saved as an immutable Workflow version using its recent receipt.",
|
||||
true,
|
||||
)
|
||||
|
||||
const manualRun = await denFetch(den.admin, `/v1/workflows/${configObjectId}/run`, {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
body: JSON.stringify({ pluginId, configObjectVersionId, input: { topic: firstMarker } }),
|
||||
})
|
||||
expect(manualRun.response.ok, manualRun.text).toBe(true)
|
||||
const manualResult = requireRecord(manualRun.body, "manual Script result")
|
||||
expect(manualResult.status).toBe("succeeded")
|
||||
expect(JSON.stringify(manualResult.value)).toContain(firstMarker)
|
||||
expect(String(manualResult.receiptId ?? "")).not.toBe("")
|
||||
evidence.recordAssertionEvidence(
|
||||
"The Workflow produces a validated artifact-ready result",
|
||||
"A direct run of the immutable version returned a schema-valid result and durable receipt.",
|
||||
true,
|
||||
)
|
||||
|
||||
const scheduledAfter = new Date().toISOString()
|
||||
const automationResponse = await denFetch(den.admin, "/v1/cloud-automations", {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
body: JSON.stringify({
|
||||
name: `${scriptName} once`,
|
||||
schedule: { kind: "once", timezone: "UTC", at: Date.now() + 30_000 },
|
||||
action: {
|
||||
kind: "saved_script",
|
||||
script: { pluginId, configObjectId, configObjectVersionId },
|
||||
input: { topic: scheduledMarker },
|
||||
},
|
||||
}),
|
||||
})
|
||||
expect(automationResponse.response.status, automationResponse.text).toBe(201)
|
||||
const automationDetail = requireRecord(automationResponse.body, "Automation")
|
||||
const automation = requireRecord(automationDetail.automation, "Automation identity")
|
||||
const automationId = typeof automation.id === "string" ? automation.id : ""
|
||||
expect(automationId).not.toBe("")
|
||||
|
||||
const scheduledRun = await eventually(async () => {
|
||||
const response = await denFetch(den.admin, `/v1/automations/${automationId}/runs`, {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(response.response.ok, response.text).toBe(true)
|
||||
return isRecord(response.body)
|
||||
? records(response.body.items).find((run) => run.trigger === "scheduled")
|
||||
: undefined
|
||||
}, (run) => run?.status === "succeeded", "scheduled Workflow Automation to succeed", 5 * 60_000)
|
||||
const scheduledRunId = typeof scheduledRun?.id === "string" ? scheduledRun.id : ""
|
||||
expect(scheduledRunId).not.toBe("")
|
||||
|
||||
const scheduledExternalCalls = await den.mocks.reports.toolCalls({
|
||||
name: "mock_echo",
|
||||
sinceIso: scheduledAfter,
|
||||
})
|
||||
expect(scheduledExternalCalls).toHaveLength(0)
|
||||
|
||||
const scheduledReceiptResponse = await denFetch(den.admin, `/v1/automation-runs/${scheduledRunId}`, {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(scheduledReceiptResponse.response.ok, scheduledReceiptResponse.text).toBe(true)
|
||||
const scheduledReceipt = requireRecord(scheduledReceiptResponse.body, "scheduled Automation receipt")
|
||||
const scheduledReceiptRun = requireRecord(scheduledReceipt.run, "scheduled Automation run")
|
||||
const scheduledReceiptAutomation = requireRecord(scheduledReceipt.automation, "scheduled Automation identity")
|
||||
const scheduledReceiptRevision = requireRecord(scheduledReceipt.revision, "scheduled Automation revision")
|
||||
const scheduledExecutionThread = requireRecord(scheduledReceiptRun.executionThread, "scheduled Automation execution thread")
|
||||
expect(JSON.stringify(scheduledReceipt)).toContain(scheduledMarker)
|
||||
expect(scheduledReceiptAutomation.id).toBe(automationId)
|
||||
expect(scheduledReceiptRevision.id).toBe(scheduledRun?.revisionId)
|
||||
expect(Array.isArray(scheduledReceipt.events)).toBe(true)
|
||||
expect(scheduledReceipt.events).toEqual([])
|
||||
expect(String(scheduledExecutionThread.id ?? "")).not.toBe("")
|
||||
expect(scheduledExecutionThread).toMatchObject({
|
||||
threadKind: "automation",
|
||||
executionLocation: "cloud",
|
||||
automationId,
|
||||
automationRunId: scheduledRunId,
|
||||
engineKind: "openwork-cloud-codemode-v1",
|
||||
})
|
||||
|
||||
const toolList = await agentRpc(den.ref.apiUrl, mcpToken, "tools/list", {})
|
||||
const tools = records(toolList.tools)
|
||||
const renderTool = tools.find((candidate) => candidate.name === "render_workflow_artifact")
|
||||
const renderToolMeta = isRecord(renderTool?._meta) ? renderTool._meta : {}
|
||||
const modernUi = isRecord(renderToolMeta.ui) ? renderToolMeta.ui : {}
|
||||
expect(modernUi.resourceUri).toBe("ui://openwork/workflow-artifact/v1/view.html")
|
||||
expect(renderToolMeta["ui/resourceUri"]).toBe("ui://openwork/workflow-artifact/v1/view.html")
|
||||
|
||||
const resourceList = await agentRpc(den.ref.apiUrl, mcpToken, "resources/list", {})
|
||||
const resources = records(resourceList.resources)
|
||||
const appResource = resources.find((candidate) => candidate.uri === "ui://openwork/workflow-artifact/v1/view.html")
|
||||
expect(appResource?.mimeType).toBe("text/html;profile=mcp-app")
|
||||
|
||||
const resourceRead = await agentRpc(den.ref.apiUrl, mcpToken, "resources/read", {
|
||||
uri: "ui://openwork/workflow-artifact/v1/view.html",
|
||||
})
|
||||
const resourceContents = records(resourceRead.contents)
|
||||
expect(resourceContents[0]?.mimeType).toBe("text/html;profile=mcp-app")
|
||||
expect(String(resourceContents[0]?.text ?? "")).toContain("ui/initialize")
|
||||
expect(String(resourceContents[0]?.text ?? "")).not.toContain("fetch(")
|
||||
|
||||
const rendered = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", {
|
||||
name: "render_workflow_artifact",
|
||||
arguments: { configObjectId },
|
||||
})
|
||||
expect(rendered.isError).not.toBe(true)
|
||||
const structured = requireRecord(rendered.structuredContent, "Workflow Artifact structuredContent")
|
||||
const artifact = requireRecord(structured.artifact, "Workflow Artifact lineage")
|
||||
const fallback = records(rendered.content)
|
||||
expect(structured.schemaVersion).toBe("1")
|
||||
expect(artifact.configObjectId).toBe(configObjectId)
|
||||
expect(artifact.source).toBe("scheduled")
|
||||
expect(String(artifact.receiptId ?? "")).not.toBe("")
|
||||
expect(JSON.stringify(structured.data)).toContain(scheduledMarker)
|
||||
expect(String(fallback[0]?.text ?? "")).toContain(scheduledMarker)
|
||||
evidence.recordAssertionEvidence(
|
||||
"The latest Automation snapshot is portable as a standards-based MCP App",
|
||||
"The agent endpoint returns the scheduled result as versioned structuredContent and a Markdown fallback linked to a self-contained ui:// resource.",
|
||||
true,
|
||||
)
|
||||
|
||||
const externalMarker = `launch-external-${stamp}`
|
||||
const externalCode = "return { briefing: await tools.report_source.mock_echo({ text: input.topic }) }"
|
||||
const externalRunStartedAt = new Date().toISOString()
|
||||
const externalExecuted = await agentRpc(den.ref.apiUrl, mcpToken, "tools/call", {
|
||||
name: "execute_capability_script",
|
||||
arguments: { code: externalCode, input: { topic: externalMarker } },
|
||||
})
|
||||
expect(externalExecuted.isError).not.toBe(true)
|
||||
expect(JSON.stringify(externalExecuted.content)).toContain(externalMarker)
|
||||
const interactiveExternalCalls = await den.mocks.reports.toolCalls({
|
||||
name: "mock_echo",
|
||||
atLeast: 1,
|
||||
sinceIso: externalRunStartedAt,
|
||||
timeoutMs: 60_000,
|
||||
})
|
||||
expect(interactiveExternalCalls.filter((call) => call.args.text === externalMarker)).toHaveLength(1)
|
||||
|
||||
const externalSavedResponse = await denFetch(den.admin, "/v1/workflows", {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
body: JSON.stringify({
|
||||
name: `${scriptName} external`,
|
||||
description: "Checks the unattended Cloud boundary for external MCP tools.",
|
||||
code: externalCode,
|
||||
currentInput: { topic: externalMarker },
|
||||
inputSchema,
|
||||
outputSchema,
|
||||
}),
|
||||
})
|
||||
expect(externalSavedResponse.response.status, externalSavedResponse.text).toBe(201)
|
||||
const externalSaved = requireRecord(externalSavedResponse.body, "external saved Workflow")
|
||||
const externalPluginId = typeof externalSaved.pluginId === "string" ? externalSaved.pluginId : ""
|
||||
const externalConfigObjectId = typeof externalSaved.configObjectId === "string" ? externalSaved.configObjectId : ""
|
||||
const externalConfigObjectVersionId = typeof externalSaved.configObjectVersionId === "string" ? externalSaved.configObjectVersionId : ""
|
||||
expect(externalPluginId).not.toBe("")
|
||||
expect(externalConfigObjectId).not.toBe("")
|
||||
expect(externalConfigObjectVersionId).not.toBe("")
|
||||
|
||||
const externalAutomation = await denFetch(den.admin, `/v1/automations/${automationId}`, {
|
||||
method: "PATCH",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
body: JSON.stringify({
|
||||
action: {
|
||||
kind: "saved_script",
|
||||
script: {
|
||||
pluginId: externalPluginId,
|
||||
configObjectId: externalConfigObjectId,
|
||||
configObjectVersionId: externalConfigObjectVersionId,
|
||||
},
|
||||
input: { topic: externalMarker },
|
||||
},
|
||||
}),
|
||||
})
|
||||
expect(externalAutomation.response.ok, externalAutomation.text).toBe(true)
|
||||
|
||||
const unattendedRunStartedAt = new Date().toISOString()
|
||||
const failedRunResponse = await denFetch(den.admin, `/v1/automations/${automationId}/run`, {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(failedRunResponse.response.status, failedRunResponse.text).toBe(202)
|
||||
const queued = isRecord(failedRunResponse.body) ? requireRecord(failedRunResponse.body.run, "queued Automation run") : {}
|
||||
const failedRunId = typeof queued.id === "string" ? queued.id : ""
|
||||
expect(failedRunId).not.toBe("")
|
||||
|
||||
const failedReceipt = await eventually(async () => {
|
||||
const response = await denFetch(den.admin, `/v1/automation-runs/${failedRunId}`, {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(response.response.ok, response.text).toBe(true)
|
||||
return requireRecord(response.body, "failed Automation receipt")
|
||||
}, (receipt) => isRecord(receipt.run)
|
||||
&& ["failed", "skipped", "cancelled"].includes(String(receipt.run.status)), "external-capability run to finish")
|
||||
const failedReceiptRun = requireRecord(failedReceipt.run, "failed Automation run")
|
||||
expect(failedReceiptRun.status).toBe("failed")
|
||||
const failedRunError = requireRecord(failedReceiptRun.error, "failed Automation error")
|
||||
expect(String(failedRunError.message ?? "")).toContain("must be read-only and explicitly approved")
|
||||
|
||||
const afterBoundaryRejection = await eventually(async () => {
|
||||
const response = await denFetch(den.admin, `/v1/automations/${automationId}`, {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
expect(response.response.ok, response.text).toBe(true)
|
||||
return requireRecord(response.body, "Automation after unattended boundary rejection")
|
||||
}, (detail) => isRecord(detail.automation) && detail.automation.state === "needs_attention", "Automation to need attention")
|
||||
expect(JSON.stringify(afterBoundaryRejection)).toContain(scheduledMarker)
|
||||
|
||||
let unattendedExternalCalls = 0
|
||||
try {
|
||||
unattendedExternalCalls = (await den.mocks.reports.toolCalls({
|
||||
name: "mock_echo",
|
||||
atLeast: 1,
|
||||
sinceIso: unattendedRunStartedAt,
|
||||
timeoutMs: 5_000,
|
||||
})).length
|
||||
} catch {
|
||||
unattendedExternalCalls = 0
|
||||
}
|
||||
expect(unattendedExternalCalls).toBe(0)
|
||||
evidence.recordAssertionEvidence(
|
||||
"Unattended Cloud rejects external MCP capability access before provider I/O and preserves the last good result",
|
||||
`Provider calls from the unattended run: ${unattendedExternalCalls}; the previous ${scheduledMarker} result remains durable.`,
|
||||
unattendedExternalCalls === 0 && JSON.stringify(afterBoundaryRejection).includes(scheduledMarker),
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user