import { expect, onTestFinished } from "vitest"; import { createNativeConnector, createOrgConnection, deleteConnection, denFetch, freshSession, evalIn, readComposerState, readSessionToolCalls, selectModel, waitFor, waitForAssistantReply, writeComposerText, } from "@openwork/behaviors"; import type { DenSession } from "@openwork/behaviors"; import { navigate } from "@openwork/cdp"; import { screenshot, validate } from "@openwork/test-evidence"; import { chrome } from "@openwork/hosts"; import { app, mcpMock, needs, server, test, unmetNeeds } from "@openwork/testkit"; import type { TestNeeds } from "@openwork/testkit"; const requirements: TestNeeds = { model: "tool-capable", optIn: ["OPENWORK_EVAL_E2E_TESTS"], }; const missingRequirements = unmetNeeds(requirements, process.env); const title = missingRequirements.length > 0 ? `Workflows skipped — needs: ${missingRequirements.join(", ")}` : "Workflows: a 40-step question becomes one step, gets saved, and a teammate reuses it"; const modelId = process.env.OPENWORK_EVAL_MODEL?.trim() || ""; let requestId = 0; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function requireRecord(value: unknown, label: string): Record { if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`); return value; } function toolText(result: unknown): string { const record = requireRecord(result, "MCP tool result"); const first = Array.isArray(record.content) ? record.content[0] : null; if (!isRecord(first) || typeof first.text !== "string") { throw new Error(`MCP tool result had no text content: ${JSON.stringify(result).slice(0, 500)}`); } return first.text; } async function agentRpc(apiUrl: string, mcpToken: string, method: string, params: Record): Promise { const response = await fetch(`${apiUrl}/mcp/agent`, { method: "POST", headers: { authorization: `Bearer ${mcpToken}`, "content-type": "application/json", accept: "application/json, text/event-stream", }, body: JSON.stringify({ jsonrpc: "2.0", id: ++requestId, method, params }), signal: AbortSignal.timeout(180_000), }); const raw = await response.text(); if (!response.ok) throw new Error(`MCP ${method} failed: HTTP ${response.status} ${raw.slice(0, 500)}`); const dataLine = raw.split("\n").find((line) => line.startsWith("data:")); if (!dataLine) throw new Error(`MCP ${method} returned no SSE data frame: ${raw.slice(0, 500)}`); const payload: unknown = JSON.parse(dataLine.slice(5)); const record = requireRecord(payload, "MCP JSON-RPC payload"); if (record.error) throw new Error(`MCP ${method} returned JSON-RPC error: ${JSON.stringify(record.error)}`); return record.result; } async function listAgentToolNames(apiUrl: string, mcpToken: string): Promise { const result = requireRecord(await agentRpc(apiUrl, mcpToken, "tools/list", {}), "tools/list result"); const tools = Array.isArray(result.tools) ? result.tools : []; return tools.filter(isRecord).map((tool) => String(tool.name)); } async function callAgentTool( apiUrl: string, mcpToken: string, name: string, args: Record, ): Promise { return agentRpc(apiUrl, mcpToken, "tools/call", { name, arguments: args }); } async function organizationIdOf(session: DenSession): Promise { const result = await denFetch(session, "/v1/me/orgs", { headers: { authorization: `Bearer ${session.token}` }, }); const orgs = isRecord(result.body) && Array.isArray(result.body.orgs) ? result.body.orgs.filter(isRecord) : []; const id = orgs[0] && typeof orgs[0].id === "string" ? orgs[0].id : ""; if (!result.response.ok || !id) { throw new Error(`Finding the active organization failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`); } return id; } async function mintMcpToken(session: DenSession, orgId: string): Promise { const result = await denFetch(session, "/v1/mcp/token", { method: "POST", headers: { authorization: `Bearer ${session.token}`, "x-openwork-org-id": orgId, }, body: JSON.stringify({ scopes: ["mcp:read", "mcp:write"] }), }); const token = isRecord(result.body) && typeof result.body.token === "string" ? result.body.token : ""; if (!result.response.ok || !token.startsWith("ow_mcp_at_")) { throw new Error(`Minting MCP token failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`); } return token; } async function dumpScriptReceipts(session: DenSession): Promise { try { const receipts = await denFetch(session, "/v1/workflow-runs", { headers: { authorization: `Bearer ${session.token}` }, }); const runs = isRecord(receipts.body) && Array.isArray(receipts.body.runs) ? receipts.body.runs.filter(isRecord) : []; return JSON.stringify(runs.map((run) => ({ source: run.source, status: run.status, errorKind: run.errorKind, errorMessage: typeof run.errorMessage === "string" ? run.errorMessage.slice(0, 300) : run.errorMessage, toolCallCount: run.toolCallCount, durationMs: run.durationMs, }))); } catch (error) { return `receipt fetch failed: ${String(error)}`; } } async function validateWithRetry( shot: Awaited>, expectations: string[], ): Promise>> { let lastError: unknown; for (let attempt = 1; attempt <= 3; attempt += 1) { try { return await validate(shot, expectations); } catch (error) { lastError = error; await new Promise((resolve) => setTimeout(resolve, attempt * 10_000)); } } throw lastError; } async function sendPrompt( desktopApp: Awaited>, prompt: string, doneWhen?: (assistantText: string) => boolean, ): Promise { await writeComposerText(desktopApp, prompt); const before = await readComposerState(desktopApp); await waitFor(desktopApp, `(() => { const button = [...document.querySelectorAll('button')] .find((entry) => (entry.textContent ?? '').trim() === 'Run task' && !entry.disabled); button?.click(); return Boolean(button); })()`, { timeoutMs: 30_000, label: "Run task clicked" }); await waitFor( desktopApp, `document.querySelectorAll('[data-message-role="user"]').length > ${before.userMessageCount}`, { timeoutMs: 60_000, label: "prompt submitted" }, ); await waitForAssistantReply(desktopApp, { timeoutMs: 420_000 }); // The turn is over when the Stop control disappears — or, as a stall guard, // when the reply already contains what the caller is waiting for. const turnDeadline = Date.now() + 420_000; while (Date.now() < turnDeadline) { const stopVisible = await evalIn(desktopApp, `[...document.querySelectorAll('button')] .some((button) => (button.textContent ?? '').trim() === 'Stop')`); if (stopVisible !== true) break; if (doneWhen && doneWhen(await lastAssistantText(desktopApp))) return; await new Promise((resolve) => setTimeout(resolve, 2_000)); } if (Date.now() >= turnDeadline) { throw new Error("Timed out waiting for the assistant turn to complete."); } const deadline = Date.now() + 60_000; let stableFor = 0; let last = ""; while (Date.now() < deadline && stableFor < 2) { const current = await lastAssistantText(desktopApp); const settled = current === last && current.trim().length > 0 && !current.includes("Thinking"); stableFor = settled ? stableFor + 1 : 0; last = current; await new Promise((resolve) => setTimeout(resolve, 2_000)); } } async function countScriptToolCalls(desktopApp: Awaited>): Promise<{ script: number; single: number }> { const calls = await readSessionToolCalls(desktopApp, { timeoutMs: 30_000 }); const script = calls.filter((call) => call.capability.includes("execute_capability_script")).length; const single = calls.filter((call) => call.capability.includes("execute_capability") && !call.capability.includes("execute_capability_script")).length; return { script, single }; } async function lastAssistantText(desktopApp: Awaited>): Promise { const text = await evalIn(desktopApp, `(() => { const messages = [...document.querySelectorAll('[data-message-role="assistant"]')]; return messages.length > 0 ? messages[messages.length - 1].innerText : document.body.innerText; })()`); return String(text ?? ""); } test(title, { timeout: 1_500_000 }, async ({ evidence, place }) => { needs(requirements); await using den = await server({ place, org: { name: `Codemode Scripts Eval ${Date.now()}`, admin: { name: "Sarah" }, members: { jordan: { name: "Jordan Eval" } }, }, mocks: { drive: mcpMock({ allowUnauthenticatedMcp: true }), gmail: mcpMock({ allowUnauthenticatedMcp: true }), }, }); const orgId = await organizationIdOf(den.admin); const adminMcpToken = await mintMcpToken(den.admin, orgId); // ---- Frame 1: Code Mode is on for every organization by default ------------ const toolsAfter = await listAgentToolNames(den.ref.apiUrl, adminMcpToken); evidence.recordAssertionEvidence( "execute_capability_script is registered for a brand-new organization without any rollout flag", `tools/list for a fresh org: ${JSON.stringify(toolsAfter)}`, toolsAfter.includes("execute_capability") && toolsAfter.includes("execute_capability_script"), ); expect(toolsAfter).toContain("execute_capability"); expect(toolsAfter).toContain("execute_capability_script"); // ---- Seed: three workers with no activity + two mock org connections ------ for (const name of ["builder-1", "builder-2", "builder-3"]) { const created = await denFetch(den.admin, "/v1/workers", { method: "POST", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ name, destination: "local", workspacePath: "/tmp/openwork-eval-worker" }), }); if (!created.response.ok) { throw new Error(`Seeding worker ${name} failed: HTTP ${created.response.status} ${created.text.slice(0, 500)}`); } } const driveConnection = await createOrgConnection(den.admin, { name: "Drive Mock", url: den.mocks.drive.mcpUrl, authType: "none", credentialMode: "shared", access: { orgWide: true }, }); await createOrgConnection(den.admin, { name: "Gmail Mock", url: den.mocks.gmail.mcpUrl, authType: "none", credentialMode: "shared", access: { orgWide: true }, }); // ---- Rail probes: the script tool works deterministically before any model const namespacesProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: "return Object.keys(tools).sort()", }); const namespacesText = toolText(namespacesProbe); evidence.recordAssertionEvidence( "The script runtime exposes the Den catalog and both org connections as namespaces", `Object.keys(tools): ${namespacesText.slice(0, 300)}`, namespacesText.includes("den") && namespacesText.includes("drive_mock") && namespacesText.includes("gmail_mock"), ); expect(namespacesText).toContain("drive_mock"); expect(namespacesText).toContain("gmail_mock"); const searchProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `return await tools.$codemode.search({ query: "list workers" })`, }); const searchProbeText = toolText(searchProbe); const probeItems = (() => { const parsed: unknown = JSON.parse(searchProbeText); const record = isRecord(parsed) ? parsed : {}; return Array.isArray(record.items) ? record.items.filter(isRecord) : []; })(); const workersPath = probeItems .map((item) => String(item.path ?? "")) .find((path) => path.startsWith("tools.den.") && path.toLowerCase().includes("workers")); evidence.recordAssertionEvidence( "In-program tool search finds the Den workers operation", `search("list workers") paths: ${JSON.stringify(probeItems.map((item) => item.path)).slice(0, 300)}`, Boolean(workersPath), ); if (!workersPath) throw new Error(`No workers operation found via $codemode.search: ${searchProbeText.slice(0, 500)}`); const workersProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `return await ${workersPath}({})`, }); const workersProbeText = toolText(workersProbe); evidence.recordAssertionEvidence( "A script can call a Den catalog operation and see the seeded workers", `${workersPath}({}) → ${workersProbeText.slice(0, 300)}`, workersProbeText.includes("builder-1") && workersProbeText.includes("builder-3"), ); expect(workersProbeText).toContain("builder-1"); const denNamesProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `return Object.keys(tools.den).sort()`, }); const denNamesText = toolText(denNamesProbe); const denNamesValue: unknown = JSON.parse(denNamesText); const denNames = Array.isArray(denNamesValue) ? denNamesValue.filter((name): name is string => typeof name === "string") : []; const workersOperationName = workersPath.slice("tools.den.".length); const leakedNativeNames = denNames.filter((name) => { const normalized = name.toLowerCase(); return normalized.includes("googleworkspace") || normalized.includes("microsoft365"); }); evidence.recordAssertionEvidence( "Credential-bound native provider operations are not reachable through the shared Den namespace, while ordinary Den capabilities remain available", `tools.den names: ${JSON.stringify(denNames).slice(0, 500)}; native operation names: ${JSON.stringify(leakedNativeNames)}; expected workers operation: ${workersOperationName}`, leakedNativeNames.length === 0 && denNames.length > 0 && denNames.includes(workersOperationName), ); expect(leakedNativeNames).toEqual([]); expect(denNames.length).toBeGreaterThan(0); expect(denNames).toContain(workersOperationName); const unconnectedGoogle = await createNativeConnector(den.admin, { providerKey: "google-workspace", name: "Unconnected Google Rail Probe", clientId: "unconnected-google-rail-client", clientSecret: "unconnected-google-rail-secret", access: { orgWide: true }, }); let unconnectedGoogleDeleted = false; onTestFinished(async () => { if (!unconnectedGoogleDeleted) { try { await deleteConnection(den.admin, unconnectedGoogle.id); } catch { // A local test Den may already be disposed when Vitest runs cleanup. } } }); const unconnectedNamespace = "unconnected_google_rail_probe"; const unconnectedNamespacesProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `return Object.keys(tools).sort()`, }); const unconnectedNamespacesText = toolText(unconnectedNamespacesProbe); const unconnectedNamespacesValue: unknown = JSON.parse(unconnectedNamespacesText); const unconnectedNamespaces = Array.isArray(unconnectedNamespacesValue) ? unconnectedNamespacesValue.filter((name): name is string => typeof name === "string") : []; // Closed-world: asserting one guessed namespace is absent would pass even if the // sanitizer produced a different name, so pin the whole namespace set instead. // The content rails (skills, marketplace, admin) are present because the capability // set is one set for all three verbs — this admin is on the bootstrap allowlist, so // admin capabilities are in their set; a non-admin member's set has none. No // marketplace namespace yet: this org's only marketplace object is the Workflow // created later, and Workflows are deliberately excluded (no Workflow recursion). const expectedNamespaces = ["$codemode", "admin", "den", "drive_mock", "gmail_mock", "skills"]; evidence.recordAssertionEvidence( "An unconnected native provider does not surface a callable script namespace", `Object.keys(tools): ${JSON.stringify(unconnectedNamespaces)}; expected exactly: ${JSON.stringify(expectedNamespaces)}`, JSON.stringify(unconnectedNamespaces) === JSON.stringify(expectedNamespaces), ); expect(unconnectedNamespaces).toEqual(expectedNamespaces); expect(unconnectedNamespaces).not.toContain(unconnectedNamespace); // Interchangeability: whatever search finds, execute runs AND a script can call. // A built-in skill is the sharpest case — it used to be searchable and executable // but deliberately absent from the script tree. const skillSearch = await callAgentTool(den.ref.apiUrl, adminMcpToken, "search_capabilities", { query: "skill", type: "skills", limit: 5, }); const skillMatches = (() => { const payload = requireRecord(JSON.parse(toolText(skillSearch)), "skill search payload"); return Array.isArray(payload.matches) ? payload.matches.filter(isRecord) : []; })(); const skillWithPath = skillMatches.find((match) => typeof match.scriptPath === "string" && match.scriptPath.length > 0); evidence.recordAssertionEvidence( "A skill capability found by search carries a scriptPath, so the same set is reachable from a script", `skills matches: ${JSON.stringify(skillMatches.map((m) => ({ name: m.name, scriptPath: m.scriptPath }))).slice(0, 400)}`, Boolean(skillWithPath), ); expect(skillWithPath).toBeTruthy(); const skillScriptPath = String((skillWithPath ?? {}).scriptPath ?? ""); const skillCallProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `const result = await ${skillScriptPath}({})\nreturn typeof result === "string" ? result.slice(0, 120) : result`, }); const skillCallText = toolText(skillCallProbe); evidence.recordAssertionEvidence( "That same skill capability is callable from inside a script at its advertised scriptPath", `${skillScriptPath}({}) -> ${skillCallText.slice(0, 300)}`, skillCallText.trim().length > 0 && !skillCallText.includes("script_failed"), ); expect(skillCallText).not.toContain("script_failed"); const unconnectedSearchProbe = await callAgentTool(den.ref.apiUrl, adminMcpToken, "search_capabilities", { query: "Unconnected Google Rail Probe Gmail", limit: 20, }); const unconnectedSearchPayload = requireRecord(JSON.parse(toolText(unconnectedSearchProbe)), "unconnected Google search payload"); const unconnectedSearchMatches = Array.isArray(unconnectedSearchPayload.matches) ? unconnectedSearchPayload.matches.filter(isRecord) : []; const unconnectedConnectionMatches = unconnectedSearchMatches.filter((match) => String(match.name ?? "").includes(unconnectedGoogle.id) || (isRecord(match.connectionStatus) && match.connectionStatus.connectionName === unconnectedGoogle.name)); const unconnectedStatusOnly = unconnectedConnectionMatches.length > 0 && unconnectedConnectionMatches.every((match) => match.kind === "connection_status" && match.status === "needs_connection" && typeof match.scriptPath === "undefined"); evidence.recordAssertionEvidence( "Capability search reports the unconnected Google provider as needing connection, never as a callable capability", `Matching search rows: ${JSON.stringify(unconnectedConnectionMatches).slice(0, 500)}`, unconnectedStatusOnly, ); expect(unconnectedConnectionMatches.length).toBeGreaterThan(0); expect(unconnectedStatusOnly).toBe(true); await deleteConnection(den.admin, unconnectedGoogle.id); unconnectedGoogleDeleted = true; // ---- Frames 2+3: one script step instead of a call per worker ------------- await using desktopApp = await app({ den, as: "admin", place, ...(modelId ? { model: modelId } : {}) }); await waitFor(desktopApp, `location.hash.includes('/session')`, { timeoutMs: 60_000, label: "session route" }); if (modelId) { try { await selectModel(desktopApp, modelId); } catch { // Provider models can lag the first boot; give the engine one settle pass. await new Promise((resolve) => setTimeout(resolve, 20_000)); await selectModel(desktopApp, modelId); } } try { await sendPrompt( desktopApp, "Answer this question: which of my workers are idle? " + "First call search_capabilities with query \"list workers\" and note the scriptPath field on the matches. " + "Then call execute_capability_script exactly once with one script that calls that scriptPath, " + "reads the response's workers array, and returns exactly result.workers.filter((worker) => worker.lastActiveAt === null).map((worker) => worker.name). " + "Do not call execute_capability. Reply with just the idle worker names.", (text) => text.includes("builder-1") && text.includes("builder-2") && text.includes("builder-3"), ); } catch (error) { evidence.recordAssertionEvidence( "DEBUG: idle-workers prompt did not complete; Workflow run receipts follow", await dumpScriptReceipts(den.admin), false, ); throw error; } const idleReply = await lastAssistantText(desktopApp); const idleAnswerSeen = idleReply.includes("builder-1") && idleReply.includes("builder-2") && idleReply.includes("builder-3"); const partsAfterIdle = await countScriptToolCalls(desktopApp); evidence.recordAssertionEvidence( "The idle-workers question ran as a single script step, not one call per worker", `Session tool calls: ${JSON.stringify(partsAfterIdle)}; assistant reply: ${idleReply.slice(0, 300)}`, partsAfterIdle.script >= 1 && idleAnswerSeen, ); expect(idleAnswerSeen).toBe(true); expect(partsAfterIdle.script).toBeGreaterThanOrEqual(1); const idleShot = await screenshot(desktopApp); const idleSeen = await validateWithRetry(idleShot, [ "A chat conversation answers which workers are idle, naming builder workers", "No error banner or crash message is visible", ]); expect(idleSeen.ok, idleSeen.why).toBe(true); // ---- Frame 4: one script fans out across two org connections -------------- const markerDrive = `drive-${Date.now()}`; const markerGmail = `gmail-${Date.now()}`; const fanoutStartedAt = new Date().toISOString(); const fanoutResult = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability_script", { code: `const [drive, gmail] = await Promise.all([ tools.drive_mock.mock_echo({ text: ${JSON.stringify(markerDrive)} }), tools.gmail_mock.mock_echo({ text: ${JSON.stringify(markerGmail)} }), ]) return { drive, gmail }`, }); const fanoutText = toolText(fanoutResult); const driveCalls = await den.mocks.drive.toolCalls({ name: "mock_echo", atLeast: 1, timeoutMs: 60_000, sinceIso: fanoutStartedAt }); const gmailCalls = await den.mocks.gmail.toolCalls({ name: "mock_echo", atLeast: 1, timeoutMs: 60_000, sinceIso: fanoutStartedAt }); const driveHit = driveCalls.some((call) => String(call.args.text ?? "").includes(markerDrive)); const gmailHit = gmailCalls.some((call) => String(call.args.text ?? "").includes(markerGmail)); evidence.recordAssertionEvidence( "One script reached both org connections in parallel and returned only the combined result", `Result: ${fanoutText.slice(0, 200)}; Drive witness saw ${markerDrive}: ${String(driveHit)}; Gmail witness saw ${markerGmail}: ${String(gmailHit)}`, driveHit && gmailHit && fanoutText.includes(markerDrive) && fanoutText.includes(markerGmail), ); expect(driveHit).toBe(true); expect(gmailHit).toBe(true); expect(fanoutText).toContain(markerDrive); // ---- Frame 5: save it as an org capability through the capability rail ----- const pluginOpSearch = await callAgentTool(den.ref.apiUrl, adminMcpToken, "search_capabilities", { query: "create plugin", limit: 10, }); const pluginOpMatches = requireRecord(JSON.parse(toolText(pluginOpSearch)), "plugin op search"); const pluginOpList = Array.isArray(pluginOpMatches.matches) ? pluginOpMatches.matches.filter(isRecord) : []; const createPluginOp = pluginOpList.find((match) => match.method === "POST" && String(match.path ?? "").endsWith("/v1/plugins")); if (!createPluginOp) throw new Error(`No create-plugin capability found: ${JSON.stringify(pluginOpList.map((m) => `${m.method} ${m.path}`))}`); const idleScriptSource = [ `const result = await tools.den.getWorkers({})`, `const idle = result.workers.filter((worker) => !worker.lastActiveAt)`, `return idle.map((worker) => worker.name)`, ].join("\n"); const saveResult = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability", { name: String(createPluginOp.name), body: { name: "idle-workers-report", description: "Lists idle workers", orgWide: true, components: [{ type: "workflow", input: { rawSourceText: idleScriptSource, normalizedPayloadJson: { language: "codemode-js", inputSchema: { type: "object", properties: { days: { type: "number" } }, additionalProperties: false }, requiredCapabilities: [ { capabilityName: "getWorkers", scriptPath: "tools.den.getWorkers" }, ], }, metadata: { title: "idle-workers-report", description: "Lists idle workers" }, }, }], }, }); const saveText = toolText(saveResult); evidence.recordAssertionEvidence( "The idle-workers script is saved as an org capability through the capability rail", `execute_capability(${String(createPluginOp.name)}) → ${saveText.slice(0, 200)}`, saveText.includes("idle-workers-report"), ); const savedObjects = await denFetch(den.admin, "/v1/config-objects?type=workflow", { headers: { authorization: `Bearer ${den.admin.token}` }, }); const savedList = isRecord(savedObjects.body) && Array.isArray(savedObjects.body.items) ? savedObjects.body.items.filter(isRecord) : []; const savedWorkflow = savedList.find((object) => String(object.title ?? "").includes("idle-workers-report")); evidence.recordAssertionEvidence( "The agent saved the procedure as an org config object of type workflow", `config-objects?type=workflow returned ${savedList.length} row(s); idle-workers-report present: ${String(Boolean(savedWorkflow))}`, Boolean(savedWorkflow), ); expect(savedWorkflow).toBeTruthy(); // ---- Frame 6: a teammate finds it by search and runs it ------------------- const jordan = den.members.jordan; if (!jordan) throw new Error("Default org member jordan was not provisioned."); const jordanMcpToken = await mintMcpToken(jordan, orgId); const searchResult = await callAgentTool(den.ref.apiUrl, jordanMcpToken, "search_capabilities", { query: "idle workers report", limit: 5, }); const searchPayload = requireRecord(JSON.parse(toolText(searchResult)), "search payload"); const matches = Array.isArray(searchPayload.matches) ? searchPayload.matches.filter(isRecord) : []; const scriptMatch = matches.find((match) => String(match.name ?? "").startsWith("plugin:") && match.kind === "workflow"); evidence.recordAssertionEvidence( "A teammate's agent discovers the Workflow through capability search", `Member search matches: ${JSON.stringify(matches.map((match) => ({ name: match.name, kind: match.kind })))}`, Boolean(scriptMatch), ); expect(scriptMatch).toBeTruthy(); const scriptCapabilityName = String((scriptMatch ?? {}).name ?? ""); const memberRun = await callAgentTool(den.ref.apiUrl, jordanMcpToken, "execute_capability", { name: scriptCapabilityName, body: { days: 14 }, }); const memberRunPayload = requireRecord(JSON.parse(toolText(memberRun)), "member run payload"); const memberRunExecuted = memberRunPayload.status === "executed"; const memberRunValue = JSON.stringify(memberRunPayload.value ?? null); evidence.recordAssertionEvidence( "The teammate runs the Workflow by name with typed input and gets fresh data from the same procedure", `status: ${String(memberRunPayload.status)}; value: ${memberRunValue.slice(0, 300)}`, memberRunExecuted && memberRunValue.includes("builder"), ); expect(memberRunExecuted).toBe(true); expect(memberRunValue).toContain("builder"); // ---- Frame 7a: every run left a receipt ------------------------------------ const receipts = await denFetch(den.admin, "/v1/workflow-runs", { headers: { authorization: `Bearer ${den.admin.token}` }, }); const runs = isRecord(receipts.body) && Array.isArray(receipts.body.runs) ? receipts.body.runs.filter(isRecord) : []; const adhocRuns = runs.filter((run) => run.source === "adhoc"); const savedRuns = runs.filter((run) => String(run.source ?? "").startsWith("plugin:")); const receiptsHaveDetail = runs.every((run) => typeof run.durationMs === "number" && Array.isArray(run.toolCalls)); evidence.recordAssertionEvidence( "Every Workflow run left a receipt with its tool calls and duration", `runs: ${runs.length} (adhoc: ${adhocRuns.length}, saved: ${savedRuns.length}); all carry toolCalls+durationMs: ${String(receiptsHaveDetail)}`, adhocRuns.length >= 2 && savedRuns.length >= 1 && receiptsHaveDetail, ); expect(adhocRuns.length).toBeGreaterThanOrEqual(2); expect(savedRuns.length).toBeGreaterThanOrEqual(1); await using browser = await chrome({ name: "codemode-script-runs", startUrl: den.ref.webUrl, headless: true, host: place.host(), }); await browser.client.send("Emulation.setDeviceMetricsOverride", { width: 1680, height: 1000, deviceScaleFactor: 1, mobile: false, }); await waitFor(browser, `location.href.startsWith(${JSON.stringify(den.ref.webUrl)}) && document.readyState === "complete"`, { timeoutMs: 60_000, label: "Den Web loaded", }); await evalIn(browser, `localStorage.setItem("openwork:web:auth-token", ${JSON.stringify(den.admin.token)})`); await navigate(browser.client, `${den.ref.webUrl}/dashboard/script-runs`); await waitFor(browser, `document.body.innerText.includes("Workflow Runs") || document.body.innerText.includes("workflow runs")`, { timeoutMs: 60_000, label: "Workflow Runs dashboard", }); // The banner renders before the table; screenshot only after the loading // placeholder is gone and real rows (status chip + duration header) exist. // Table headers render uppercase (DURATION), so match case-insensitively. await waitFor( browser, `!document.body.innerText.includes("Loading workflow runs") && /duration/i.test(document.body.innerText) && /succeeded|failed/i.test(document.body.innerText)`, { timeoutMs: 60_000, label: "Workflow Runs rows loaded" }, ); const runsShot = await screenshot(browser); const runsSeen = await validateWithRetry(runsShot, [ "A dashboard screen lists Workflow runs with status and duration information", ]); expect(runsSeen.ok, runsSeen.why).toBe(true); // ---- Frame 7b: disabling a tool makes dependent Workflows fail closed -- // Org-wide plugin creation is a step-up-protected admin write, and this spec runs // long enough for the admin session's fresh-auth window to lapse. Re-authenticate // first, the same way behaviors' connection helpers do on `reauth`. const freshAdmin = await freshSession(den.admin); const mentionsPlugin = await denFetch(freshAdmin, "/v1/plugins", { method: "POST", headers: { authorization: `Bearer ${freshAdmin.token}` }, body: JSON.stringify({ name: "offsite-mentions-report", description: "Echoes offsite mentions through the Drive Mock connection", orgWide: true, components: [{ type: "workflow", input: { rawSourceText: `return await tools.drive_mock.mock_echo({ text: "offsite" })`, normalizedPayloadJson: { language: "codemode-js", requiredCapabilities: [{ capabilityName: `mcp:${driveConnection.id}:mock_echo`, scriptPath: "tools.drive_mock.mock_echo", }], }, metadata: { title: "offsite-mentions-report", description: "Echoes offsite mentions" }, }, }], }), }); if (!mentionsPlugin.response.ok) { throw new Error(`Creating offsite-mentions-report failed: HTTP ${mentionsPlugin.response.status} ${mentionsPlugin.text.slice(0, 500)}`); } const mentionsSearch = await callAgentTool(den.ref.apiUrl, adminMcpToken, "search_capabilities", { query: "offsite mentions report", limit: 5, }); const mentionsMatches = requireRecord(JSON.parse(toolText(mentionsSearch)), "mentions search payload"); const mentionsMatchList = Array.isArray(mentionsMatches.matches) ? mentionsMatches.matches.filter(isRecord) : []; const mentionsMatch = mentionsMatchList.find((match) => String(match.name ?? "").startsWith("plugin:") && match.kind === "workflow"); if (!mentionsMatch) throw new Error(`offsite-mentions-report not discoverable: ${JSON.stringify(mentionsMatchList)}`); const mentionsName = String(mentionsMatch.name); const healthyRun = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability", { name: mentionsName, body: {}, }); const healthyPayload = requireRecord(JSON.parse(toolText(healthyRun)), "healthy mentions run"); expect(healthyPayload.status).toBe("executed"); const policyFlippedAt = new Date().toISOString(); const policy = await denFetch(den.admin, `/v1/mcp-connections/${driveConnection.id}/tool-policy`, { method: "PUT", headers: { authorization: `Bearer ${den.admin.token}` }, body: JSON.stringify({ allDisabled: false, disabledTools: ["mock_echo"] }), }); if (!policy.response.ok) { throw new Error(`Disabling mock_echo failed: HTTP ${policy.response.status} ${policy.text.slice(0, 500)}`); } const blockedRun = await callAgentTool(den.ref.apiUrl, adminMcpToken, "execute_capability", { name: mentionsName, body: {}, }); const blockedText = toolText(blockedRun); const blockedFailedClosed = blockedText.includes("capability_unavailable") && blockedText.includes("providerCallAttempted") === blockedText.includes(`"providerCallAttempted":false`); let blockedWitnessCalls = 0; try { const calls = await den.mocks.drive.toolCalls({ name: "mock_echo", atLeast: 1, timeoutMs: 5_000, sinceIso: policyFlippedAt }); blockedWitnessCalls = calls.length; } catch { blockedWitnessCalls = 0; } evidence.recordAssertionEvidence( "Disabling a connection tool makes the dependent Workflow fail closed without reaching the provider", `Blocked run payload: ${blockedText.slice(0, 400)}; provider calls after policy flip: ${blockedWitnessCalls}`, blockedText.includes("capability_unavailable") && blockedWitnessCalls === 0, ); expect(blockedText).toContain("capability_unavailable"); expect(blockedWitnessCalls).toBe(0); expect(blockedFailedClosed).toBe(true); });