源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
@@ -0,0 +1,428 @@
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http"
|
||||
import { expect, onTestFinished } from "vitest"
|
||||
import { clickButton, createAndSelectWorkspace, denFetch, evalIn, waitFor } from "@openwork/behaviors"
|
||||
import { connect, debuggerUrlFor, evaluate, listTargets } from "@openwork/cdp"
|
||||
import { desktop } from "@openwork/hosts"
|
||||
import { screenshot } from "@openwork/test-evidence"
|
||||
import { localMysqlIsRunning, needs, server, test } from "@openwork/testkit"
|
||||
|
||||
const providerId = "skill-created-mcp-app-provider"
|
||||
const modelId = "skill-created-mcp-app-model"
|
||||
const resourceUri = "ui://openwork/skill-created/v1/view.html"
|
||||
const closingReply = "The beautiful tomatoes skill is ready to use."
|
||||
const e2eTestsEnabled = process.env.OPENWORK_EVAL_E2E_TESTS === "1"
|
||||
const localPlacement = process.env.OPENWORK_EVAL_DAYTONA !== "1"
|
||||
&& !process.env.OPENWORK_EVAL_DEN_API_URL?.trim()
|
||||
const mysqlOpen = await localMysqlIsRunning()
|
||||
const title = !e2eTestsEnabled
|
||||
? "skill-created MCP App skipped — needs: set OPENWORK_EVAL_E2E_TESTS=1"
|
||||
: !localPlacement
|
||||
? "skill-created MCP App skipped — needs local placement without OPENWORK_EVAL_DEN_API_URL"
|
||||
: !mysqlOpen
|
||||
? "skill-created MCP App skipped — needs MySQL on 127.0.0.1:3306"
|
||||
: "creating a Cloud skill renders the first-party skill-created MCP App"
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function readBody(request: IncomingMessage): Promise<string> {
|
||||
request.setEncoding("utf8")
|
||||
return new Promise((resolve, reject) => {
|
||||
let body = ""
|
||||
request.on("data", (chunk: string) => {
|
||||
body += chunk
|
||||
})
|
||||
request.on("end", () => resolve(body))
|
||||
request.on("error", reject)
|
||||
})
|
||||
}
|
||||
|
||||
function sendJson(response: ServerResponse, status: number, body: unknown): void {
|
||||
response.writeHead(status, {
|
||||
"access-control-allow-origin": "*",
|
||||
"cache-control": "no-store",
|
||||
"content-type": "application/json",
|
||||
})
|
||||
response.end(JSON.stringify(body))
|
||||
}
|
||||
|
||||
function streamChunk(delta: Record<string, unknown>, finishReason: string | null = null) {
|
||||
return {
|
||||
id: "chatcmpl-skill-created-mcp-app",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: modelId,
|
||||
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
||||
}
|
||||
}
|
||||
|
||||
function sendStream(response: ServerResponse, chunks: Record<string, unknown>[]): void {
|
||||
response.writeHead(200, {
|
||||
"cache-control": "no-cache",
|
||||
connection: "keep-alive",
|
||||
"content-type": "text/event-stream",
|
||||
})
|
||||
let delay = 250
|
||||
for (const chunk of chunks) {
|
||||
setTimeout(() => response.write(`data: ${JSON.stringify(chunk)}\n\n`), delay)
|
||||
delay += 250
|
||||
}
|
||||
setTimeout(() => response.end("data: [DONE]\n\n"), delay)
|
||||
}
|
||||
|
||||
function projectedCreateSkillTool(payload: Record<string, unknown>): string | null {
|
||||
if (!Array.isArray(payload.tools)) return null
|
||||
for (const tool of payload.tools) {
|
||||
if (!isRecord(tool) || !isRecord(tool.function)) continue
|
||||
const name = tool.function.name
|
||||
if (typeof name === "string" && name.endsWith("_create_skill")) return name
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function completedToolCount(payload: Record<string, unknown>): number {
|
||||
return Array.isArray(payload.messages)
|
||||
? payload.messages.filter((message) => isRecord(message) && message.role === "tool").length
|
||||
: 0
|
||||
}
|
||||
|
||||
async function waitForMountedSkill(app: Awaited<ReturnType<typeof desktop>>, timeoutMs = 15_000): Promise<{ mounted: boolean; text: string }> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
let lastText = ""
|
||||
while (Date.now() < deadline) {
|
||||
const targets = await listTargets(app.handle.cdpUrl)
|
||||
const sandbox = targets.find((target) => target.type === "iframe"
|
||||
&& target.url.includes("/mcp-apps/sandbox.html")
|
||||
&& target.webSocketDebuggerUrl)
|
||||
if (sandbox) {
|
||||
const client = await connect(debuggerUrlFor(app.handle.cdpUrl, sandbox))
|
||||
try {
|
||||
const text = await evaluate(client, `(() => {
|
||||
const text = document.querySelector("iframe")?.contentDocument?.body?.innerText ?? "";
|
||||
return text;
|
||||
})()`)
|
||||
if (typeof text === "string") {
|
||||
lastText = text
|
||||
const normalized = text.toLocaleLowerCase()
|
||||
const mounted = normalized.includes("skill created")
|
||||
&& normalized.includes("beautiful-tomatoes")
|
||||
&& normalized.includes("ready")
|
||||
&& normalized.includes("use beautiful tomatoes whenever the user says go.")
|
||||
if (mounted) return { mounted: true, text }
|
||||
}
|
||||
} finally {
|
||||
client.close()
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return { mounted: false, text: lastText }
|
||||
}
|
||||
|
||||
test.skipIf(!e2eTestsEnabled || !localPlacement || !mysqlOpen)(title, { timeout: 360_000 }, async ({ evidence, place }) => {
|
||||
needs({ optIn: ["OPENWORK_EVAL_E2E_TESTS"] })
|
||||
|
||||
let modelCreateCalls = 0
|
||||
const fixture = createServer((request, response) => {
|
||||
void (async () => {
|
||||
const url = new URL(request.url ?? "/", "http://127.0.0.1")
|
||||
if (request.method === "GET" && url.pathname === "/v1/models") {
|
||||
sendJson(response, 200, { object: "list", data: [{ id: modelId, object: "model" }] })
|
||||
return
|
||||
}
|
||||
if (request.method === "POST" && (url.pathname === "/v1/chat/completions" || url.pathname === "/chat/completions")) {
|
||||
const parsed: unknown = JSON.parse(await readBody(request))
|
||||
if (!isRecord(parsed)) throw new Error("Mock provider received a non-object request.")
|
||||
if (completedToolCount(parsed) > 0) {
|
||||
sendStream(response, [
|
||||
streamChunk({ role: "assistant" }),
|
||||
streamChunk({ content: closingReply }),
|
||||
streamChunk({}, "stop"),
|
||||
])
|
||||
return
|
||||
}
|
||||
const toolName = projectedCreateSkillTool(parsed)
|
||||
if (!toolName) throw new Error("The create_skill MCP App tool was not projected to the model.")
|
||||
modelCreateCalls += 1
|
||||
sendStream(response, [
|
||||
streamChunk({ role: "assistant" }),
|
||||
streamChunk({
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
id: "call_create_beautiful_tomatoes",
|
||||
type: "function",
|
||||
function: {
|
||||
name: toolName,
|
||||
arguments: JSON.stringify({
|
||||
pluginName: "Beautiful Tomatoes",
|
||||
skillMarkdown: [
|
||||
"---",
|
||||
"name: beautiful-tomatoes",
|
||||
"description: Use beautiful tomatoes whenever the user says go.",
|
||||
"---",
|
||||
"",
|
||||
"Whenever the user says go, respond using beautiful tomatoes.",
|
||||
].join("\n"),
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}),
|
||||
streamChunk({}, "tool_calls"),
|
||||
])
|
||||
return
|
||||
}
|
||||
sendJson(response, 404, { error: { message: "not found" } })
|
||||
})().catch((error: unknown) => {
|
||||
if (!response.headersSent) sendJson(response, 500, { error: String(error) })
|
||||
else response.destroy(error instanceof Error ? error : undefined)
|
||||
})
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
fixture.once("error", reject)
|
||||
fixture.listen(0, "127.0.0.1", resolve)
|
||||
})
|
||||
onTestFinished(async () => {
|
||||
await new Promise<void>((resolve, reject) => fixture.close((error) => error ? reject(error) : resolve()))
|
||||
})
|
||||
const address = fixture.address()
|
||||
if (!address || typeof address === "string") throw new Error("Skill-created model fixture did not bind a port.")
|
||||
const fixtureUrl = `http://127.0.0.1:${address.port}`
|
||||
|
||||
await using den = await server({
|
||||
place,
|
||||
org: { name: `Skill Created App ${Date.now()}`, admin: { name: "Avery" } },
|
||||
})
|
||||
const orgsResult = await denFetch(den.admin, "/v1/me/orgs", {
|
||||
headers: { authorization: `Bearer ${den.admin.token}` },
|
||||
})
|
||||
const organizations = isRecord(orgsResult.body) && Array.isArray(orgsResult.body.orgs)
|
||||
? orgsResult.body.orgs.filter(isRecord)
|
||||
: []
|
||||
const organizationId = String(organizations[0]?.id ?? "")
|
||||
expect(organizationId).toMatch(/^org_/)
|
||||
|
||||
const tokenResult = await denFetch(den.admin, "/v1/mcp/token", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${den.admin.token}`,
|
||||
"x-openwork-org-id": organizationId,
|
||||
},
|
||||
body: JSON.stringify({ scopes: ["mcp:read", "mcp:write"] }),
|
||||
})
|
||||
expect(tokenResult.response.ok, tokenResult.text).toBe(true)
|
||||
const mcpToken = isRecord(tokenResult.body) && typeof tokenResult.body.token === "string"
|
||||
? tokenResult.body.token
|
||||
: ""
|
||||
expect(mcpToken).toMatch(/^ow_mcp_at_/)
|
||||
|
||||
await using app = await desktop({
|
||||
name: "skill-created-mcp-app",
|
||||
mode: process.env.OPENWORK_EVAL_CDP_URL?.trim() ? "attach" : "spawn",
|
||||
env: {
|
||||
ANTHROPIC_API_KEY: "",
|
||||
OPENAI_API_KEY: "",
|
||||
OPENROUTER_API_KEY: "",
|
||||
GOOGLE_GENERATIVE_AI_API_KEY: "",
|
||||
OPENWORK_API_KEY: "",
|
||||
OPENWORK_INFERENCE_BASE_URL: "",
|
||||
},
|
||||
})
|
||||
const workspace = await createAndSelectWorkspace(app, {
|
||||
path: `/tmp/openwork-skill-created-mcp-app-${Date.now()}`,
|
||||
})
|
||||
const configured = await evalIn(app, `(async () => {
|
||||
const port = localStorage.getItem("openwork.server.port");
|
||||
const token = localStorage.getItem("openwork.server.token");
|
||||
if (!port || !token) return "missing local server credentials";
|
||||
const request = async (path, init) => {
|
||||
const response = await fetch("http://127.0.0.1:" + port + path, {
|
||||
...init,
|
||||
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
|
||||
});
|
||||
if (!response.ok) return path + " failed: " + response.status + " " + (await response.text()).slice(0, 500);
|
||||
return "ok";
|
||||
};
|
||||
const workspaceId = ${JSON.stringify(workspace.workspaceId)};
|
||||
const patched = await request("/workspace/" + encodeURIComponent(workspaceId) + "/config", {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify({
|
||||
opencode: {
|
||||
provider: {
|
||||
[${JSON.stringify(providerId)}]: {
|
||||
npm: "@ai-sdk/openai-compatible",
|
||||
name: "Skill-created MCP App model",
|
||||
options: { baseURL: ${JSON.stringify(`${fixtureUrl}/v1`)}, apiKey: "sk-skill-created-mcp-app" },
|
||||
models: { [${JSON.stringify(modelId)}]: { name: "Skill-created MCP App model", tool_call: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (patched !== "ok") return patched;
|
||||
const reloaded = await request("/workspace/" + encodeURIComponent(workspaceId) + "/engine/reload", { method: "POST" });
|
||||
if (reloaded !== "ok" && !reloaded.includes("opencode_reload_timeout")) return reloaded;
|
||||
const reconcileResponse = await fetch("http://127.0.0.1:" + port + "/workspace/" + encodeURIComponent(workspaceId) + "/mcp/openwork-cloud/reconcile", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
config: {
|
||||
type: "remote",
|
||||
url: ${JSON.stringify(`${den.ref.apiUrl}/mcp/agent`)},
|
||||
enabled: true,
|
||||
headers: { Authorization: ${JSON.stringify(`Bearer ${mcpToken}`)} },
|
||||
oauth: false,
|
||||
},
|
||||
provider: ${JSON.stringify(providerId)},
|
||||
model: ${JSON.stringify(modelId)},
|
||||
trigger: "skill-created-mcp-app-e2e",
|
||||
}),
|
||||
});
|
||||
const reconcileText = await reconcileResponse.text();
|
||||
if (!reconcileResponse.ok) return "Cloud MCP reconcile failed: " + reconcileResponse.status + " " + reconcileText.slice(0, 1_000);
|
||||
const health = JSON.parse(reconcileText);
|
||||
if (health?.phase !== "ready") return "Cloud MCP reconcile was not ready: " + JSON.stringify(health).slice(0, 2_000);
|
||||
const raw = localStorage.getItem("openwork.preferences");
|
||||
let preferences = {};
|
||||
try { preferences = raw ? JSON.parse(raw) : {}; } catch { preferences = {}; }
|
||||
if (!preferences || typeof preferences !== "object" || Array.isArray(preferences)) preferences = {};
|
||||
localStorage.setItem("openwork.preferences", JSON.stringify({
|
||||
...preferences,
|
||||
defaultModel: { providerID: ${JSON.stringify(providerId)}, modelID: ${JSON.stringify(modelId)} },
|
||||
modelVariant: null,
|
||||
providerStepCompleted: true,
|
||||
}));
|
||||
localStorage.setItem("openwork.defaultModel", ${JSON.stringify(`${providerId}/${modelId}`)});
|
||||
localStorage.removeItem("openwork.sessionModels." + workspaceId);
|
||||
return "ok";
|
||||
})()`, { awaitPromise: true, timeoutMs: 90_000 })
|
||||
expect(configured).toBe("ok")
|
||||
|
||||
await evalIn(app, "location.reload(); true")
|
||||
await waitFor(app, "Boolean(window.__openworkControl)", { timeoutMs: 30_000, label: "desktop control after reload" })
|
||||
await waitFor(app, `window.__openworkControl.listActions().some((action) => action.id === "session.create_task" && !action.disabled)`, {
|
||||
timeoutMs: 60_000,
|
||||
label: "new task action ready",
|
||||
})
|
||||
const task = await evalIn(app, `(async () => {
|
||||
const deadline = Date.now() + 60_000;
|
||||
let last = null;
|
||||
while (Date.now() < deadline) {
|
||||
last = await window.__openworkControl.execute("session.create_task", null);
|
||||
if (last?.ok === true) return last;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
return { ...last, hash: location.hash, text: (document.body?.innerText ?? "").slice(0, 2_000) };
|
||||
})()`, {
|
||||
awaitPromise: true,
|
||||
timeoutMs: 70_000,
|
||||
})
|
||||
expect(task, JSON.stringify(task)).toMatchObject({ ok: true })
|
||||
await waitFor(app, `Boolean(document.querySelector('[contenteditable="true"][data-lexical-editor="true"]'))`, {
|
||||
timeoutMs: 30_000,
|
||||
label: "composer ready",
|
||||
})
|
||||
const focused = await evalIn(app, `(() => {
|
||||
const editor = document.querySelector('[contenteditable="true"][data-lexical-editor="true"]');
|
||||
if (!(editor instanceof HTMLElement)) return false;
|
||||
editor.focus();
|
||||
return true;
|
||||
})()`)
|
||||
expect(focused).toBe(true)
|
||||
await app.client.send("Input.insertText", {
|
||||
text: "Create a skill about using beautiful tomatoes each time I say go.",
|
||||
})
|
||||
await clickButton(app, "Run task", { timeoutMs: 30_000 })
|
||||
|
||||
await waitFor(app, `document.body.innerText.includes(${JSON.stringify(closingReply)})`, {
|
||||
timeoutMs: 120_000,
|
||||
label: "skill-created closing reply",
|
||||
})
|
||||
expect(modelCreateCalls).toBe(1)
|
||||
const persistedTool = await evalIn(app, `(async () => {
|
||||
const port = localStorage.getItem("openwork.server.port");
|
||||
const token = localStorage.getItem("openwork.server.token");
|
||||
if (!port || !token) return { error: "missing local server credentials" };
|
||||
const routeParts = location.hash.split("/");
|
||||
const sessionIndex = routeParts.indexOf("session");
|
||||
const sessionId = sessionIndex >= 0 ? decodeURIComponent(routeParts[sessionIndex + 1] || "") : "";
|
||||
const response = await fetch(
|
||||
"http://127.0.0.1:" + port + "/workspace/" + encodeURIComponent(${JSON.stringify(workspace.workspaceId)})
|
||||
+ "/opencode/session/" + encodeURIComponent(sessionId) + "/message?limit=50",
|
||||
{ headers: { Authorization: "Bearer " + token } },
|
||||
);
|
||||
const payload = await response.json();
|
||||
for (const message of Array.isArray(payload) ? payload : []) {
|
||||
for (const part of Array.isArray(message?.parts) ? message.parts : []) {
|
||||
if (typeof part?.tool === "string" && part.tool.endsWith("_create_skill")) {
|
||||
return { tool: part.tool, state: part.state };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { error: "create_skill part missing", payload };
|
||||
})()`, { awaitPromise: true, timeoutMs: 30_000 })
|
||||
expect(persistedTool, JSON.stringify(persistedTool)).toMatchObject({
|
||||
tool: "openwork-cloud_create_skill",
|
||||
state: { status: "completed" },
|
||||
})
|
||||
await waitFor(app, `Boolean(document.querySelector(${JSON.stringify(`[data-mcp-app-resource="${resourceUri}"] iframe`)}))`, {
|
||||
timeoutMs: 60_000,
|
||||
label: `skill-created MCP App frame after ${JSON.stringify(persistedTool)}`,
|
||||
})
|
||||
const mounted = await waitForMountedSkill(app)
|
||||
const transcript = String(await evalIn(app, "document.body?.innerText ?? ''"))
|
||||
expect(mounted.mounted, `${transcript}\nIframe: ${mounted.text}`).toBe(true)
|
||||
expect(transcript).not.toContain("🍅")
|
||||
expect(transcript).not.toContain("Interactive view unavailable")
|
||||
expect(transcript).not.toContain("MCP_APP_RESOURCE_NOT_FOUND")
|
||||
|
||||
// Session sync can briefly remount the message list (and its app frame)
|
||||
// right after the run completes; settle before capturing visual evidence.
|
||||
await waitFor(app, `!document.body.innerText.includes("Pulling in the latest messages")`, {
|
||||
timeoutMs: 60_000,
|
||||
label: "session sync settled before screenshot",
|
||||
})
|
||||
await waitFor(app, `Boolean(document.querySelector(${JSON.stringify(`[data-mcp-app-resource="${resourceUri}"] iframe`)}))`, {
|
||||
timeoutMs: 60_000,
|
||||
label: "skill-created frame after session sync",
|
||||
})
|
||||
const remounted = await waitForMountedSkill(app, 30_000)
|
||||
expect(remounted.mounted, remounted.text).toBe(true)
|
||||
await evalIn(app, `document.querySelector('[data-mcp-app-resource="${resourceUri}"]')?.scrollIntoView({ block: "center" })`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
// Ambient visual evidence of the rendered card; the iframe text assertions
|
||||
// above are the enforced proof of its contents.
|
||||
await screenshot(app)
|
||||
|
||||
const pluginsResult = await denFetch(den.admin, `/v1/plugins?q=${encodeURIComponent("Beautiful Tomatoes")}`, {
|
||||
headers: {
|
||||
authorization: `Bearer ${den.admin.token}`,
|
||||
"x-openwork-org-id": organizationId,
|
||||
},
|
||||
})
|
||||
expect(pluginsResult.response.ok, pluginsResult.text).toBe(true)
|
||||
const plugins = isRecord(pluginsResult.body) && Array.isArray(pluginsResult.body.items)
|
||||
? pluginsResult.body.items.filter(isRecord)
|
||||
: []
|
||||
expect(plugins).toHaveLength(1)
|
||||
const pluginId = String(plugins[0]?.id ?? "")
|
||||
const resolved = await denFetch(den.admin, `/v1/plugins/${encodeURIComponent(pluginId)}/resolved`, {
|
||||
headers: {
|
||||
authorization: `Bearer ${den.admin.token}`,
|
||||
"x-openwork-org-id": organizationId,
|
||||
},
|
||||
})
|
||||
expect(resolved.response.ok, resolved.text).toBe(true)
|
||||
expect(JSON.stringify(resolved.body)).toContain("beautiful-tomatoes")
|
||||
|
||||
evidence.recordAssertionEvidence(
|
||||
"A natural skill request executes the direct first-party tool",
|
||||
"The model received and called exactly one projected create_skill tool; Den persisted one Beautiful Tomatoes Plugin containing the beautiful-tomatoes skill.",
|
||||
modelCreateCalls === 1 && plugins.length === 1,
|
||||
)
|
||||
evidence.recordAssertionEvidence(
|
||||
"The completed skill call renders its standard MCP App",
|
||||
"Desktop mounted ui://openwork/skill-created/v1/view.html and the iframe showed Skill created, beautiful-tomatoes, Ready, and the skill description rather than only emoji Markdown.",
|
||||
mounted.mounted && !transcript.includes("🍅"),
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user