import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { expect, onTestFinished } from "vitest"; import { clickButton, control, createAndSelectWorkspace, evalIn, waitFor } from "@openwork/behaviors"; import { screenshot, validate } from "@openwork/test-evidence"; import { desktop } from "@openwork/hosts"; import { needs, test } from "@openwork/testkit"; const providerId = "clamp-html-errors-mock"; const modelId = "clamp-html-errors-model"; const mcpToolName = "explode_html"; const closingReply = "The session recovered after the failed upstream call."; const htmlSummary = "Upstream returned an HTML error page (502 Bad Gateway)"; const htmlError = `502 Bad Gateway

502 Bad Gateway

${"Z".repeat(1_024 * 1_024)}`; const e2eTestsEnabled = process.env.OPENWORK_EVAL_E2E_TESTS === "1"; const title = e2eTestsEnabled ? "large HTML tool errors are summarized once without breaking the session" : "large HTML tool errors skipped — needs: set OPENWORK_EVAL_E2E_TESTS=1"; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function recordValue(value: unknown, key: string): unknown { return isRecord(value) ? value[key] : undefined; } function readBody(request: IncomingMessage): Promise { request.setEncoding("utf8"); return new Promise((resolve, reject) => { let body = ""; request.on("data", (chunk: string) => { body += chunk; }); request.on("end", () => resolve(body)); request.on("error", reject); }); } async function withTimeout(task: Promise, timeoutMs: number, label: string): Promise { let timer: ReturnType | undefined; try { return await Promise.race([ task, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(`Timed out waiting for ${label}`)), timeoutMs); }), ]); } finally { if (timer) clearTimeout(timer); } } function sendJson(response: ServerResponse, status: number, body: unknown): void { response.writeHead(status, { "access-control-allow-origin": "*", "content-type": "application/json", "cache-control": "no-store", }); response.end(JSON.stringify(body)); } function rpcResponse(message: Record): Record { const method = typeof message.method === "string" ? message.method : ""; if (method === "initialize") { return { jsonrpc: "2.0", id: message.id, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "html-error-mcp", version: "1.0.0" }, }, }; } if (method === "tools/list") { return { jsonrpc: "2.0", id: message.id, result: { tools: [{ name: mcpToolName, title: "HTML upstream failure", description: "Returns a deterministic upstream HTML error page.", inputSchema: { type: "object", properties: {}, additionalProperties: false }, }], }, }; } if (method === "tools/call") { return { jsonrpc: "2.0", id: message.id, error: { code: -32_000, message: htmlError }, }; } return { jsonrpc: "2.0", id: message.id, result: {} }; } function rpcDelayMs(message: Record): number { // Let the tool execution span a few seconds like a real upstream call, so // its error part lands while the renderer's event stream is connected // instead of inside the post-reload gap. return message.method === "tools/call" ? 4_000 : 0; } function providerToolName(payload: Record): string | null { const tools = payload.tools; if (!Array.isArray(tools)) return null; for (const tool of tools) { const fn = recordValue(tool, "function"); const name = recordValue(fn, "name"); if (typeof name === "string" && name.endsWith(mcpToolName)) return name; } return null; } function hasToolResult(payload: Record): boolean { const messages = payload.messages; return Array.isArray(messages) && messages.some((message) => recordValue(message, "role") === "tool"); } function streamChunk(delta: Record, finishReason: string | null = null): Record { return { id: "chatcmpl-clamp-html-errors", object: "chat.completion.chunk", created: 1, model: modelId, choices: [{ index: 0, delta, finish_reason: finishReason }], }; } function sendStream(response: ServerResponse, chunks: Record[]): void { response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive", }); // Pace the stream like a real model. An instantly-finished turn can complete // before the renderer's engine event stream (re)connects, leaving the UI // permanently stale on this turn. let delay = 400; for (const chunk of chunks) { setTimeout(() => response.write(`data: ${JSON.stringify(chunk)}\n\n`), delay); delay += 400; } setTimeout(() => response.end("data: [DONE]\n\n"), delay); } test.skipIf(!e2eTestsEnabled)(title, async ({ evidence }) => { needs({ optIn: ["OPENWORK_EVAL_E2E_TESTS"] }); let toolsListed = 0; let toolCalls = 0; let closingRounds = 0; const mock = createServer((request, response) => { void (async () => { const url = new URL(request.url ?? "/", "http://127.0.0.1"); if (request.method === "GET" && url.pathname === "/v1/models") { sendJson(response, 200, { object: "list", data: [{ id: modelId, object: "model" }] }); return; } if (url.pathname === "/mcp") { if (request.method === "GET") { sendJson(response, 405, { error: "method_not_allowed" }); return; } const raw = await readBody(request); const parsed: unknown = raw.trim() ? JSON.parse(raw) : {}; const messages = Array.isArray(parsed) ? parsed : [parsed]; const replies: Record[] = []; let delayMs = 0; for (const candidate of messages) { if (!isRecord(candidate)) continue; if (candidate.method === "tools/list") toolsListed += 1; if (candidate.method === "tools/call") toolCalls += 1; delayMs = Math.max(delayMs, rpcDelayMs(candidate)); if (candidate.id !== undefined) replies.push(rpcResponse(candidate)); } if (replies.length === 0) { response.writeHead(202, { "access-control-allow-origin": "*" }); response.end(); return; } if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs)); sendJson(response, 200, Array.isArray(parsed) ? replies : replies[0]); return; } if (request.method === "POST" && (url.pathname === "/v1/chat/completions" || url.pathname === "/chat/completions")) { const raw = await readBody(request); const parsed: unknown = JSON.parse(raw); if (!isRecord(parsed)) throw new Error("Mock provider received a non-object request."); // The engine's session-title agent hits the same provider; keep it away // from the tool-call script so counters stay deterministic. Utility // requests carry no tools array; real agent turns always do. const requestTools = parsed.tools; if (!Array.isArray(requestTools) || requestTools.length === 0) { sendStream(response, [ streamChunk({ role: "assistant" }), streamChunk({ content: "Session title" }), streamChunk({}, "stop"), ]); return; } if (hasToolResult(parsed)) { closingRounds += 1; sendStream(response, [ streamChunk({ role: "assistant" }), streamChunk({ content: closingReply }), streamChunk({}, "stop"), ]); return; } const toolName = providerToolName(parsed); if (!toolName) { sendStream(response, [ streamChunk({ role: "assistant" }), streamChunk({ content: "The deterministic MCP error tool was unavailable." }), streamChunk({}, "stop"), ]); return; } sendStream(response, [ streamChunk({ role: "assistant" }), streamChunk({ tool_calls: [{ index: 0, id: "call_html_error", type: "function", function: { name: toolName, arguments: "{}" }, }], }), streamChunk({}, "tool_calls"), ]); return; } sendJson(response, 404, { error: { message: "not found" } }); })().catch((error: unknown) => { if (!response.headersSent) sendJson(response, 500, { error: String(error) }); else response.destroy(error instanceof Error ? error : undefined); }); }); await withTimeout(new Promise((resolve, reject) => { mock.once("error", reject); mock.listen(0, "127.0.0.1", resolve); }), 10_000, "HTML error mock to listen"); onTestFinished(async () => { await withTimeout( new Promise((resolve, reject) => mock.close((error) => error ? reject(error) : resolve())), 10_000, "HTML error mock to close", ); }); const address = mock.address(); if (!address || typeof address === "string") throw new Error("HTML error mock did not bind a TCP port."); const baseUrl = `http://127.0.0.1:${address.port}`; await using app = await desktop({ name: "clamp-html-errors", mode: process.env.OPENWORK_EVAL_CDP_URL?.trim() ? "attach" : "spawn", // Provider keys in the runner env (e.g. via infisical) would make the // engine register real providers and out-default the deterministic mock. env: { ANTHROPIC_API_KEY: "", OPENAI_API_KEY: "", OPENROUTER_API_KEY: "", GOOGLE_GENERATIVE_AI_API_KEY: "", OPENWORK_API_KEY: "", OPENWORK_INFERENCE_BASE_URL: "", }, }); const workspace = await createAndSelectWorkspace(app, { path: `/tmp/openwork-clamp-html-errors-${Date.now()}`, }); const configured = await evalIn(app, `(async () => { const port = localStorage.getItem("openwork.server.port"); const token = localStorage.getItem("openwork.server.token"); if (!port || !token) return "missing local server credentials"; const request = async (path, init) => { const response = await fetch("http://127.0.0.1:" + port + path, { ...init, headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" }, }); if (!response.ok) return path + " failed: " + response.status + " " + (await response.text()).slice(0, 500); return "ok"; }; const workspaceId = ${JSON.stringify(workspace.workspaceId)}; const patched = await request("/workspace/" + encodeURIComponent(workspaceId) + "/config", { method: "PATCH", body: JSON.stringify({ opencode: { provider: { [${JSON.stringify(providerId)}]: { npm: "@ai-sdk/openai-compatible", name: "Clamp HTML errors mock", options: { baseURL: ${JSON.stringify(`${baseUrl}/v1`)}, apiKey: "sk-clamp-html-errors" }, models: { [${JSON.stringify(modelId)}]: { name: "Clamp HTML errors model", tool_call: true }, }, }, }, mcp: { "html-error": { type: "remote", url: ${JSON.stringify(`${baseUrl}/mcp`)}, enabled: true, oauth: false }, }, }, }), }); if (patched !== "ok") return patched; const reloaded = await request("/workspace/" + encodeURIComponent(workspaceId) + "/engine/reload", { method: "POST" }); // A slow dispose reports 504 opencode_reload_timeout while the reload // keeps going; readiness is owned by the polling below. if (reloaded !== "ok" && !reloaded.includes("opencode_reload_timeout")) return reloaded; const raw = localStorage.getItem("openwork.preferences"); let preferences = {}; try { preferences = raw ? JSON.parse(raw) : {}; } catch { preferences = {}; } if (!preferences || typeof preferences !== "object" || Array.isArray(preferences)) preferences = {}; localStorage.setItem("openwork.preferences", JSON.stringify({ ...preferences, defaultModel: { providerID: ${JSON.stringify(providerId)}, modelID: ${JSON.stringify(modelId)} }, modelVariant: null, providerStepCompleted: true, })); localStorage.setItem("openwork.defaultModel", ${JSON.stringify(`${providerId}/${modelId}`)}); localStorage.removeItem("openwork.sessionModels." + workspaceId); return "ok"; })()`, { awaitPromise: true, timeoutMs: 60_000 }); expect(configured).toBe("ok"); await expect.poll(() => toolsListed, { timeout: 30_000, interval: 100 }).toBeGreaterThan(0); // Preferences hydrate at boot, so reload unconditionally: without this the // engine's built-in free model stays the default and out-competes the mock. await evalIn(app, "location.reload(); true"); await waitFor(app, "Boolean(window.__openworkControl)", { timeoutMs: 30_000, label: "app reloaded with the HTML error mock preferences", }); // The engine restarts after /engine/reload; sending into that window races // the swap and strands the run behind an "OpenCode unavailable" banner. const engineReady = await evalIn(app, `(async () => { const port = localStorage.getItem("openwork.server.port"); const token = localStorage.getItem("openwork.server.token"); if (!port || !token) return "missing local server credentials"; const deadline = Date.now() + 60_000; let last = ""; while (Date.now() < deadline) { try { const response = await fetch("http://127.0.0.1:" + port + "/workspace/" + encodeURIComponent(${JSON.stringify(workspace.workspaceId)}) + "/opencode/session", { headers: { Authorization: "Bearer " + token }, }); if (response.ok) return "ready"; last = "HTTP " + response.status; } catch (error) { last = String(error); } await new Promise((resolve) => setTimeout(resolve, 500)); } return "engine not ready: " + last; })()`, { awaitPromise: true, timeoutMs: 70_000 }); expect(engineReady).toBe("ready"); await waitFor(app, `window.__openworkControl.listActions().some((action) => action.id === "session.create_task" && !action.disabled)`, { timeoutMs: 30_000, label: "new task action enabled", }); await control(app, "session.create_task"); await waitFor(app, `Boolean(document.querySelector('[contenteditable="true"][data-lexical-editor="true"]'))`, { timeoutMs: 30_000, label: "composer editor ready", }); const prompt = "Call the HTML upstream failure tool once, then explain that the session recovered."; const focused = await evalIn(app, `(() => { const editor = document.querySelector('[contenteditable="true"][data-lexical-editor="true"]'); if (!(editor instanceof HTMLElement)) return false; editor.focus(); return true; })()`); expect(focused).toBe(true); await app.client.send("Input.insertText", { text: prompt }); await waitFor(app, `document.querySelector('[contenteditable="true"][data-lexical-editor="true"]')?.innerText === ${JSON.stringify(prompt)}`, { timeoutMs: 10_000, label: "task entered in composer", }); // A click can land while the composer is still wiring up; retry until the // send actually navigates into a session. let sent = false; for (let attempt = 0; attempt < 3 && !sent; attempt += 1) { await clickButton(app, "Run task", { timeoutMs: 30_000 }); try { await waitFor(app, `window.location.hash.includes("/session/ses_")`, { timeoutMs: 10_000, label: "task session created", }); sent = true; } catch (error) { if (attempt === 2) throw error; } } await waitFor(app, `(() => { const transcript = [...document.querySelectorAll('[data-message-role]')] .map((message) => message.textContent ?? "").join(" | "); return transcript.includes(${JSON.stringify(closingReply)}); })()`, { timeoutMs: 120_000, label: "closing assistant reply" }); await waitFor(app, `(() => { const failure = [...document.querySelectorAll('[data-capability-call]')] .find((row) => (row.textContent ?? "").includes("failed")); return Boolean(failure); })()`, { timeoutMs: 60_000, label: "failed tool row rendered" }); expect(toolCalls).toBe(1); expect(closingRounds).toBe(1); const expandedFailure = await evalIn(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); const trigger = row ? [...row.querySelectorAll("button")] .find((button) => (button.getAttribute("aria-label") ?? "").includes("Show what to do next")) : null; if (!(trigger instanceof HTMLElement)) return false; trigger.click(); return true; })()`); expect(expandedFailure).toBe(true); await waitFor(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); return (row?.innerText ?? "").includes(${JSON.stringify(htmlSummary)}); })()`, { timeoutMs: 10_000, label: "human HTML error summary visible" }); const transcriptClaim = await evalIn(app, `(() => { const rows = [...document.querySelectorAll('[data-capability-call]')]; const failureVisible = rows.some((row) => (row.innerText ?? "").includes("failed")); const transcript = [...document.querySelectorAll('[data-message-role]')] .map((message) => message.innerText ?? "").join(" | "); const lower = transcript.toLowerCase(); return failureVisible && transcript.includes(${JSON.stringify(htmlSummary)}) && !lower.includes(" { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); const toggle = row ? [...row.querySelectorAll("button")] .find((button) => (button.textContent ?? "").trim().startsWith("Technical details")) : null; if (!(toggle instanceof HTMLElement)) return false; toggle.click(); return true; })()`); expect(expandedTechnicalDetails).toBe(true); await waitFor(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); return [...(row?.querySelectorAll("pre") ?? [])] .some((preview) => (preview.textContent ?? "").includes("[Error text truncated; original size:")); })()`, { timeoutMs: 10_000, label: "bounded technical error preview visible" }); const detailsClaim = await evalIn(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); const preview = [...(row?.querySelectorAll("pre") ?? [])] .map((node) => node.textContent ?? "") .find((text) => text.includes("[Error text truncated; original size:")) ?? ""; return preview.length > 0 && preview.length < 6000 && /\\[Error text truncated; original size: [\\d,]+ characters\\]/.test(preview) && !preview.toLowerCase().includes(" { const transcript = [...document.querySelectorAll('[data-message-role]')] .map((message) => message.innerText ?? "").join(" | "); const page = document.body.innerText.toLowerCase(); return transcript.includes(${JSON.stringify(closingReply)}) && !page.includes("tool step unavailable") && !page.includes("failed to render"); })()`); expect(usableClaim).toBe(true); evidence.recordAssertionEvidence( "The session remains usable after the oversized tool failure", "The second-round closing assistant reply rendered and no tool error-boundary fallback or failed-to-render message appeared.", usableClaim === true, ); // Streaming rerenders can re-collapse the row after the DOM assertions, so // put the expanded failure state back on screen before the visual take. const reExpanded = await evalIn(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); if (!(row instanceof HTMLElement)) return "missing row"; const failureToggle = [...row.querySelectorAll("button")] .find((button) => (button.getAttribute("aria-label") ?? "").includes("failure details") || (button.getAttribute("aria-label") ?? "").includes("Show what to do next")); if (failureToggle instanceof HTMLElement && failureToggle.getAttribute("aria-expanded") !== "true") failureToggle.click(); return "ok"; })()`); expect(reExpanded).toBe("ok"); await evalIn(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); const details = row ? [...row.querySelectorAll("button")] .find((button) => (button.textContent ?? "").trim().startsWith("Technical details")) : null; if (details instanceof HTMLElement && details.getAttribute("aria-expanded") !== "true") details.click(); row?.scrollIntoView({ block: "center" }); return true; })()`); await waitFor(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); return [...(row?.querySelectorAll("pre") ?? [])] .some((preview) => (preview.textContent ?? "").includes("[Error text truncated; original size:")); })()`, { timeoutMs: 10_000, label: "expanded failure state back on screen for the visual take" }); await evalIn(app, `(() => { const row = [...document.querySelectorAll('[data-capability-call]')] .find((candidate) => (candidate.textContent ?? "").includes("failed")); const marker = [...(row?.querySelectorAll("pre") ?? [])] .find((preview) => (preview.textContent ?? "").includes("[Error text truncated; original size:")); marker?.scrollIntoView({ block: "center" }); return true; })()`); await new Promise((resolve) => setTimeout(resolve, 750)); const shot = await screenshot(app); const seen = await validate(shot, [ "A failed tool-call row is expanded with a human-readable 502 HTML error-page summary", "Technical details are expanded and show a truncation marker rather than a full raw HTML page", `The assistant closing message says: ${closingReply}`, "No crash, failed-to-render fallback, or wall of raw HTML markup is visible", ]); expect(seen.ok, seen.why).toBe(true); });