903 lines
40 KiB
TypeScript
903 lines
40 KiB
TypeScript
import { readFile } from "node:fs/promises";
|
|
import { expect } from "vitest";
|
|
import {
|
|
control,
|
|
evalIn,
|
|
go,
|
|
selectModel,
|
|
waitFor,
|
|
writeComposerText,
|
|
} from "@openwork/behaviors";
|
|
import { screenshot, validate } from "@openwork/test-evidence";
|
|
import {
|
|
app,
|
|
eventually,
|
|
localMysqlIsRunning,
|
|
localRedisIsRunning,
|
|
mcpMock,
|
|
needs,
|
|
readCloudMcpHealth,
|
|
readConnectState,
|
|
readDenClientState,
|
|
server,
|
|
sleep,
|
|
test,
|
|
} from "@openwork/testkit";
|
|
import type { App } from "@openwork/testkit";
|
|
|
|
const providerId = "active-session-storm-mock";
|
|
const modelId = "mock-agent-workload-model";
|
|
const modelName = "Active session storm model";
|
|
const workspaceCount = 3;
|
|
const e2eTestsEnabled = process.env.OPENWORK_EVAL_E2E_TESTS === "1";
|
|
const daytonaEnabled = process.env.OPENWORK_EVAL_DAYTONA === "1";
|
|
const configuredDen = Boolean(process.env.OPENWORK_EVAL_DEN_API_URL?.trim());
|
|
const localServicesRequired = !daytonaEnabled && !configuredDen;
|
|
const mysqlOpen = await localMysqlIsRunning();
|
|
const redisOpen = await localRedisIsRunning();
|
|
const runnable = e2eTestsEnabled && (!localServicesRequired || (mysqlOpen && redisOpen));
|
|
const skipSuffix = !e2eTestsEnabled
|
|
? " skipped — needs: set OPENWORK_EVAL_E2E_TESTS=1"
|
|
: localServicesRequired && !mysqlOpen
|
|
? " skipped — needs MySQL on 127.0.0.1:3306"
|
|
: localServicesRequired && !redisOpen
|
|
? " skipped — needs Redis on 127.0.0.1:6379"
|
|
: "";
|
|
|
|
function workloadMinutes(): number {
|
|
const value = Number(process.env.OPENWORK_EVAL_ACTIVE_SESSION_STORM_MINUTES ?? "2");
|
|
if (!Number.isFinite(value) || value < 1 || value > 5) {
|
|
throw new Error("OPENWORK_EVAL_ACTIVE_SESSION_STORM_MINUTES must be a number from 1 through 5.");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
const configuredMinutes = workloadMinutes();
|
|
const slowToolMs = Math.round(configuredMinutes * 60_000);
|
|
// At the default two-minute workload, prove a full minute in which all three
|
|
// slow tools overlap. A one-minute override remains useful for quick iteration
|
|
// and gets a 35-second switching window so startup skew cannot make it flaky.
|
|
const routeStormMs = Math.min(60_000, Math.max(35_000, slowToolMs - 25_000));
|
|
const diagnosticProfileDir = process.env.OPENWORK_EVAL_ACTIVE_SESSION_STORM_PROFILE_DIR?.trim();
|
|
|
|
interface WorkspacePlan {
|
|
index: number;
|
|
path: string;
|
|
filePath: string;
|
|
marker: string;
|
|
slowMarker: string;
|
|
easyMarker: string;
|
|
finalReply: string;
|
|
workspaceId: string;
|
|
sessionId: string;
|
|
}
|
|
|
|
interface WorkspaceListing {
|
|
ids: string[];
|
|
activeId: string | null;
|
|
}
|
|
|
|
interface ToolFact {
|
|
tool: string;
|
|
status: string;
|
|
input: Record<string, unknown>;
|
|
output: string;
|
|
}
|
|
|
|
interface SessionFacts {
|
|
ok: boolean;
|
|
status: number;
|
|
sessionId: string;
|
|
text: string;
|
|
tools: ToolFact[];
|
|
}
|
|
|
|
interface SurfaceFacts {
|
|
route: string;
|
|
sessionId: string;
|
|
authActions: string[];
|
|
crash: boolean;
|
|
bodyHasMarker: boolean;
|
|
bodyHasToolActivity: boolean;
|
|
}
|
|
|
|
interface EngineGenerationFact {
|
|
role: string;
|
|
pid: number | null;
|
|
port: number | null;
|
|
}
|
|
|
|
interface EngineRuntimeFacts {
|
|
lifecycleState: string;
|
|
enginePid: number | null;
|
|
engineRollover: boolean;
|
|
generations: EngineGenerationFact[];
|
|
}
|
|
|
|
interface WorkspaceFileFacts {
|
|
status: number;
|
|
content: string;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function stringArray(value: unknown): string[] {
|
|
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
|
}
|
|
|
|
function parseSessionFacts(value: unknown): SessionFacts {
|
|
if (!isRecord(value) || typeof value.ok !== "boolean" || typeof value.status !== "number") {
|
|
throw new Error(`Invalid session facts: ${JSON.stringify(value)}`);
|
|
}
|
|
const tools: ToolFact[] = [];
|
|
if (Array.isArray(value.tools)) {
|
|
for (const candidate of value.tools) {
|
|
if (!isRecord(candidate) || typeof candidate.tool !== "string" || typeof candidate.status !== "string") continue;
|
|
tools.push({
|
|
tool: candidate.tool,
|
|
status: candidate.status,
|
|
input: isRecord(candidate.input) ? candidate.input : {},
|
|
output: typeof candidate.output === "string" ? candidate.output : "",
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
ok: value.ok,
|
|
status: value.status,
|
|
sessionId: typeof value.sessionId === "string" ? value.sessionId : "",
|
|
text: typeof value.text === "string" ? value.text : "",
|
|
tools,
|
|
};
|
|
}
|
|
|
|
function parseSurfaceFacts(value: unknown): SurfaceFacts {
|
|
if (!isRecord(value)) throw new Error(`Invalid session surface facts: ${JSON.stringify(value)}`);
|
|
return {
|
|
route: typeof value.route === "string" ? value.route : "",
|
|
sessionId: typeof value.sessionId === "string" ? value.sessionId : "",
|
|
authActions: stringArray(value.authActions),
|
|
crash: value.crash === true,
|
|
bodyHasMarker: value.bodyHasMarker === true,
|
|
bodyHasToolActivity: value.bodyHasToolActivity === true,
|
|
};
|
|
}
|
|
|
|
function parseEngineRuntimeFacts(value: unknown): EngineRuntimeFacts {
|
|
const root = isRecord(value) ? value : {};
|
|
const engine = isRecord(root.engine) ? root.engine : {};
|
|
const openworkServer = isRecord(root.openworkServer) ? root.openworkServer : {};
|
|
const pool = isRecord(root.enginePool) ? root.enginePool : {};
|
|
const generations: EngineGenerationFact[] = [];
|
|
if (Array.isArray(pool.generations)) {
|
|
for (const generation of pool.generations) {
|
|
if (!isRecord(generation) || typeof generation.role !== "string") continue;
|
|
generations.push({
|
|
role: generation.role,
|
|
pid: typeof generation.pid === "number" ? generation.pid : null,
|
|
port: typeof generation.port === "number" ? generation.port : null,
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
lifecycleState: typeof root.lifecycleState === "string" ? root.lifecycleState : "",
|
|
enginePid: typeof engine.pid === "number" ? engine.pid : null,
|
|
engineRollover: openworkServer.engineRollover === true,
|
|
generations,
|
|
};
|
|
}
|
|
|
|
function shellValue(value: string): string {
|
|
if (!/^[A-Za-z0-9._/-]+$/.test(value)) throw new Error(`Unsafe workload shell value: ${value}`);
|
|
return value;
|
|
}
|
|
|
|
function buildPlans(runId: string): WorkspacePlan[] {
|
|
return Array.from({ length: workspaceCount }, (_, offset) => {
|
|
const index = offset + 1;
|
|
const path = `/tmp/openwork-active-session-storm-${runId}-w${index}`;
|
|
const marker = `STORM-W${index}-${runId}`;
|
|
return {
|
|
index,
|
|
path,
|
|
filePath: `${path}/storm-output-w${index}.txt`,
|
|
marker,
|
|
slowMarker: `SLOW-${marker}`,
|
|
easyMarker: `EASY-${marker}`,
|
|
finalReply: `COMPLETE-${marker}`,
|
|
workspaceId: "",
|
|
sessionId: "",
|
|
};
|
|
});
|
|
}
|
|
|
|
function agentWorkloads(plans: WorkspacePlan[]) {
|
|
return plans.map((plan) => ({
|
|
promptMarker: plan.marker,
|
|
finalReply: plan.finalReply,
|
|
steps: [
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `printf '%s\\n' 'INITIAL-${shellValue(plan.marker)}' > '${shellValue(plan.filePath)}'`,
|
|
timeout: 30_000,
|
|
workdir: plan.path,
|
|
description: `Create the unique output for workspace ${plan.index}`,
|
|
},
|
|
},
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `cat '${shellValue(plan.filePath)}'`,
|
|
timeout: 30_000,
|
|
workdir: plan.path,
|
|
description: `Check the initial output for workspace ${plan.index}`,
|
|
},
|
|
},
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `sleep ${Math.ceil(slowToolMs / 1_000)} && printf '%s\\n' '${shellValue(plan.slowMarker)}' >> '${shellValue(plan.filePath)}'`,
|
|
timeout: slowToolMs + 30_000,
|
|
workdir: plan.path,
|
|
description: `Hold workspace ${plan.index} live, then append its slow marker`,
|
|
},
|
|
},
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `cat '${shellValue(plan.filePath)}'`,
|
|
timeout: 30_000,
|
|
workdir: plan.path,
|
|
description: `Check the slow output for workspace ${plan.index}`,
|
|
},
|
|
},
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `printf '%s\\n' '${shellValue(plan.easyMarker)}' >> '${shellValue(plan.filePath)}'`,
|
|
timeout: 30_000,
|
|
workdir: plan.path,
|
|
description: `Append the easy marker for workspace ${plan.index}`,
|
|
},
|
|
},
|
|
{
|
|
tool: "bash",
|
|
arguments: {
|
|
command: `cat '${shellValue(plan.filePath)}'`,
|
|
timeout: 30_000,
|
|
workdir: plan.path,
|
|
description: `Read the completed output for workspace ${plan.index}`,
|
|
},
|
|
},
|
|
],
|
|
}));
|
|
}
|
|
|
|
async function listWorkspaces(desktopApp: App): Promise<WorkspaceListing> {
|
|
const value = await evalIn(desktopApp, `(async () => {
|
|
const info = await window.__OPENWORK_ELECTRON__?.invokeDesktop?.("openworkServerInfo");
|
|
if (!info?.running || !info.baseUrl) return { error: "local_server_unavailable" };
|
|
const response = await fetch(String(info.baseUrl).replace(/\\/+$/, "") + "/workspaces", {
|
|
headers: { Authorization: "Bearer " + String(info.ownerToken ?? info.clientToken ?? "") },
|
|
signal: AbortSignal.timeout(15000),
|
|
});
|
|
const body = await response.json();
|
|
const items = Array.isArray(body?.items) ? body.items : [];
|
|
return {
|
|
ok: response.ok,
|
|
ids: items.map((item) => String(item?.id ?? "")).filter(Boolean),
|
|
activeId: typeof body?.activeId === "string" ? body.activeId : null,
|
|
};
|
|
})()`, { awaitPromise: true, timeoutMs: 20_000 });
|
|
if (!isRecord(value) || value.ok !== true || !Array.isArray(value.ids)) {
|
|
throw new Error(`Listing workspaces failed: ${JSON.stringify(value)}`);
|
|
}
|
|
return {
|
|
ids: value.ids.filter((id): id is string => typeof id === "string"),
|
|
activeId: typeof value.activeId === "string" ? value.activeId : null,
|
|
};
|
|
}
|
|
|
|
async function createWorkspace(desktopApp: App, path: string): Promise<string> {
|
|
const before = await listWorkspaces(desktopApp);
|
|
await control(desktopApp, "workspace.create", { path }, { timeoutMs: 90_000 });
|
|
const after = await eventually(() => listWorkspaces(desktopApp), {
|
|
within: 90_000,
|
|
intervalMs: 500,
|
|
label: `workspace ${path} registered`,
|
|
until: (listing) => listing.ids.length === before.ids.length + 1,
|
|
});
|
|
const created = after.ids.find((id) => !before.ids.includes(id));
|
|
if (!created) throw new Error(`workspace.create produced no new id for ${path}.`);
|
|
return created;
|
|
}
|
|
|
|
async function configureWorkspaces(desktopApp: App, plans: WorkspacePlan[], baseUrl: string): Promise<void> {
|
|
const result = await evalIn(desktopApp, `(async () => {
|
|
const info = await window.__OPENWORK_ELECTRON__?.invokeDesktop?.("openworkServerInfo");
|
|
if (!info?.running || !info.baseUrl) return { error: "local_server_unavailable" };
|
|
const root = String(info.baseUrl).replace(/\\/+$/, "");
|
|
const headers = {
|
|
Authorization: "Bearer " + String(info.ownerToken ?? info.clientToken ?? ""),
|
|
"Content-Type": "application/json",
|
|
};
|
|
const outcomes = [];
|
|
for (const workspaceId of ${JSON.stringify(plans.map((plan) => plan.workspaceId))}) {
|
|
const config = await fetch(root + "/workspace/" + encodeURIComponent(workspaceId) + "/config", {
|
|
method: "PATCH",
|
|
headers,
|
|
body: JSON.stringify({
|
|
opencode: {
|
|
permission: { edit: "allow", write: "allow", read: "allow", bash: "allow" },
|
|
provider: {
|
|
[${JSON.stringify(providerId)}]: {
|
|
npm: "@ai-sdk/openai-compatible",
|
|
name: "Active session storm mock",
|
|
options: { baseURL: ${JSON.stringify(`${baseUrl}/v1`)}, apiKey: "sk-active-session-storm" },
|
|
models: {
|
|
[${JSON.stringify(modelId)}]: { name: ${JSON.stringify(modelName)}, tool_call: true },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
signal: AbortSignal.timeout(30000),
|
|
});
|
|
if (!config.ok) {
|
|
outcomes.push({ workspaceId, stage: "config", status: config.status, text: (await config.text()).slice(0, 300) });
|
|
continue;
|
|
}
|
|
const reload = await fetch(root + "/workspace/" + encodeURIComponent(workspaceId) + "/engine/reload", {
|
|
method: "POST",
|
|
headers,
|
|
signal: AbortSignal.timeout(60000),
|
|
});
|
|
outcomes.push({ workspaceId, stage: "reload", status: reload.status, text: reload.ok ? "ok" : (await reload.text()).slice(0, 300) });
|
|
}
|
|
const raw = localStorage.getItem("openwork.preferences");
|
|
let preferences = {};
|
|
try { preferences = raw ? JSON.parse(raw) : {}; } catch { preferences = {}; }
|
|
if (!preferences || typeof preferences !== "object" || Array.isArray(preferences)) preferences = {};
|
|
localStorage.setItem("openwork.preferences", JSON.stringify({
|
|
...preferences,
|
|
defaultModel: { providerID: ${JSON.stringify(providerId)}, modelID: ${JSON.stringify(modelId)} },
|
|
modelVariant: null,
|
|
providerStepCompleted: true,
|
|
}));
|
|
localStorage.setItem("openwork.defaultModel", ${JSON.stringify(`${providerId}/${modelId}`)});
|
|
return { outcomes };
|
|
})()`, { awaitPromise: true, timeoutMs: 240_000 });
|
|
if (!isRecord(result) || !Array.isArray(result.outcomes)) {
|
|
throw new Error(`Workspace provider configuration failed: ${JSON.stringify(result)}`);
|
|
}
|
|
const failures = result.outcomes.filter((outcome) => !isRecord(outcome) || outcome.status !== 200);
|
|
if (failures.length > 0) throw new Error(`Workspace provider configuration failures: ${JSON.stringify(failures)}`);
|
|
}
|
|
|
|
async function openExactSessionRoute(desktopApp: App, plan: WorkspacePlan): Promise<void> {
|
|
const route = `/workspace/${plan.workspaceId}/session/${plan.sessionId}`;
|
|
await go(desktopApp, route, { timeoutMs: 60_000 });
|
|
await waitFor(desktopApp, `(() => {
|
|
const current = window.__openworkControl?.snapshot().route.split("?")[0].replace(/\\/+$/, "") ?? "";
|
|
return current === ${JSON.stringify(route)}
|
|
&& (localStorage.getItem("openwork.react.activeWorkspace") ?? "") === ${JSON.stringify(plan.workspaceId)}
|
|
&& document.querySelector("[data-session-surface-id]")?.getAttribute("data-session-surface-id") === ${JSON.stringify(plan.sessionId)};
|
|
})()`, { timeoutMs: 60_000, label: `exact route ${route}` });
|
|
}
|
|
|
|
async function readSessionFacts(desktopApp: App, workspaceId: string, sessionId: string): Promise<SessionFacts> {
|
|
const value = await evalIn(desktopApp, `(async () => {
|
|
const info = await window.__OPENWORK_ELECTRON__?.invokeDesktop?.("openworkServerInfo");
|
|
if (!info?.running || !info.baseUrl) return { ok: false, status: 0, error: "local_server_unavailable" };
|
|
const base = String(info.baseUrl).replace(/\\/+$/, "") + "/workspace/" + encodeURIComponent(${JSON.stringify(workspaceId)})
|
|
+ "/opencode/session";
|
|
const encodedSessionId = encodeURIComponent(${JSON.stringify(sessionId)});
|
|
const options = {
|
|
headers: { Authorization: "Bearer " + String(info.ownerToken ?? info.clientToken ?? "") },
|
|
signal: AbortSignal.timeout(15000),
|
|
};
|
|
const responses = await Promise.all([
|
|
fetch(base + "/" + encodedSessionId, options),
|
|
fetch(base + "/" + encodedSessionId + "/message?limit=50", options),
|
|
fetch(base + "/" + encodedSessionId + "/todo", options),
|
|
fetch(base + "/status", options),
|
|
]);
|
|
const failed = responses.find((response) => !response.ok);
|
|
if (failed) return { ok: false, status: failed.status, sessionId: ${JSON.stringify(sessionId)}, text: "", tools: [] };
|
|
const [session, messageWires, todos, statuses] = await Promise.all(responses.map((response) => response.json()));
|
|
const item = { session, messages: messageWires, todos, status: statuses?.[session?.id] ?? { type: "idle" } };
|
|
const messages = Array.isArray(item?.messages) ? item.messages : [];
|
|
const text = messages.flatMap((message) => Array.isArray(message?.parts) ? message.parts : [])
|
|
.flatMap((part) => typeof part?.text === "string" ? [part.text] : []).join("\\n");
|
|
const tools = messages.flatMap((message) => Array.isArray(message?.parts) ? message.parts : [])
|
|
.flatMap((part) => {
|
|
if (!part || typeof part.tool !== "string") return [];
|
|
const state = part.state && typeof part.state === "object" ? part.state : {};
|
|
const output = typeof state.output === "string"
|
|
? state.output
|
|
: typeof state.metadata?.output === "string"
|
|
? state.metadata.output
|
|
: "";
|
|
return [{
|
|
tool: part.tool,
|
|
status: typeof state.status === "string" ? state.status : "",
|
|
input: state.input && typeof state.input === "object" && !Array.isArray(state.input) ? state.input : {},
|
|
output,
|
|
}];
|
|
});
|
|
return {
|
|
ok: true,
|
|
status: responses[0].status,
|
|
sessionId: typeof item?.session?.id === "string" ? item.session.id : "",
|
|
text,
|
|
tools,
|
|
};
|
|
})()`, { awaitPromise: true, timeoutMs: 20_000 });
|
|
return parseSessionFacts(value);
|
|
}
|
|
|
|
async function readSurfaceFacts(desktopApp: App, marker: string): Promise<SurfaceFacts> {
|
|
const value = await evalIn(desktopApp, `(() => {
|
|
const body = document.body.innerText ?? "";
|
|
const authActions = [...document.querySelectorAll("button, a")]
|
|
.map((element) => (element.textContent ?? "").trim())
|
|
.filter((text) => /^(sign in|reconnect|connect again|log in)$/i.test(text));
|
|
return {
|
|
route: window.__openworkControl?.snapshot().route ?? window.location.hash,
|
|
sessionId: document.querySelector("[data-session-surface-id]")?.getAttribute("data-session-surface-id") ?? "",
|
|
authActions,
|
|
crash: /aw, snap|renderer process gone|application error|uncaught exception/i.test(body),
|
|
bodyHasMarker: body.includes(${JSON.stringify(marker)}),
|
|
bodyHasToolActivity: body.includes("Hold workspace") || body.includes("sleep "),
|
|
};
|
|
})()`);
|
|
return parseSurfaceFacts(value);
|
|
}
|
|
|
|
async function readEngineRuntimeFacts(desktopApp: App): Promise<EngineRuntimeFacts> {
|
|
return parseEngineRuntimeFacts(await evalIn(
|
|
desktopApp,
|
|
`window.__OPENWORK_ELECTRON__.invokeDesktop("runtimeStatus")`,
|
|
{ awaitPromise: true, timeoutMs: 15_000 },
|
|
));
|
|
}
|
|
|
|
async function readWorkspaceFileFacts(desktopApp: App, plan: WorkspacePlan): Promise<WorkspaceFileFacts> {
|
|
const value = await evalIn(desktopApp, `(async () => {
|
|
const info = await window.__OPENWORK_ELECTRON__?.invokeDesktop?.("openworkServerInfo");
|
|
if (!info?.running || !info.baseUrl) return { status: 0, content: "" };
|
|
const response = await fetch(
|
|
String(info.baseUrl).replace(/\\/+$/, "") + "/workspace/" + encodeURIComponent(${JSON.stringify(plan.workspaceId)})
|
|
+ "/files/content?path=" + encodeURIComponent(${JSON.stringify(plan.filePath.split("/").pop() ?? "")}),
|
|
{
|
|
headers: { Authorization: "Bearer " + String(info.ownerToken ?? info.clientToken ?? "") },
|
|
signal: AbortSignal.timeout(10000),
|
|
},
|
|
);
|
|
const body = await response.json().catch(() => ({}));
|
|
return { status: response.status, content: typeof body?.content === "string" ? body.content : "" };
|
|
})()`, { awaitPromise: true, timeoutMs: 15_000 });
|
|
if (!isRecord(value) || typeof value.status !== "number") {
|
|
throw new Error(`Invalid workspace file facts: ${JSON.stringify(value)}`);
|
|
}
|
|
return {
|
|
status: value.status,
|
|
content: typeof value.content === "string" ? value.content : "",
|
|
};
|
|
}
|
|
|
|
async function readEngineLogSignals(desktopApp: App): Promise<string[]> {
|
|
if (desktopApp.handle.hostKind !== "local") return [];
|
|
const logPath = desktopApp.handle.meta?.log;
|
|
if (!logPath) return [];
|
|
const text = await readFile(logPath, "utf8").catch(() => "");
|
|
return text.split("\n")
|
|
.filter((line) => /engine|dispose|drain|reload|activate|session/i.test(line))
|
|
.slice(-80)
|
|
.map((line) => line.slice(0, 500));
|
|
}
|
|
|
|
function slowToolRunning(facts: SessionFacts): boolean {
|
|
return facts.tools.some((tool) => tool.tool.endsWith("bash")
|
|
&& (tool.status === "running" || tool.status === "pending")
|
|
&& typeof tool.input.command === "string"
|
|
&& tool.input.command.includes("sleep "));
|
|
}
|
|
|
|
function sessionCorpus(facts: SessionFacts): string {
|
|
return `${facts.text}\n${JSON.stringify(facts.tools)}`;
|
|
}
|
|
|
|
async function allowVisibleToolPermission(desktopApp: App): Promise<boolean> {
|
|
const clicked = await evalIn(desktopApp, `(() => {
|
|
const button = [...document.querySelectorAll("button")]
|
|
.find((candidate) => (candidate.textContent ?? "").trim() === "Allow for session" && !candidate.disabled);
|
|
if (!(button instanceof HTMLButtonElement)) return false;
|
|
button.click();
|
|
return true;
|
|
})()`);
|
|
if (clicked === true) await sleep(250);
|
|
return clicked === true;
|
|
}
|
|
|
|
async function approvePendingToolPermissions(desktopApp: App, plan: WorkspacePlan): Promise<number> {
|
|
const result = await evalIn(desktopApp, `(async () => {
|
|
const info = await window.__OPENWORK_ELECTRON__?.invokeDesktop?.("openworkServerInfo");
|
|
if (!info?.running || !info.baseUrl) return { error: "local_server_unavailable" };
|
|
const root = String(info.baseUrl).replace(/\\/+$/, "")
|
|
+ "/workspace/" + encodeURIComponent(${JSON.stringify(plan.workspaceId)}) + "/opencode";
|
|
const headers = {
|
|
Authorization: "Bearer " + String(info.ownerToken ?? info.clientToken ?? ""),
|
|
"Content-Type": "application/json",
|
|
};
|
|
const sessionId = ${JSON.stringify(plan.sessionId)};
|
|
const candidates = [];
|
|
const v2 = await fetch(root + "/api/session/" + encodeURIComponent(sessionId) + "/permission", {
|
|
headers,
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
if (v2.ok) {
|
|
const requests = await v2.json();
|
|
for (const request of Array.isArray(requests) ? requests : []) {
|
|
if (typeof request?.id === "string") candidates.push({ id: request.id, protocol: "v2" });
|
|
}
|
|
}
|
|
if (candidates.length === 0) {
|
|
const legacy = await fetch(root + "/permission", { headers, signal: AbortSignal.timeout(10000) });
|
|
if (legacy.ok) {
|
|
const requests = await legacy.json();
|
|
for (const request of Array.isArray(requests) ? requests : []) {
|
|
if (typeof request?.id === "string" && request?.sessionID === sessionId) {
|
|
candidates.push({ id: request.id, protocol: "legacy" });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const replies = [];
|
|
for (const candidate of candidates) {
|
|
const path = candidate.protocol === "v2"
|
|
? "/api/session/" + encodeURIComponent(sessionId) + "/permission/" + encodeURIComponent(candidate.id) + "/reply"
|
|
: "/permission/" + encodeURIComponent(candidate.id) + "/reply";
|
|
const response = await fetch(root + path, {
|
|
method: "POST",
|
|
headers,
|
|
body: JSON.stringify({ reply: "once" }),
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
replies.push({ id: candidate.id, protocol: candidate.protocol, status: response.status });
|
|
}
|
|
return { replies };
|
|
})()`, { awaitPromise: true, timeoutMs: 30_000 });
|
|
if (!isRecord(result) || !Array.isArray(result.replies)) {
|
|
throw new Error(`Permission approval failed for ${plan.sessionId}: ${JSON.stringify(result)}`);
|
|
}
|
|
const failures = result.replies.filter((reply) => !isRecord(reply) || typeof reply.status !== "number" || reply.status < 200 || reply.status >= 300);
|
|
if (failures.length > 0) throw new Error(`Permission replies failed for ${plan.sessionId}: ${JSON.stringify(failures)}`);
|
|
return result.replies.length;
|
|
}
|
|
|
|
async function waitForSlowTool(desktopApp: App, plan: WorkspacePlan): Promise<SessionFacts> {
|
|
const deadline = Date.now() + 90_000;
|
|
let latest: SessionFacts | null = null;
|
|
while (Date.now() < deadline) {
|
|
const clicked = await allowVisibleToolPermission(desktopApp);
|
|
const approved = clicked ? 0 : await approvePendingToolPermissions(desktopApp, plan);
|
|
latest = await readSessionFacts(desktopApp, plan.workspaceId, plan.sessionId);
|
|
// A tool part is already labelled running while it waits for permission.
|
|
// Require one clean poll with no request left to approve before treating
|
|
// the slow command as an actually in-flight workload.
|
|
if (slowToolRunning(latest) && approved === 0 && !clicked) return latest;
|
|
await sleep(500);
|
|
}
|
|
throw new Error(`Timed out waiting for workspace ${plan.index} slow tool to start after permission approval; latest=${JSON.stringify(latest)}`);
|
|
}
|
|
|
|
test.skipIf(!runnable)(
|
|
`three workspaces keep independent live tool runs through exact-route switching${skipSuffix}`,
|
|
{ timeout: 25 * 60_000 },
|
|
async ({ evidence, place }) => {
|
|
needs({ optIn: ["OPENWORK_EVAL_E2E_TESTS"] });
|
|
const runId = `${Date.now().toString(36)}-${process.pid}`;
|
|
const plans = buildPlans(runId);
|
|
await using den = await server({
|
|
place,
|
|
mocks: { agent: mcpMock({ agentWorkloads: agentWorkloads(plans) }) },
|
|
org: {
|
|
name: "Active Session Workspace Storm",
|
|
admin: { name: "Storm Admin" },
|
|
members: { member: { name: "Storm Member" } },
|
|
},
|
|
});
|
|
await using desktopApp = await app({
|
|
den,
|
|
as: "member",
|
|
place,
|
|
...(diagnosticProfileDir ? { profileDir: diagnosticProfileDir } : {}),
|
|
});
|
|
|
|
const initialDen = await eventually(() => readDenClientState(desktopApp), {
|
|
within: 60_000,
|
|
label: "initial authenticated organization",
|
|
until: (state) => state.authTokenPresent && Boolean(state.activeOrgId),
|
|
});
|
|
const initialConnect = await eventually(() => readConnectState(desktopApp), {
|
|
within: 120_000,
|
|
intervalMs: 1_000,
|
|
label: "initial Connect availability",
|
|
until: (state) => state.ok && state.status === "available" && state.connectEnabled === true,
|
|
});
|
|
const orgId = initialDen.activeOrgId;
|
|
if (!orgId) throw new Error("The signed-in baseline has no active organization.");
|
|
evidence.recordAssertionEvidence(
|
|
"The workload begins with coherent Den auth, organization, and Connect state",
|
|
`authTokenPresent=${initialDen.authTokenPresent}; activeOrgId=${orgId}; Connect=${initialConnect.status}/${initialConnect.connectEnabled}.`,
|
|
initialDen.authTokenPresent && initialConnect.ok && initialConnect.status === "available" && initialConnect.connectEnabled === true,
|
|
);
|
|
|
|
for (const plan of plans) plan.workspaceId = await createWorkspace(desktopApp, plan.path);
|
|
expect(new Set(plans.map((plan) => plan.workspaceId)).size).toBe(workspaceCount);
|
|
await configureWorkspaces(desktopApp, plans, den.mocks.agent.url);
|
|
// The already-mounted model store predates these workspace configs. Reload
|
|
// once, before any session exists, so every workspace sees the same mock
|
|
// provider without perturbing an in-flight engine event subscription.
|
|
await evalIn(desktopApp, "location.reload(); true");
|
|
await waitFor(desktopApp, "Boolean(window.__openworkControl)", {
|
|
timeoutMs: 60_000,
|
|
label: "desktop reloaded with the storm model preference",
|
|
});
|
|
await eventually(() => readDenClientState(desktopApp), {
|
|
within: 60_000,
|
|
label: "Den session restored after provider setup reload",
|
|
until: (state) => state.authTokenPresent && state.activeOrgId === orgId,
|
|
});
|
|
|
|
const workloadStartedAt = new Date().toISOString();
|
|
for (const plan of plans) {
|
|
await go(desktopApp, `/workspace/${plan.workspaceId}/session`);
|
|
await waitFor(desktopApp, `(localStorage.getItem("openwork.react.activeWorkspace") ?? "") === ${JSON.stringify(plan.workspaceId)}`, {
|
|
timeoutMs: 60_000,
|
|
label: `workspace ${plan.index} active before task creation`,
|
|
});
|
|
const selected = await selectModel(desktopApp, modelId);
|
|
expect(selected.id).toBe(modelId);
|
|
const created = await control(desktopApp, "session.create_task", undefined, { timeoutMs: 60_000 });
|
|
if (typeof created !== "string" || !created.startsWith("ses_")) {
|
|
throw new Error(`Workspace ${plan.index} did not create a session: ${JSON.stringify(created)}`);
|
|
}
|
|
plan.sessionId = created;
|
|
await control(desktopApp, "session.rename", {
|
|
sessionId: plan.sessionId,
|
|
title: `Active storm workspace ${plan.index}`,
|
|
}, { timeoutMs: 30_000 });
|
|
await openExactSessionRoute(desktopApp, plan);
|
|
const prompt = `Run the deterministic active-session workload identified by ${plan.marker}. You may run its shell commands and create, append, and read its workspace file. Keep working through every step and report only when complete.`;
|
|
await writeComposerText(desktopApp, prompt);
|
|
await control(desktopApp, "composer.send", undefined, { timeoutMs: 120_000 });
|
|
// Under a remote Daytona renderer the message list can lag behind the
|
|
// composer transition even though the turn is already admitted (the UI
|
|
// visibly says Thinking). Use the workspace-scoped server snapshot as
|
|
// the authoritative admission witness instead of waiting for a DOM row.
|
|
await eventually(
|
|
() => readSessionFacts(desktopApp, plan.workspaceId, plan.sessionId),
|
|
{
|
|
within: 60_000,
|
|
intervalMs: 500,
|
|
label: `workspace ${plan.index} prompt admitted to its session`,
|
|
until: (facts) => facts.text.includes(plan.marker),
|
|
},
|
|
);
|
|
const running = await waitForSlowTool(desktopApp, plan);
|
|
expect(running.ok).toBe(true);
|
|
expect(running.sessionId).toBe(plan.sessionId);
|
|
}
|
|
|
|
evidence.recordAssertionEvidence(
|
|
"All three originating workspaces reached a live slow tool concurrently",
|
|
`${plans.map((plan) => `${plan.workspaceId}/${plan.sessionId}`).join(", ")} each exposed a pending or running bash sleep before route switching began; configured slow duration=${slowToolMs}ms.`,
|
|
plans.length === workspaceCount && plans.every((plan) => Boolean(plan.workspaceId && plan.sessionId)),
|
|
);
|
|
const engineBeforeStorm = await readEngineRuntimeFacts(desktopApp);
|
|
|
|
await openExactSessionRoute(desktopApp, plans[0]);
|
|
const liveSurface = await readSurfaceFacts(desktopApp, plans[0].marker);
|
|
expect(liveSurface.bodyHasMarker).toBe(true);
|
|
expect(liveSurface.bodyHasToolActivity).toBe(true);
|
|
expect(liveSurface.sessionId).toBe(plans[0].sessionId);
|
|
expect(liveSurface.authActions).toEqual([]);
|
|
const liveShot = await screenshot(desktopApp);
|
|
const liveValidation = await validate(liveShot, [
|
|
"The active workspace session visibly shows its deterministic workload prompt or marker",
|
|
"The session visibly shows live agent activity such as Thinking, a running tool, or a bash command",
|
|
"No sign-in, reconnect, or application crash screen is visible",
|
|
]);
|
|
expect(liveValidation.ok, liveValidation.why).toBe(true);
|
|
|
|
const authDrops: string[] = [];
|
|
const orgChanges: string[] = [];
|
|
const connectFailures: string[] = [];
|
|
const reconnectSurfaces: string[] = [];
|
|
const nonLiveSamples: string[] = [];
|
|
const liveSamples = new Map(plans.map((plan) => [plan.sessionId, 0]));
|
|
const stormStartedAt = Date.now();
|
|
let switches = 0;
|
|
while (Date.now() - stormStartedAt < routeStormMs) {
|
|
for (const plan of plans) {
|
|
await openExactSessionRoute(desktopApp, plan);
|
|
switches += 1;
|
|
const [facts, denState, surface] = await Promise.all([
|
|
readSessionFacts(desktopApp, plan.workspaceId, plan.sessionId),
|
|
readDenClientState(desktopApp),
|
|
readSurfaceFacts(desktopApp, plan.marker),
|
|
]);
|
|
if (slowToolRunning(facts)) {
|
|
liveSamples.set(plan.sessionId, (liveSamples.get(plan.sessionId) ?? 0) + 1);
|
|
} else {
|
|
nonLiveSamples.push(`switch ${switches}: ${plan.sessionId} tools=${JSON.stringify(facts.tools)}`);
|
|
}
|
|
if (!denState.authTokenPresent) authDrops.push(`switch ${switches}: ${plan.sessionId}`);
|
|
if (denState.activeOrgId !== orgId) orgChanges.push(`switch ${switches}: ${String(denState.activeOrgId)}`);
|
|
if (surface.authActions.length > 0 || surface.crash || !surface.bodyHasMarker) {
|
|
reconnectSurfaces.push(`switch ${switches}: ${JSON.stringify(surface)}`);
|
|
}
|
|
await sleep(250);
|
|
}
|
|
const connect = await readConnectState(desktopApp);
|
|
if (!connect.ok || connect.status !== "available" || connect.connectEnabled !== true) {
|
|
connectFailures.push(`after switch ${switches}: ${JSON.stringify(connect)}`);
|
|
}
|
|
}
|
|
|
|
expect(switches).toBeGreaterThanOrEqual(9);
|
|
expect([...liveSamples.values()].every((count) => count >= 3), JSON.stringify([...liveSamples])).toBe(true);
|
|
expect(nonLiveSamples).toEqual([]);
|
|
expect(authDrops).toEqual([]);
|
|
expect(orgChanges).toEqual([]);
|
|
expect(connectFailures).toEqual([]);
|
|
expect(reconnectSurfaces).toEqual([]);
|
|
evidence.recordAssertionEvidence(
|
|
"Repeated exact-session-route switching preserves every in-flight turn and Cloud identity",
|
|
`${switches} exact route switches over ${Date.now() - stormStartedAt}ms; live samples=${JSON.stringify([...liveSamples])}; non-live=${JSON.stringify(nonLiveSamples)}; auth drops=${JSON.stringify(authDrops)}; org changes=${JSON.stringify(orgChanges)}; Connect failures=${JSON.stringify(connectFailures)}; reconnect/crash surfaces=${JSON.stringify(reconnectSurfaces)}.`,
|
|
switches >= 9
|
|
&& [...liveSamples.values()].every((count) => count >= 3)
|
|
&& nonLiveSamples.length === 0
|
|
&& authDrops.length === 0
|
|
&& orgChanges.length === 0
|
|
&& connectFailures.length === 0
|
|
&& reconnectSurfaces.length === 0,
|
|
);
|
|
const engineAfterStorm = await readEngineRuntimeFacts(desktopApp);
|
|
evidence.recordAssertionEvidence(
|
|
"The route storm records the managed engine generation topology",
|
|
`Before=${JSON.stringify(engineBeforeStorm)}; after=${JSON.stringify(engineAfterStorm)}.`,
|
|
engineBeforeStorm.enginePid !== null && engineAfterStorm.enginePid !== null,
|
|
);
|
|
|
|
const expectedTools = ["bash", "bash", "bash", "bash", "bash", "bash"];
|
|
for (const plan of plans) {
|
|
await openExactSessionRoute(desktopApp, plan);
|
|
const complete = await eventually(
|
|
async () => {
|
|
const facts = await readSessionFacts(desktopApp, plan.workspaceId, plan.sessionId);
|
|
if (!facts.text.includes(plan.finalReply)) {
|
|
const clicked = await allowVisibleToolPermission(desktopApp);
|
|
if (!clicked) await approvePendingToolPermissions(desktopApp, plan);
|
|
}
|
|
return facts;
|
|
},
|
|
{
|
|
within: slowToolMs + 60_000,
|
|
intervalMs: 1_000,
|
|
label: `workspace ${plan.index} independent completion`,
|
|
until: (facts) => facts.text.includes(plan.finalReply)
|
|
&& facts.tools.length === expectedTools.length
|
|
&& facts.tools.every((tool) => tool.status === "completed"),
|
|
},
|
|
).catch(async (error: unknown) => {
|
|
const [stuck, file, engine, logSignals] = await Promise.all([
|
|
readSessionFacts(desktopApp, plan.workspaceId, plan.sessionId),
|
|
readWorkspaceFileFacts(desktopApp, plan),
|
|
readEngineRuntimeFacts(desktopApp),
|
|
readEngineLogSignals(desktopApp),
|
|
]);
|
|
evidence.recordAssertionEvidence(
|
|
`Workspace ${plan.index} independently completes after route switching`,
|
|
`Completion exceeded its strict bound; origin=${plan.workspaceId}/${plan.sessionId}; last snapshot=${JSON.stringify(stuck)}; file=${JSON.stringify(file)}; engine=${JSON.stringify(engine)}; engine log signals=${JSON.stringify(logSignals)}.`,
|
|
false,
|
|
);
|
|
throw error;
|
|
});
|
|
const corpus = sessionCorpus(complete);
|
|
expect(complete.sessionId).toBe(plan.sessionId);
|
|
expect(complete.tools.map((tool) => tool.tool)).toEqual(expectedTools);
|
|
expect(corpus).toContain(plan.filePath);
|
|
expect(corpus).toContain(`INITIAL-${plan.marker}`);
|
|
expect(corpus).toContain(plan.slowMarker);
|
|
expect(corpus).toContain(plan.easyMarker);
|
|
expect(corpus).toContain(plan.finalReply);
|
|
for (const other of plans.filter((candidate) => candidate.index !== plan.index)) {
|
|
expect(corpus).not.toContain(other.marker);
|
|
expect(corpus).not.toContain(other.filePath);
|
|
}
|
|
|
|
const providerRequests = await den.mocks.agent.agentRequests({
|
|
promptMarker: plan.marker,
|
|
sinceIso: workloadStartedAt,
|
|
});
|
|
const mainRequests = providerRequests.filter((request) => request.kind !== "utility");
|
|
expect(mainRequests.map((request) => request.kind)).toEqual([
|
|
"tool", "tool", "tool", "tool", "tool", "tool", "final",
|
|
]);
|
|
expect(mainRequests.filter((request) => request.kind === "tool").map((request) => request.toolName)).toEqual(expectedTools);
|
|
expect(mainRequests.every((request) => request.matchedMarkers.length === 1
|
|
&& request.matchedMarkers[0] === plan.marker)).toBe(true);
|
|
expect(mainRequests.filter((request) => request.kind === "tool")
|
|
.every((request) => JSON.stringify(request.arguments).includes(plan.filePath))).toBe(true);
|
|
|
|
const wrongWorkspaceStatuses: number[] = [];
|
|
for (const other of plans.filter((candidate) => candidate.index !== plan.index)) {
|
|
wrongWorkspaceStatuses.push((await readSessionFacts(desktopApp, other.workspaceId, plan.sessionId)).status);
|
|
}
|
|
expect(wrongWorkspaceStatuses.every((status) => status === 404)).toBe(true);
|
|
|
|
const finalSurface = await readSurfaceFacts(desktopApp, plan.marker);
|
|
expect(finalSurface.sessionId).toBe(plan.sessionId);
|
|
expect(finalSurface.bodyHasMarker).toBe(true);
|
|
expect(finalSurface.authActions).toEqual([]);
|
|
expect(finalSurface.crash).toBe(false);
|
|
const completeShot = await screenshot(desktopApp);
|
|
const completeValidation = await validate(completeShot, [
|
|
`The session visibly shows the completed workload for workspace ${plan.index}`,
|
|
"The session transcript visibly contains agent or tool activity rather than an empty New session screen",
|
|
"No sign-in, reconnect, or application crash screen is visible",
|
|
]);
|
|
expect(completeValidation.ok, completeValidation.why).toBe(true);
|
|
evidence.recordAssertionEvidence(
|
|
`Workspace ${plan.index} completed only its own six-step workload in its originating session`,
|
|
`Origin=${plan.workspaceId}/${plan.sessionId}; tools=${JSON.stringify(complete.tools)}; provider=${JSON.stringify(mainRequests)}; wrong-workspace snapshot statuses=${JSON.stringify(wrongWorkspaceStatuses)}; transcript=${JSON.stringify(complete.text)}.`,
|
|
complete.text.includes(plan.finalReply)
|
|
&& complete.tools.map((tool) => tool.tool).join(",") === expectedTools.join(",")
|
|
&& plans.filter((candidate) => candidate.index !== plan.index).every((other) => !corpus.includes(other.marker))
|
|
&& wrongWorkspaceStatuses.every((status) => status === 404),
|
|
);
|
|
}
|
|
|
|
const finalDen = await readDenClientState(desktopApp);
|
|
const finalConnect = await eventually(() => readConnectState(desktopApp), {
|
|
within: 120_000,
|
|
intervalMs: 1_000,
|
|
label: "Connect available after all active sessions complete",
|
|
until: (state) => state.ok && state.status === "available" && state.connectEnabled === true,
|
|
});
|
|
expect(finalDen.authTokenPresent).toBe(true);
|
|
expect(finalDen.activeOrgId).toBe(orgId);
|
|
expect(finalConnect.status).toBe("available");
|
|
expect(finalConnect.connectEnabled).toBe(true);
|
|
|
|
const healthSummaries = [];
|
|
for (const plan of plans) {
|
|
healthSummaries.push(await eventually(
|
|
() => readCloudMcpHealth(desktopApp, plan.workspaceId, { probe: true, timeoutMs: 30_000 }),
|
|
{
|
|
within: 180_000,
|
|
intervalMs: 3_000,
|
|
label: `workspace ${plan.index} Cloud MCP usable after completion`,
|
|
until: (health) => health.ok && health.usable === true,
|
|
},
|
|
));
|
|
}
|
|
expect(healthSummaries.every((health) => health.ok && health.usable === true)).toBe(true);
|
|
evidence.recordAssertionEvidence(
|
|
"Den auth, organization selection, and Connect remain coherent after every session completes",
|
|
`Final authTokenPresent=${finalDen.authTokenPresent}; activeOrgId=${String(finalDen.activeOrgId)}; Connect=${finalConnect.status}/${finalConnect.connectEnabled}; per-workspace Cloud MCP health=${JSON.stringify(healthSummaries)}.`,
|
|
finalDen.authTokenPresent
|
|
&& finalDen.activeOrgId === orgId
|
|
&& finalConnect.ok
|
|
&& finalConnect.status === "available"
|
|
&& finalConnect.connectEnabled === true
|
|
&& healthSummaries.every((health) => health.ok && health.usable === true),
|
|
);
|
|
},
|
|
);
|