源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)
This commit is contained in:
agent-radar
+357
@@ -0,0 +1,357 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import type { DaytonaExec } from "@openwork/hosts";
|
||||
import {
|
||||
createDaytonaK3sCluster,
|
||||
exposeK3sService,
|
||||
installK3sHelmRelease,
|
||||
parseDaytonaK3sPreviewUrl,
|
||||
provisionDaytonaK3sSandbox,
|
||||
} from "../src/daytona-k3s.ts";
|
||||
import { createPlacement } from "../src/network-world.ts";
|
||||
import type { DaytonaK3sClusterHandle, DaytonaK3sSandboxOwnership } from "../src/daytona-k3s.ts";
|
||||
import type { Placement } from "../src/network-world.ts";
|
||||
|
||||
interface ExecCall {
|
||||
args: string[];
|
||||
opts?: { input?: string; timeoutMs?: number };
|
||||
}
|
||||
|
||||
interface FakeOptions {
|
||||
uid?: string;
|
||||
sudoFails?: boolean;
|
||||
failAt?: "sandbox-create" | "sandbox-readiness" | "install" | "start" | "readiness" | "preview";
|
||||
helmMissing?: boolean;
|
||||
}
|
||||
|
||||
const SANDBOX = "owned-sandbox-1";
|
||||
const OFFICIAL_VERSION = "v1.31.6+k3s1";
|
||||
const OFFICIAL_URL = "https://github.com/k3s-io/k3s/releases/download/v1.31.6%2Bk3s1/k3s";
|
||||
const OFFICIAL_SHA256 = "9f82f06b4cf318fcf4eeda3f4fedaa10c0cebc418b1a047e72b104f5ea7874c5";
|
||||
const placement = createPlacement({ id: "unit-cluster", provider: "daytona-k3s" });
|
||||
const root = "/tmp/openwork-world-k3s/unit-cluster";
|
||||
|
||||
function remoteScript(call: ExecCall): string {
|
||||
if (call.args[0] !== "exec") return "";
|
||||
assert.equal(call.args.length, 4);
|
||||
assert.deepEqual(call.args.slice(0, 3), ["exec", SANDBOX, "--"]);
|
||||
const wrapped = call.args[3] ?? "";
|
||||
const prefix = "bash -lc '";
|
||||
assert(wrapped.startsWith(prefix) && wrapped.endsWith("'"), `Unexpected Daytona exec transport: ${wrapped}`);
|
||||
return wrapped.slice(prefix.length, -1).replaceAll(`'"'"'`, "'");
|
||||
}
|
||||
|
||||
function createFake(options: FakeOptions = {}): { exec: DaytonaExec; calls: ExecCall[] } {
|
||||
const calls: ExecCall[] = [];
|
||||
const exec: DaytonaExec = async (args, opts) => {
|
||||
const call = { args: [...args], opts };
|
||||
calls.push(call);
|
||||
if (args[0] === "create") {
|
||||
if (options.failAt === "sandbox-create") return { stdout: "", stderr: "creation failed\n", code: 1 };
|
||||
return { stdout: "created\n", stderr: "", code: 0 };
|
||||
}
|
||||
if (args[0] === "delete") return { stdout: "deleted\n", stderr: "", code: 0 };
|
||||
if (args[0] === "preview-url") {
|
||||
if (options.failAt === "preview") return { stdout: "", stderr: "preview failed\n", code: 1 };
|
||||
return { stdout: "Preview URL: https://30443.preview.example.test/signed?token=unit\n", stderr: "", code: 0 };
|
||||
}
|
||||
const script = remoteScript(call);
|
||||
if (script === "'true'") {
|
||||
if (options.failAt === "sandbox-readiness") return { stdout: "", stderr: "not ready\n", code: 1 };
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
}
|
||||
if (script === "'id' '-u'") return { stdout: `${options.uid ?? "0"}\n`, stderr: "", code: 0 };
|
||||
if (script === "'sudo' '-n' 'true'" && options.sudoFails) {
|
||||
return { stdout: "", stderr: "sudo: a password is required\n", code: 1 };
|
||||
}
|
||||
if (script.includes("'curl' '--fail'")) {
|
||||
if (options.failAt === "install") return { stdout: "", stderr: "checksum mismatch\n", code: 1 };
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
}
|
||||
if (script.includes("'nohup'") && script.includes("'server'")) {
|
||||
if (options.failAt === "start") throw new Error("ambiguous Daytona transport failure");
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
}
|
||||
if (script.includes("'--raw=/readyz'") && options.failAt === "readiness") {
|
||||
return { stdout: "", stderr: "owned server exited\n", code: 1 };
|
||||
}
|
||||
if (script.startsWith("'helm' ") && options.helmMissing) {
|
||||
return { stdout: "", stderr: "bash: helm: command not found\n", code: 127 };
|
||||
}
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
};
|
||||
return { exec, calls };
|
||||
}
|
||||
|
||||
async function provision(fake: { exec: DaytonaExec }): Promise<DaytonaK3sSandboxOwnership> {
|
||||
return provisionDaytonaK3sSandbox({ name: SANDBOX, exec: fake.exec });
|
||||
}
|
||||
|
||||
async function createCluster(fake: { exec: DaytonaExec }): Promise<DaytonaK3sClusterHandle> {
|
||||
const ownership = await provision(fake);
|
||||
return createDaytonaK3sCluster({ placement, ownership });
|
||||
}
|
||||
|
||||
function scripts(calls: ExecCall[]): string[] {
|
||||
return calls.filter((call) => call.args[0] === "exec").map(remoteScript);
|
||||
}
|
||||
|
||||
function deletionCalls(calls: ExecCall[]): ExecCall[] {
|
||||
return calls.filter((call) => call.args[0] === "delete");
|
||||
}
|
||||
|
||||
test("provisioning uses a safe private Daytona sandbox and v0.173 one-argument bash transport", async () => {
|
||||
const fake = createFake();
|
||||
const ownership = await provision(fake);
|
||||
assert.deepEqual(fake.calls[0], {
|
||||
args: ["create", "--name", SANDBOX, "--snapshot", "daytona-large", "--auto-delete", "0", "--target", "us"],
|
||||
opts: { timeoutMs: 300_000 },
|
||||
});
|
||||
assert.equal(fake.calls[0]?.args.includes("--public"), false);
|
||||
assert.deepEqual(fake.calls[1], {
|
||||
args: ["exec", SANDBOX, "--", "bash -lc ''\"'\"'true'\"'\"''"],
|
||||
opts: { timeoutMs: 60_000 },
|
||||
});
|
||||
|
||||
const cluster = await createDaytonaK3sCluster({ placement, ownership });
|
||||
await cluster.stop();
|
||||
assert.equal(parseDaytonaK3sPreviewUrl("Preview URL: https://preview.example.test/path?token=abc\n"), "https://preview.example.test/path?token=abc");
|
||||
assert.equal(parseDaytonaK3sPreviewUrl(`Preview URL: https://preview.example.test/path${",".repeat(10_000)}`), "https://preview.example.test/path");
|
||||
});
|
||||
|
||||
test("provisioning rejects unsafe names and non-allowlisted snapshots before execution", async () => {
|
||||
const fake = createFake();
|
||||
await assert.rejects(provisionDaytonaK3sSandbox({ name: "unsafe sandbox", exec: fake.exec }), /sandbox name/);
|
||||
await assert.rejects(provisionDaytonaK3sSandbox({
|
||||
name: SANDBOX,
|
||||
// @ts-expect-error Verify the runtime boundary as well as the typed snapshot allowlist.
|
||||
snapshot: "moving-snapshot",
|
||||
exec: fake.exec,
|
||||
}), /not allowlisted/);
|
||||
assert.equal(fake.calls.length, 0);
|
||||
});
|
||||
|
||||
test("partial sandbox creation and exec-readiness failures delete the whole sandbox", async () => {
|
||||
const failures: readonly ("sandbox-create" | "sandbox-readiness")[] = ["sandbox-create", "sandbox-readiness"];
|
||||
for (const failAt of failures) {
|
||||
const fake = createFake({ failAt });
|
||||
await assert.rejects(provision(fake));
|
||||
assert.equal(deletionCalls(fake.calls).length, 1, `expected sandbox deletion after ${failAt}`);
|
||||
assert.equal(scripts(fake.calls).some((script) => /\b(?:kill|pkill)\b|\.pid|PID/.test(script)), false);
|
||||
}
|
||||
});
|
||||
|
||||
test("a forged ownership receipt is rejected before cluster exec or deletion", async () => {
|
||||
const fake = createFake();
|
||||
const ownership = await provision(fake);
|
||||
const forgedOwnership = structuredClone(ownership);
|
||||
const before = fake.calls.length;
|
||||
await assert.rejects(
|
||||
createDaytonaK3sCluster({ placement, ownership: forgedOwnership }),
|
||||
/ownership receipt returned by provisionDaytonaK3sSandbox/,
|
||||
);
|
||||
assert.equal(fake.calls.length, before);
|
||||
assert.equal(deletionCalls(fake.calls).length, 0);
|
||||
|
||||
const cluster = await createDaytonaK3sCluster({ placement, ownership });
|
||||
const afterClaim = fake.calls.length;
|
||||
await assert.rejects(createDaytonaK3sCluster({ placement, ownership }), /unused ownership receipt/);
|
||||
assert.equal(fake.calls.length, afterClaim);
|
||||
await cluster.stop();
|
||||
});
|
||||
|
||||
test("root lifecycle downloads only the hardcoded official binary and deletes its sandbox idempotently", async () => {
|
||||
const fake = createFake();
|
||||
const cluster = await createCluster(fake);
|
||||
const observed = scripts(fake.calls);
|
||||
|
||||
assert(observed.includes("'id' '-u'"));
|
||||
const downloads = observed.filter((script) => script.includes("'curl' '--fail'"));
|
||||
assert.equal(downloads.length, 1);
|
||||
const install = downloads[0] ?? "";
|
||||
assert(install.includes(`'curl' '--fail' '--silent' '--show-error' '--location' '${OFFICIAL_URL}' '--output' '${root}/download/k3s'`));
|
||||
assert(install.includes(`'printf' '%s\\n' '${OFFICIAL_SHA256} ${root}/download/k3s' | 'sha256sum' '--check' '--status' '-'`));
|
||||
assert(install.includes(`'mv' '-f' '${root}/download/k3s' '${root}/bin/k3s'`));
|
||||
assert.doesNotMatch(install, /get\.k3s\.io|systemctl|openrc|rc-service/);
|
||||
|
||||
const start = observed.find((script) => script.includes("'server'")) ?? "";
|
||||
assert(start.includes(`'nohup' '${root}/bin/k3s' 'server' '--data-dir' '${root}/data' '--write-kubeconfig' '${root}/kubeconfig.yaml' '--write-kubeconfig-mode' '0600'`));
|
||||
assert.doesNotMatch(start, /'--write-kubeconfig-mode' '0644'/);
|
||||
assert(start.includes("'--node-name' 'openwork-unit-cluster'"));
|
||||
assert(start.includes("'--snapshotter' 'native'"));
|
||||
const readiness = observed.find((script) => script.includes("'--raw=/readyz'")) ?? "";
|
||||
assert(readiness.includes(`'${root}/bin/k3s' 'kubectl' '--kubeconfig' '${root}/kubeconfig.yaml' 'get' '--raw=/readyz'`));
|
||||
assert.doesNotMatch(readiness, /pgrep/);
|
||||
assert.equal(cluster.version, OFFICIAL_VERSION);
|
||||
assert.deepEqual(cluster.paths, {
|
||||
root,
|
||||
binary: `${root}/bin/k3s`,
|
||||
download: `${root}/download/k3s`,
|
||||
dataDir: `${root}/data`,
|
||||
kubeconfig: `${root}/kubeconfig.yaml`,
|
||||
serverLog: `${root}/server.log`,
|
||||
});
|
||||
|
||||
await cluster.stop();
|
||||
await cluster.stop();
|
||||
await cluster[Symbol.asyncDispose]();
|
||||
assert.equal(deletionCalls(fake.calls).length, 1);
|
||||
assert.deepEqual(deletionCalls(fake.calls)[0], {
|
||||
args: ["delete", SANDBOX],
|
||||
opts: { timeoutMs: 60_000, input: "y\n" },
|
||||
});
|
||||
assert(scripts(fake.calls).every((script) => !/\b(?:kill|pkill)\b|\.pid|PID/.test(script)));
|
||||
});
|
||||
|
||||
test("non-root lifecycle requires passwordless sudo for k3s, kubectl, and Helm kubeconfig access", async () => {
|
||||
const fake = createFake({ uid: "1000" });
|
||||
const cluster = await createCluster(fake);
|
||||
const observed = scripts(fake.calls);
|
||||
|
||||
assert(observed.includes("'id' '-u'"));
|
||||
assert(observed.includes("'sudo' '-n' 'true'"));
|
||||
assert(observed.some((script) => script.includes(`'nohup' 'sudo' '-n' '${root}/bin/k3s' 'server'`)));
|
||||
assert(observed.some((script) => script.includes(`'sudo' '-n' '${root}/bin/k3s' 'kubectl' '--kubeconfig' '${root}/kubeconfig.yaml'`)));
|
||||
await cluster.kubectl(["get", "pods"]);
|
||||
assert.equal(scripts(fake.calls).at(-1), `'sudo' '-n' '${root}/bin/k3s' 'kubectl' '--kubeconfig' '${root}/kubeconfig.yaml' 'get' 'pods'`);
|
||||
await cluster.helm(["list", "--all-namespaces"]);
|
||||
assert.equal(scripts(fake.calls).at(-1), `'sudo' '-n' 'helm' '--kubeconfig' '${root}/kubeconfig.yaml' 'list' '--all-namespaces'`);
|
||||
await cluster.stop();
|
||||
});
|
||||
|
||||
test("lack of root and passwordless sudo fails before start and deletes the owned sandbox", async () => {
|
||||
const fake = createFake({ uid: "1000", sudoFails: true });
|
||||
await assert.rejects(createCluster(fake), /passwordless sudo.*failed|password is required/s);
|
||||
assert(scripts(fake.calls).every((script) => !script.includes("'server'")));
|
||||
assert.equal(deletionCalls(fake.calls).length, 1);
|
||||
});
|
||||
|
||||
test("every install, ambiguous start, and readiness failure deletes the entire owned sandbox", async () => {
|
||||
const failures: readonly NonNullable<FakeOptions["failAt"]>[] = ["install", "start", "readiness"];
|
||||
for (const failAt of failures) {
|
||||
const fake = createFake({ failAt });
|
||||
await assert.rejects(createCluster(fake));
|
||||
assert.equal(deletionCalls(fake.calls).length, 1, `expected owned sandbox deletion after ${failAt}`);
|
||||
assert(scripts(fake.calls).every((script) => !/\b(?:kill|pkill)\b|\.pid|PID/.test(script)));
|
||||
}
|
||||
});
|
||||
|
||||
test("Helm uses the root-selected privilege and missing Helm fails without curl-pipe installation", async () => {
|
||||
const fake = createFake({ helmMissing: true });
|
||||
const cluster = await createCluster(fake);
|
||||
const before = fake.calls.length;
|
||||
await assert.rejects(
|
||||
installK3sHelmRelease(cluster, { release: "demo", namespace: "demo-ns", chart: "oci://registry.example.test/team/chart" }),
|
||||
/helm: command not found/,
|
||||
);
|
||||
assert.equal(
|
||||
remoteScript(fake.calls[before] ?? { args: [] }),
|
||||
`'helm' '--kubeconfig' '${root}/kubeconfig.yaml' 'upgrade' '--install' 'demo' 'oci://registry.example.test/team/chart' '--namespace' 'demo-ns' '--create-namespace'`,
|
||||
);
|
||||
assert.equal(fake.calls.slice(before).filter((call) => remoteScript(call).includes("curl")).length, 0);
|
||||
await cluster.stop();
|
||||
});
|
||||
|
||||
test("cluster-owned exposure reserves ports, accepts the maximum expiry, and has no independent cleanup", async () => {
|
||||
const fake = createFake();
|
||||
const cluster = await createCluster(fake);
|
||||
const exposure = await exposeK3sService(cluster, {
|
||||
namespace: "demo-ns",
|
||||
service: "demo-api",
|
||||
localPort: 30_443,
|
||||
servicePort: 443,
|
||||
expiresInSeconds: 86_400,
|
||||
});
|
||||
|
||||
assert.equal(exposure.ephemeral, true);
|
||||
assert.equal(exposure.persistableInDesktopConfig, false);
|
||||
assert.equal(exposure.validUntil, "cluster-disposal-or-expiry");
|
||||
assert.equal(exposure.ownedSandboxId, SANDBOX);
|
||||
assert.deepEqual(fake.calls.find((call) => call.args[0] === "preview-url")?.args, [
|
||||
"preview-url", SANDBOX, "-p", "30443", "--expires", "86400",
|
||||
]);
|
||||
const portStart = scripts(fake.calls).find((script) => script.includes("'port-forward'"));
|
||||
assert(portStart?.includes(`'${root}/bin/k3s' 'kubectl' '--kubeconfig' '${root}/kubeconfig.yaml' 'port-forward'`));
|
||||
const beforeDuplicate = fake.calls.length;
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "other",
|
||||
service: "other-api",
|
||||
localPort: 30_443,
|
||||
servicePort: 80,
|
||||
expiresInSeconds: 60,
|
||||
}), /already reserved/);
|
||||
assert.equal(fake.calls.length, beforeDuplicate);
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "other",
|
||||
service: "other-api",
|
||||
localPort: 30_444,
|
||||
servicePort: 80,
|
||||
expiresInSeconds: 86_401,
|
||||
}), /between 1 and 86400/);
|
||||
assert.equal(fake.calls.length, beforeDuplicate);
|
||||
assert.equal(deletionCalls(fake.calls).length, 0);
|
||||
await cluster.stop();
|
||||
assert.equal(deletionCalls(fake.calls).length, 1);
|
||||
assert(scripts(fake.calls).every((script) => !/\b(?:kill|pkill)\b|\.pid|PID/.test(script)));
|
||||
});
|
||||
|
||||
test("an ambiguous exposure failure deletes the cluster sandbox instead of process cleanup", async () => {
|
||||
const fake = createFake({ failAt: "preview" });
|
||||
const cluster = await createCluster(fake);
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "demo",
|
||||
service: "demo-api",
|
||||
localPort: 8080,
|
||||
servicePort: 80,
|
||||
expiresInSeconds: 300,
|
||||
}), /preview failed/);
|
||||
assert.equal(deletionCalls(fake.calls).length, 1);
|
||||
assert(scripts(fake.calls).every((script) => !/\b(?:kill|pkill)\b|\.pid|PID/.test(script)));
|
||||
});
|
||||
|
||||
test("malformed placement, version, Helm, and exposure inputs fail before cluster execution", async () => {
|
||||
const fake = createFake();
|
||||
const ownership = await provision(fake);
|
||||
const local = createPlacement({ id: "local-unit", provider: "local" });
|
||||
const missingCapability: Placement = { ...placement, capabilities: ["command:bash", "port:daytona-preview"] };
|
||||
const before = fake.calls.length;
|
||||
await assert.rejects(createDaytonaK3sCluster({ placement: local, ownership }), /requires placement provider/);
|
||||
await assert.rejects(createDaytonaK3sCluster({ placement: missingCapability, ownership }), /missing capability/);
|
||||
await assert.rejects(createDaytonaK3sCluster({
|
||||
placement,
|
||||
ownership,
|
||||
// @ts-expect-error Verify the runtime boundary as well as the typed version allowlist.
|
||||
version: "v1.99.0+k3s1",
|
||||
}), /not supported/);
|
||||
assert.equal(fake.calls.length, before);
|
||||
|
||||
const cluster = await createDaytonaK3sCluster({ placement, ownership });
|
||||
const beforeInvalidTools = fake.calls.length;
|
||||
assert.throws(() => installK3sHelmRelease(cluster, { release: "Bad Release", namespace: "demo", chart: "repo/chart" }), /Helm release/);
|
||||
assert.throws(() => installK3sHelmRelease(cluster, { release: "demo", namespace: "Bad_Namespace", chart: "repo/chart" }), /Helm namespace/);
|
||||
assert.throws(() => installK3sHelmRelease(cluster, { release: "demo", namespace: "demo", chart: "--set" }), /Helm chart/);
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "demo",
|
||||
service: "Bad_Service",
|
||||
localPort: 8080,
|
||||
servicePort: 80,
|
||||
expiresInSeconds: 300,
|
||||
}), /Kubernetes service/);
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "demo",
|
||||
service: "demo-api",
|
||||
localPort: 0,
|
||||
servicePort: 80,
|
||||
expiresInSeconds: 300,
|
||||
}), /local port/);
|
||||
await assert.rejects(exposeK3sService(cluster, {
|
||||
namespace: "demo",
|
||||
service: "demo-api",
|
||||
localPort: 8080,
|
||||
servicePort: 70_000,
|
||||
expiresInSeconds: 300,
|
||||
}), /service port/);
|
||||
await assert.rejects(cluster.kubectl(["get\npods"]), /control characters/);
|
||||
assert.equal(fake.calls.length, beforeInvalidTools);
|
||||
await cluster.stop();
|
||||
});
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer, request as httpRequest } from "node:http";
|
||||
import test from "node:test";
|
||||
import { denFetch } from "@openwork/behaviors";
|
||||
import type { Place } from "../src/place.ts";
|
||||
import { faultProxy } from "../src/faults.ts";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
function listen(server: Server): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (typeof address === "object" && address !== null) resolve(address.port);
|
||||
else reject(new Error("Upstream test server did not expose a port."));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function close(server: Server): Promise<void> {
|
||||
return new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
|
||||
function absoluteGet(proxyUrl: string, target: string): Promise<string> {
|
||||
const proxy = new URL(proxyUrl);
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = httpRequest({ hostname: proxy.hostname, port: proxy.port, path: target }, (response) => {
|
||||
response.setEncoding("utf8");
|
||||
let body = "";
|
||||
response.on("data", (chunk: string) => body += chunk);
|
||||
response.on("end", () => resolve(body));
|
||||
});
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
test("faultProxy consumes status and latency rules before passing through", async () => {
|
||||
const upstream = createServer((request, response) => {
|
||||
response.writeHead(200, { "content-type": "application/json", "x-upstream": "yes" });
|
||||
response.end(JSON.stringify({ method: request.method, path: request.url }));
|
||||
});
|
||||
const port = await listen(upstream);
|
||||
try {
|
||||
await using proxy = await faultProxy({
|
||||
apiUrl: `http://127.0.0.1:${port}`,
|
||||
webUrl: `http://127.0.0.1:${port}`,
|
||||
});
|
||||
await proxy.faults.status("/api/den/flaky", 429, { times: 2, body: { error: "slow down" } });
|
||||
|
||||
const first = await fetch(`${proxy.ref.webUrl}/api/den/flaky`);
|
||||
const second = await fetch(`${proxy.ref.webUrl}/api/den/flaky`);
|
||||
const passed = await fetch(`${proxy.ref.webUrl}/api/den/flaky`);
|
||||
|
||||
assert.equal(first.status, 429);
|
||||
assert.deepEqual(await first.json(), { error: "slow down" });
|
||||
assert.equal(second.status, 429);
|
||||
assert.equal(passed.status, 200);
|
||||
assert.equal(passed.headers.get("x-upstream"), "yes");
|
||||
assert.deepEqual(await passed.json(), { method: "GET", path: "/api/den/flaky" });
|
||||
|
||||
await proxy.faults.latency("/delayed", 25);
|
||||
const startedAt = Date.now();
|
||||
const delayed = await fetch(`${proxy.ref.webUrl}/delayed`);
|
||||
assert.equal(delayed.status, 200);
|
||||
assert(Date.now() - startedAt >= 15);
|
||||
assert.equal((await fetch(`${proxy.ref.webUrl}/delayed`)).status, 200);
|
||||
const behaviorResult = await denFetch(proxy.ref, "/behavior");
|
||||
assert.equal(behaviorResult.response.status, 200);
|
||||
assert.deepEqual(behaviorResult.body, { method: "GET", path: "/api/den/behavior" });
|
||||
|
||||
assert.deepEqual(
|
||||
proxy.requests.map(({ path, status, faulted }) => ({ path, status, faulted })),
|
||||
[
|
||||
{ path: "/api/den/flaky", status: 429, faulted: true },
|
||||
{ path: "/api/den/flaky", status: 429, faulted: true },
|
||||
{ path: "/api/den/flaky", status: 200, faulted: false },
|
||||
{ path: "/delayed", status: 200, faulted: true },
|
||||
{ path: "/delayed", status: 200, faulted: false },
|
||||
{ path: "/api/den/behavior", status: 200, faulted: false },
|
||||
],
|
||||
);
|
||||
assert.deepEqual(await proxy.requestLog(), proxy.requests);
|
||||
assert.notEqual(await proxy.requestLog(), proxy.requests);
|
||||
} finally {
|
||||
await close(upstream);
|
||||
}
|
||||
});
|
||||
|
||||
test("faultProxy clear removes pending rules", async () => {
|
||||
const upstream = createServer((_request, response) => {
|
||||
response.writeHead(204);
|
||||
response.end();
|
||||
});
|
||||
const port = await listen(upstream);
|
||||
try {
|
||||
await using proxy = await faultProxy({
|
||||
apiUrl: `http://127.0.0.1:${port}`,
|
||||
webUrl: `http://127.0.0.1:${port}`,
|
||||
});
|
||||
await proxy.faults.status("/", 500, { times: 3 });
|
||||
await proxy.faults.clear();
|
||||
|
||||
assert.equal((await fetch(proxy.ref.webUrl)).status, 204);
|
||||
assert.equal(proxy.requests[0]?.faulted, false);
|
||||
} finally {
|
||||
await close(upstream);
|
||||
}
|
||||
});
|
||||
|
||||
test("faultProxy requires the Den sandbox id for Daytona placement", async () => {
|
||||
const place: Place = {
|
||||
kind: "daytona",
|
||||
host: () => undefined,
|
||||
db: async () => { throw new Error("unused"); },
|
||||
exposeMock: async () => { throw new Error("unused"); },
|
||||
denBase: () => ({ kind: "daytona", ref: "dev" }),
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
faultProxy(
|
||||
{ apiUrl: "https://den-api.example.test", webUrl: "https://den.example.test" },
|
||||
{ place, sandbox: undefined },
|
||||
),
|
||||
/fault proxy on Daytona needs the Den sandbox id; pass `sandbox: den\.placement\.sandboxId`/,
|
||||
);
|
||||
});
|
||||
|
||||
test("faultProxy pins absolute-form request targets to its upstream", async () => {
|
||||
let attackerRequests = 0;
|
||||
let upstreamRequestUrl: string | undefined;
|
||||
const upstream = createServer((request, response) => {
|
||||
upstreamRequestUrl = request.url;
|
||||
response.end("upstream");
|
||||
});
|
||||
const attacker = createServer((_request, response) => {
|
||||
attackerRequests += 1;
|
||||
response.end("attacker");
|
||||
});
|
||||
const [upstreamPort, attackerPort] = await Promise.all([listen(upstream), listen(attacker)]);
|
||||
try {
|
||||
await using proxy = await faultProxy({
|
||||
apiUrl: `http://127.0.0.1:${upstreamPort}`,
|
||||
webUrl: `http://127.0.0.1:${upstreamPort}`,
|
||||
});
|
||||
assert.equal(
|
||||
await absoluteGet(proxy.ref.webUrl, `http://127.0.0.1:${attackerPort}/steered?x=1`),
|
||||
"upstream",
|
||||
);
|
||||
assert.equal(upstreamRequestUrl, "/steered?x=1");
|
||||
assert.equal(attackerRequests, 0);
|
||||
} finally {
|
||||
await Promise.all([close(upstream), close(attacker)]);
|
||||
}
|
||||
});
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
currentDockerPlatform,
|
||||
ensureKindDenReady,
|
||||
ensureRelease,
|
||||
exposeEndpointHandles,
|
||||
helmUpgradeArgs,
|
||||
kubeProfileConfig,
|
||||
kubeStackDown,
|
||||
manifestSupportsPlatform,
|
||||
portForwardArgs,
|
||||
resolveKubeImagePlan,
|
||||
rolloutStatusArgs,
|
||||
} from "../src/kind-stack.ts";
|
||||
import type { KubeExec, KubeExecOptions, KubeExecResult, KubeImagePlan, KubeSpawnDetached } from "../src/kind-stack.ts";
|
||||
|
||||
interface ExecCall {
|
||||
command: string;
|
||||
args: string[];
|
||||
options?: KubeExecOptions;
|
||||
}
|
||||
|
||||
function success(stdout = ""): KubeExecResult {
|
||||
return { stdout, stderr: "", code: 0 };
|
||||
}
|
||||
|
||||
function createExec(handler: (call: ExecCall) => KubeExecResult): { exec: KubeExec; calls: ExecCall[] } {
|
||||
const calls: ExecCall[] = [];
|
||||
const exec: KubeExec = async (command, args, options) => {
|
||||
const call = { command, args: [...args], options };
|
||||
calls.push(call);
|
||||
return handler(call);
|
||||
};
|
||||
return { exec, calls };
|
||||
}
|
||||
|
||||
function imagePlan(mode: "published" | "local"): KubeImagePlan {
|
||||
return {
|
||||
mode,
|
||||
denApiRepository: mode === "local" ? "openwork-den-api" : "ghcr.io/different-ai/openwork-den-api",
|
||||
denWebRepository: mode === "local" ? "openwork-den-web" : "ghcr.io/different-ai/openwork-den-web",
|
||||
tag: mode === "local" ? "kube-lab" : "latest",
|
||||
pullPolicy: "IfNotPresent",
|
||||
reason: "test",
|
||||
};
|
||||
}
|
||||
|
||||
function manifestFor(architecture: string): unknown {
|
||||
return { manifests: [{ platform: { os: "linux", architecture } }] };
|
||||
}
|
||||
|
||||
function withCleanImageEnv<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const previous = process.env.OPENWORK_EVAL_KUBE_IMAGES;
|
||||
delete process.env.OPENWORK_EVAL_KUBE_IMAGES;
|
||||
return fn().finally(() => {
|
||||
if (previous === undefined) delete process.env.OPENWORK_EVAL_KUBE_IMAGES;
|
||||
else process.env.OPENWORK_EVAL_KUBE_IMAGES = previous;
|
||||
});
|
||||
}
|
||||
|
||||
test("kube profile selection maps values files and org modes", () => {
|
||||
assert.deepEqual(kubeProfileConfig("single-org"), {
|
||||
profile: "single-org",
|
||||
orgMode: "single_org",
|
||||
valuesPath: "evals/fixtures/kube/values/single-org.yaml",
|
||||
});
|
||||
assert.deepEqual(kubeProfileConfig("multi-org"), {
|
||||
profile: "multi-org",
|
||||
orgMode: "multi_org",
|
||||
valuesPath: "evals/fixtures/kube/values/multi-org.yaml",
|
||||
});
|
||||
});
|
||||
|
||||
test("helm upgrade argv includes release, chart, profile, context, and local image overrides", () => {
|
||||
const args = helmUpgradeArgs(kubeProfileConfig("multi-org"), imagePlan("local"));
|
||||
|
||||
assert.deepEqual(args.slice(0, 7), [
|
||||
"upgrade",
|
||||
"--install",
|
||||
"openwork-ee",
|
||||
"packaging/helm/openwork-ee",
|
||||
"-f",
|
||||
"evals/fixtures/kube/values/multi-org.yaml",
|
||||
"--set",
|
||||
]);
|
||||
assert(args.includes("image.tag=kube-lab"));
|
||||
assert(args.includes("image.pullPolicy=IfNotPresent"));
|
||||
assert(args.includes("denApi.image.repository=openwork-den-api"));
|
||||
assert(args.includes("denWeb.image.repository=openwork-den-web"));
|
||||
assert(args.includes("--kube-context"));
|
||||
assert(args.includes("kind-openwork-kube-lab"));
|
||||
});
|
||||
|
||||
test("kubectl rollout and port-forward argv use the kind context", () => {
|
||||
assert.deepEqual(rolloutStatusArgs("openwork-ee-den-api", "300s"), [
|
||||
"--context",
|
||||
"kind-openwork-kube-lab",
|
||||
"rollout",
|
||||
"status",
|
||||
"deployment/openwork-ee-den-api",
|
||||
"--timeout=300s",
|
||||
]);
|
||||
assert.deepEqual(portForwardArgs("openwork-ee-den-web", 3005, 3005), [
|
||||
"--context",
|
||||
"kind-openwork-kube-lab",
|
||||
"port-forward",
|
||||
"service/openwork-ee-den-web",
|
||||
"3005:3005",
|
||||
]);
|
||||
});
|
||||
|
||||
test("published image decision uses manifest platform support", async () => {
|
||||
await withCleanImageEnv(async () => {
|
||||
const platform = currentDockerPlatform();
|
||||
assert.equal(manifestSupportsPlatform(manifestFor(platform.architecture), platform), true);
|
||||
assert.equal(manifestSupportsPlatform(manifestFor("s390x"), platform), false);
|
||||
|
||||
const { exec, calls } = createExec(() => success(JSON.stringify(manifestFor(platform.architecture))));
|
||||
const plan = await resolveKubeImagePlan({ exec });
|
||||
|
||||
assert.equal(plan.mode, "published");
|
||||
assert.equal(calls.filter((call) => call.command === "docker" && call.args.join(" ").includes("manifest inspect")).length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test("local image decision skips manifest inspection when explicitly requested", async () => {
|
||||
await withCleanImageEnv(async () => {
|
||||
const { exec, calls } = createExec(() => {
|
||||
throw new Error("manifest inspection should not run");
|
||||
});
|
||||
|
||||
const plan = await resolveKubeImagePlan({ exec, images: "local" });
|
||||
|
||||
assert.equal(plan.mode, "local");
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("explicit published image mode fails when manifests do not support this platform", async () => {
|
||||
await withCleanImageEnv(async () => {
|
||||
const { exec } = createExec(() => success(JSON.stringify(manifestFor("s390x"))));
|
||||
|
||||
await assert.rejects(
|
||||
() => resolveKubeImagePlan({ exec, images: "published" }),
|
||||
/Published Den images do not advertise/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("endpoint handles return credentials and stop only their recorded port-forwards", async () => {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), "openwork-kind-endpoints-test-"));
|
||||
const spawned: { command: string; args: string[]; pid: number }[] = [];
|
||||
const killed: number[] = [];
|
||||
const pids = [987_651, 987_652];
|
||||
const spawnDetached: KubeSpawnDetached = (command, args) => {
|
||||
const pid = pids[spawned.length];
|
||||
if (pid === undefined) throw new Error("Unexpected extra detached process.");
|
||||
spawned.push({ command, args: [...args], pid });
|
||||
return pid;
|
||||
};
|
||||
const { exec } = createExec((call) => call.command === "lsof"
|
||||
? { stdout: "", stderr: "", code: 1 }
|
||||
: success());
|
||||
const previousFetch = globalThis.fetch;
|
||||
const fetchedUrls: string[] = [];
|
||||
const fakeFetch: typeof fetch = async (input) => {
|
||||
const url = typeof input === "string"
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url;
|
||||
fetchedUrls.push(url);
|
||||
if (url.endsWith("/api/auth/sign-in/email")) {
|
||||
return new Response(JSON.stringify({ token: "kind-owner-token" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response("ok", { status: 200 });
|
||||
};
|
||||
const envBefore = {
|
||||
apiUrl: process.env.OPENWORK_EVAL_DEN_API_URL,
|
||||
webUrl: process.env.OPENWORK_EVAL_DEN_WEB_URL,
|
||||
token: process.env.OPENWORK_EVAL_DEN_TOKEN,
|
||||
multiOrg: process.env.OPENWORK_EVAL_DEN_MULTI_ORG,
|
||||
};
|
||||
globalThis.fetch = fakeFetch;
|
||||
try {
|
||||
const endpoints = await exposeEndpointHandles(kubeProfileConfig("single-org"), {
|
||||
stateDir,
|
||||
exec,
|
||||
spawnDetached,
|
||||
sleep: async () => undefined,
|
||||
killProcess: (pid) => killed.push(pid),
|
||||
});
|
||||
|
||||
assert.equal(endpoints.apiUrl, "http://127.0.0.1:8790");
|
||||
assert.equal(endpoints.webUrl, "http://127.0.0.1:3005");
|
||||
assert.equal(endpoints.token, "kind-owner-token");
|
||||
assert.equal(endpoints.adminEmail, "alex@acme.test");
|
||||
assert(fetchedUrls.includes("http://127.0.0.1:3005/api/ready"));
|
||||
assert(!fetchedUrls.includes("http://127.0.0.1:3005/api/den/health"));
|
||||
assert.deepEqual(spawned.map(({ command, pid }) => ({ command, pid })), [
|
||||
{ command: "kubectl", pid: 987_651 },
|
||||
{ command: "kubectl", pid: 987_652 },
|
||||
]);
|
||||
assert.equal(await readFile(join(stateDir, "api-port-forward.pid"), "utf8"), "987651");
|
||||
assert.equal(await readFile(join(stateDir, "web-port-forward.pid"), "utf8"), "987652");
|
||||
|
||||
await endpoints.stop();
|
||||
await endpoints.stop();
|
||||
|
||||
assert.deepEqual(killed, [-987_651, 987_651, -987_652, 987_652]);
|
||||
await assert.rejects(() => readFile(join(stateDir, "api-port-forward.pid"), "utf8"), /ENOENT/);
|
||||
await assert.rejects(() => readFile(join(stateDir, "web-port-forward.pid"), "utf8"), /ENOENT/);
|
||||
assert.deepEqual({
|
||||
apiUrl: process.env.OPENWORK_EVAL_DEN_API_URL,
|
||||
webUrl: process.env.OPENWORK_EVAL_DEN_WEB_URL,
|
||||
token: process.env.OPENWORK_EVAL_DEN_TOKEN,
|
||||
multiOrg: process.env.OPENWORK_EVAL_DEN_MULTI_ORG,
|
||||
}, envBefore);
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
await rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("endpoint acquisition failure stops both newly started port-forwards", async () => {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), "openwork-kind-endpoints-failure-test-"));
|
||||
const killed: number[] = [];
|
||||
const pids = [987_661, 987_662];
|
||||
let spawned = 0;
|
||||
const spawnDetached: KubeSpawnDetached = () => {
|
||||
const pid = pids[spawned];
|
||||
spawned += 1;
|
||||
if (pid === undefined) throw new Error("Unexpected extra detached process.");
|
||||
return pid;
|
||||
};
|
||||
const { exec } = createExec((call) => call.command === "lsof"
|
||||
? { stdout: "", stderr: "", code: 1 }
|
||||
: success());
|
||||
const previousFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (input) => {
|
||||
const url = typeof input === "string"
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url;
|
||||
return new Response(url.endsWith("/api/auth/sign-in/email") ? "no" : "ok", {
|
||||
status: url.endsWith("/api/auth/sign-in/email") ? 401 : 200,
|
||||
});
|
||||
};
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => exposeEndpointHandles(kubeProfileConfig("single-org"), {
|
||||
stateDir,
|
||||
exec,
|
||||
spawnDetached,
|
||||
sleep: async () => undefined,
|
||||
killProcess: (pid) => killed.push(pid),
|
||||
}),
|
||||
/Could not obtain a demo-owner session.*HTTP 401/,
|
||||
);
|
||||
|
||||
assert.deepEqual(killed, [-987_662, 987_662, -987_661, 987_661]);
|
||||
await assert.rejects(() => readFile(join(stateDir, "api-port-forward.pid"), "utf8"), /ENOENT/);
|
||||
await assert.rejects(() => readFile(join(stateDir, "web-port-forward.pid"), "utf8"), /ENOENT/);
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
await rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ensureKindDenReady fails fast when the shared kind cluster is absent", async () => {
|
||||
const { exec, calls } = createExec(() => success(""));
|
||||
|
||||
await assert.rejects(
|
||||
() => ensureKindDenReady({ exec }),
|
||||
/ensureKubeStack\(\{ cdpCandidates: \[\], skipApp: true \}\)/,
|
||||
);
|
||||
assert.deepEqual(calls.map((call) => `${call.command} ${call.args.join(" ")}`), ["kind get clusters"]);
|
||||
});
|
||||
|
||||
test("ensureKindDenReady verifies both Den rollouts before reading shared state", async () => {
|
||||
const { exec, calls } = createExec((call) => {
|
||||
const text = `${call.command} ${call.args.join(" ")}`;
|
||||
if (text === "kind get clusters") return success("openwork-kube-lab\n");
|
||||
if (text.includes("SHOW TABLES LIKE")) {
|
||||
return success("organization\ndesktop_connect_grant\nscim_group\ngroup_mapping_mode\n");
|
||||
}
|
||||
if (text.includes("SELECT id FROM `user`")) return success("seeded-owner-id\n");
|
||||
return success();
|
||||
});
|
||||
|
||||
await ensureKindDenReady({ exec });
|
||||
|
||||
const commands = calls.map((call) => `${call.command} ${call.args.join(" ")}`);
|
||||
const apiRollout = commands.findIndex((command) => command.includes("rollout status deployment/openwork-ee-den-api"));
|
||||
const webRollout = commands.findIndex((command) => command.includes("rollout status deployment/openwork-ee-den-web"));
|
||||
const schemaQuery = commands.findIndex((command) => command.includes("SHOW TABLES LIKE"));
|
||||
assert(apiRollout >= 0);
|
||||
assert(webRollout > apiRollout);
|
||||
assert(schemaQuery > webRollout);
|
||||
});
|
||||
|
||||
test("kubeStackDown stops port-forwards before uninstalling the release", async () => {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), "openwork-kube-stack-test-"));
|
||||
const order: string[] = [];
|
||||
const { exec } = createExec((call) => {
|
||||
order.push(`${call.command}:${call.args[0] ?? ""}`);
|
||||
return success();
|
||||
});
|
||||
try {
|
||||
await writeFile(join(stateDir, "api-port-forward.pid"), "111");
|
||||
await writeFile(join(stateDir, "web-port-forward.pid"), "222");
|
||||
await kubeStackDown({
|
||||
stateDir,
|
||||
exec,
|
||||
sleep: async () => undefined,
|
||||
killProcess: (pid) => {
|
||||
order.push(`kill:${pid}`);
|
||||
},
|
||||
});
|
||||
|
||||
const firstHelm = order.findIndex((entry) => entry === "helm:uninstall");
|
||||
assert(firstHelm > 0, `expected helm uninstall after kills, got ${order.join(", ")}`);
|
||||
assert(order.slice(0, firstHelm).some((entry) => entry === "kill:-111"));
|
||||
assert(order.slice(0, firstHelm).some((entry) => entry === "kill:111"));
|
||||
assert(order.slice(0, firstHelm).some((entry) => entry === "kill:-222"));
|
||||
assert(order.slice(0, firstHelm).some((entry) => entry === "kill:222"));
|
||||
} finally {
|
||||
await rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("rollout failure surfaces pod status and recent logs", async () => {
|
||||
const { exec } = createExec((call) => {
|
||||
const text = `${call.command} ${call.args.join(" ")}`;
|
||||
if (text.includes("helm upgrade")) return success("release upgraded");
|
||||
if (text.includes("rollout status deployment/openwork-ee-den-api")) return { stdout: "", stderr: "rollout failed", code: 1 };
|
||||
if (text.includes("get pods")) return success("pod/openwork-ee-den-api pending");
|
||||
if (text.includes("describe pods")) return success("Events: image pull backoff");
|
||||
if (text.includes("logs")) return success("pod log line: config missing");
|
||||
return success();
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => ensureRelease(kubeProfileConfig("single-org"), imagePlan("published"), { exec }),
|
||||
/pod log line: config missing/,
|
||||
);
|
||||
});
|
||||
+308
@@ -0,0 +1,308 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
import { liteLlm, liteLlmSandboxName } from "../src/litellm.ts";
|
||||
import { SkipError } from "../src/needs.ts";
|
||||
import type { DaytonaExec } from "@openwork/hosts";
|
||||
import type { LiteLlmUpstreamRequest, Place } from "../src/index.ts";
|
||||
|
||||
const MODEL_ID = "openwork-litellm-unit-model";
|
||||
const PINNED_IMAGE = "ghcr.io/berriai/litellm:v1.97.0@sha256:468c25f35f3e5ec4e414974f00deab93337b1b4d9953cabcfd3722e59415f834";
|
||||
|
||||
interface ExecCall {
|
||||
args: string[];
|
||||
opts?: { input?: string; timeoutMs?: number };
|
||||
}
|
||||
|
||||
interface FakeOptions {
|
||||
deleteFailures?: number;
|
||||
invalidHealth?: boolean;
|
||||
previewOnStderr?: boolean;
|
||||
requests?: () => LiteLlmUpstreamRequest[];
|
||||
}
|
||||
|
||||
const daytonaPlace: Place = {
|
||||
kind: "daytona",
|
||||
host() {
|
||||
return undefined;
|
||||
},
|
||||
async db() {
|
||||
throw new Error("unused test database");
|
||||
},
|
||||
async exposeMock(handle) {
|
||||
return new URL(handle.url);
|
||||
},
|
||||
denBase() {
|
||||
return { kind: "daytona", ref: "unit-test" };
|
||||
},
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function remoteScript(args: string[]): string {
|
||||
const wrapped = args[3] ?? "";
|
||||
const prefix = "bash -lc '";
|
||||
return wrapped.startsWith(prefix) && wrapped.endsWith("'")
|
||||
? wrapped.slice(prefix.length, -1)
|
||||
: "";
|
||||
}
|
||||
|
||||
function makeFake(options: FakeOptions = {}): {
|
||||
calls: ExecCall[];
|
||||
exec: DaytonaExec;
|
||||
fetchImpl: typeof fetch;
|
||||
dockerfile(): string;
|
||||
masterKey(): string;
|
||||
upstreamKey(): string;
|
||||
controlKey(): string;
|
||||
modelInfo(): Record<string, unknown>;
|
||||
} {
|
||||
const calls: ExecCall[] = [];
|
||||
const uploads = new Map<string, string>();
|
||||
let dockerfile = "";
|
||||
let masterKey = "";
|
||||
let upstreamKey = "";
|
||||
let controlKey = "";
|
||||
let modelInfo: Record<string, unknown> = {};
|
||||
let deleteAttempts = 0;
|
||||
const exec: DaytonaExec = async (args, opts) => {
|
||||
calls.push({ args: [...args], opts });
|
||||
if (args[0] === "create") {
|
||||
const dockerfileIndex = args.indexOf("--dockerfile") + 1;
|
||||
dockerfile = await readFile(args[dockerfileIndex] ?? "", "utf8");
|
||||
return { stdout: "created\n", stderr: "", code: 0 };
|
||||
}
|
||||
if (args[0] === "preview-url") {
|
||||
const port = args[args.indexOf("-p") + 1] ?? "";
|
||||
if (options.previewOnStderr) {
|
||||
return { stdout: "", stderr: "Upgrade at https://updates.example.test\n", code: 0 };
|
||||
}
|
||||
return { stdout: `Preview URL: https://port-${port}.example.test\n`, stderr: "", code: 0 };
|
||||
}
|
||||
if (args[0] === "delete") {
|
||||
deleteAttempts += 1;
|
||||
if (deleteAttempts <= (options.deleteFailures ?? 0)) {
|
||||
return { stdout: "transient deletion failure\n", stderr: "", code: 1 };
|
||||
}
|
||||
return { stdout: "deleted\n", stderr: "", code: 0 };
|
||||
}
|
||||
if (args[0] !== "exec") return { stdout: "", stderr: "", code: 0 };
|
||||
|
||||
const script = remoteScript(args);
|
||||
const chunk = /^printf %s ([A-Za-z0-9+/=]+) >> (\/tmp\/[A-Za-z0-9._/-]+\.b64)$/.exec(script);
|
||||
if (chunk?.[1] && chunk[2]) uploads.set(chunk[2], `${uploads.get(chunk[2]) ?? ""}${chunk[1]}`);
|
||||
const finalize = /base64 -d (\/tmp\/[A-Za-z0-9._/-]+\.b64) > (\/tmp\/[A-Za-z0-9._/-]+)/.exec(script);
|
||||
if (finalize?.[1] && finalize[2]) {
|
||||
const content = Buffer.from(uploads.get(finalize[1]) ?? "", "base64").toString("utf8");
|
||||
if (finalize[2].endsWith("config.json")) {
|
||||
const parsed: unknown = JSON.parse(content);
|
||||
const general = isRecord(parsed) && isRecord(parsed.general_settings) ? parsed.general_settings : {};
|
||||
const models = isRecord(parsed) && Array.isArray(parsed.model_list) ? parsed.model_list.filter(isRecord) : [];
|
||||
const params = models[0] && isRecord(models[0].litellm_params) ? models[0].litellm_params : {};
|
||||
masterKey = typeof general.master_key === "string" ? general.master_key : "";
|
||||
upstreamKey = typeof params.api_key === "string" ? params.api_key : "";
|
||||
modelInfo = models[0] && isRecord(models[0].model_info) ? models[0].model_info : {};
|
||||
}
|
||||
}
|
||||
if (script.includes("tail -80")) {
|
||||
return { stdout: `${masterKey}\n${upstreamKey}\n${controlKey}\n`, stderr: "", code: 0 };
|
||||
}
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
};
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
if (url.pathname === "/__openwork_litellm/health") {
|
||||
const authorization = new Headers(init?.headers).get("authorization") ?? "";
|
||||
controlKey = authorization.startsWith("Bearer ") ? authorization.slice("Bearer ".length) : "";
|
||||
return Response.json(options.invalidHealth ? { ok: false, sequence: 0 } : { ok: true, sequence: 3 });
|
||||
}
|
||||
if (url.pathname === "/__openwork_litellm/requests") {
|
||||
return Response.json({ sequence: 3, requests: options.requests?.() ?? [] });
|
||||
}
|
||||
if (url.pathname === "/v1/models") {
|
||||
return Response.json({ object: "list", data: [{ id: MODEL_ID }] });
|
||||
}
|
||||
return Response.json({ error: "not found" }, { status: 404 });
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
exec,
|
||||
fetchImpl,
|
||||
dockerfile: () => dockerfile,
|
||||
masterKey: () => masterKey,
|
||||
upstreamKey: () => upstreamKey,
|
||||
controlKey: () => controlKey,
|
||||
modelInfo: () => modelInfo,
|
||||
};
|
||||
}
|
||||
|
||||
test("LiteLLM Daytona sandbox names are safe and unique", () => {
|
||||
const first = liteLlmSandboxName();
|
||||
const second = liteLlmSandboxName();
|
||||
|
||||
assert.match(first, /^openwork-litellm-eval-[0-9]+-[a-z0-9]+-[0-9a-f]{8}$/);
|
||||
assert.notEqual(first, second);
|
||||
});
|
||||
|
||||
test("LiteLLM database mode skips Daytona placement before provisioning", async () => {
|
||||
const fake = makeFake();
|
||||
await assert.rejects(
|
||||
liteLlm({
|
||||
place: daytonaPlace,
|
||||
modelId: MODEL_ID,
|
||||
reply: "deterministic reply",
|
||||
database: true,
|
||||
daytonaExec: fake.exec,
|
||||
fetchImpl: fake.fetchImpl,
|
||||
}),
|
||||
(error: unknown) => error instanceof SkipError
|
||||
&& error.reason === "LiteLLM database mode currently requires docker placement",
|
||||
);
|
||||
assert.equal(fake.calls.length, 0);
|
||||
});
|
||||
|
||||
test("Daytona LiteLLM pins its image, verifies bounded uploads, exposes both ports, and uses sequence cursors", async () => {
|
||||
let upstreamFingerprint = "";
|
||||
const fake = makeFake({
|
||||
requests: () => [
|
||||
{ sequence: 1, model: "older", tokenId: "older-token", bodyText: "older" },
|
||||
{ sequence: 2, model: MODEL_ID, tokenId: upstreamFingerprint, bodyText: "probe" },
|
||||
{ sequence: 3, model: MODEL_ID, tokenId: "other-token", bodyText: "chat" },
|
||||
],
|
||||
});
|
||||
const gateway = await liteLlm({
|
||||
place: daytonaPlace,
|
||||
modelId: MODEL_ID,
|
||||
reply: "deterministic reply",
|
||||
maxInputTokens: 96_000,
|
||||
maxOutputTokens: 12_345,
|
||||
daytonaExec: fake.exec,
|
||||
fetchImpl: fake.fetchImpl,
|
||||
});
|
||||
upstreamFingerprint = gateway.tokenId(gateway.upstreamKey);
|
||||
|
||||
const create = fake.calls.find((call) => call.args[0] === "create");
|
||||
assert(create);
|
||||
assert.match(fake.dockerfile(), new RegExp(`^FROM ${PINNED_IMAGE.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, "m"));
|
||||
assert.match(fake.dockerfile(), /^ENTRYPOINT \["\/usr\/bin\/bash", "-lc"\]$/m);
|
||||
assert.match(fake.dockerfile(), /^CMD \["sleep infinity"\]$/m);
|
||||
for (const [flag, value] of [
|
||||
["--auto-stop", "60"],
|
||||
["--auto-delete", "0"],
|
||||
["--ttl", "120"],
|
||||
["--target", "us"],
|
||||
]) {
|
||||
assert.equal(create.args[create.args.indexOf(flag) + 1], value);
|
||||
}
|
||||
assert(create.args.includes("--dockerfile"));
|
||||
assert(create.args.includes("--public"));
|
||||
|
||||
const launchIndex = fake.calls.findIndex((call) => remoteScript(call.args).includes("/app/.venv/bin/litellm"));
|
||||
assert(launchIndex > 0);
|
||||
const uploadCalls = fake.calls
|
||||
.map((call, index) => ({ index, script: remoteScript(call.args) }))
|
||||
.filter(({ script }) => script.includes(".b64"));
|
||||
assert(uploadCalls.length > 4);
|
||||
assert(uploadCalls.every(({ index }) => index < launchIndex));
|
||||
const chunks = uploadCalls.flatMap(({ script }) => {
|
||||
const match = /^printf %s ([A-Za-z0-9+/=]+) >> /.exec(script);
|
||||
return match?.[1] ? [match[1]] : [];
|
||||
});
|
||||
assert(chunks.length >= 2);
|
||||
assert(chunks.every((chunk) => chunk.length <= 8 * 1_024));
|
||||
const finalizers = uploadCalls.filter(({ script }) => script.includes("actual_bytes=$(wc -c"));
|
||||
assert.equal(finalizers.length, 2);
|
||||
assert(finalizers.every(({ script }) => script.includes('test "$decode_status" -eq 0 && test "$actual_bytes"')));
|
||||
assert(finalizers.every(({ script }) => /test "\$actual_bytes" -eq [1-9][0-9]*/.test(script)));
|
||||
|
||||
const previews = fake.calls.filter((call) => call.args[0] === "preview-url");
|
||||
assert.deepEqual(
|
||||
previews.map((call) => ({
|
||||
port: call.args[call.args.indexOf("-p") + 1],
|
||||
expires: call.args[call.args.indexOf("--expires") + 1],
|
||||
})),
|
||||
[{ port: "4000", expires: "7200" }, { port: "4001", expires: "7200" }],
|
||||
);
|
||||
assert.equal(gateway.baseUrl, "https://port-4000.example.test/v1");
|
||||
assert.deepEqual(fake.modelInfo(), {
|
||||
max_input_tokens: 96_000,
|
||||
max_output_tokens: 12_345,
|
||||
supports_function_calling: true,
|
||||
supports_vision: true,
|
||||
supports_reasoning: false,
|
||||
supports_response_schema: true,
|
||||
supported_openai_params: ["temperature", "tools", "response_format"],
|
||||
});
|
||||
assert.equal(await gateway.checkpoint(), 3);
|
||||
assert.deepEqual((await gateway.upstreamRequests({ after: 1 })).map((request) => request.sequence), [2, 3]);
|
||||
assert.equal((await gateway.waitForUpstreamRequest({
|
||||
after: 1,
|
||||
model: MODEL_ID,
|
||||
key: gateway.upstreamKey,
|
||||
timeoutMs: 1_000,
|
||||
})).sequence, 2);
|
||||
|
||||
await gateway[Symbol.asyncDispose]();
|
||||
await gateway[Symbol.asyncDispose]();
|
||||
const deletions = fake.calls.filter((call) => call.args[0] === "delete");
|
||||
assert.equal(deletions.length, 1);
|
||||
assert.equal(deletions[0]?.opts?.input, "y\n");
|
||||
});
|
||||
|
||||
test("Daytona LiteLLM redacts every key and deletes its sandbox after startup failure", async () => {
|
||||
const fake = makeFake({ invalidHealth: true });
|
||||
let failure: unknown;
|
||||
try {
|
||||
await liteLlm({
|
||||
place: daytonaPlace,
|
||||
modelId: MODEL_ID,
|
||||
reply: "deterministic reply",
|
||||
daytonaExec: fake.exec,
|
||||
fetchImpl: fake.fetchImpl,
|
||||
});
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
|
||||
assert(failure instanceof Error);
|
||||
for (const secret of [fake.masterKey(), fake.upstreamKey(), fake.controlKey()]) {
|
||||
assert(secret.length > 20);
|
||||
assert(!failure.message.includes(secret));
|
||||
}
|
||||
assert.match(failure.message, /\[REDACTED\]/);
|
||||
assert.match(failure.message, /Daytona logs/);
|
||||
assert.equal(fake.calls.filter((call) => call.args[0] === "delete").length, 1);
|
||||
});
|
||||
|
||||
test("Daytona LiteLLM ignores unrelated HTTPS URLs on CLI stderr", async () => {
|
||||
const fake = makeFake({ previewOnStderr: true });
|
||||
|
||||
await assert.rejects(
|
||||
liteLlm({
|
||||
place: daytonaPlace,
|
||||
modelId: MODEL_ID,
|
||||
reply: "deterministic reply",
|
||||
daytonaExec: fake.exec,
|
||||
fetchImpl: fake.fetchImpl,
|
||||
}),
|
||||
/did not return an HTTPS URL/,
|
||||
);
|
||||
assert.equal(fake.calls.filter((call) => call.args[0] === "delete").length, 1);
|
||||
});
|
||||
|
||||
test("Daytona LiteLLM disposal retries a transient sandbox deletion failure", async () => {
|
||||
const fake = makeFake({ deleteFailures: 1 });
|
||||
const gateway = await liteLlm({
|
||||
place: daytonaPlace,
|
||||
modelId: MODEL_ID,
|
||||
reply: "deterministic reply",
|
||||
daytonaExec: fake.exec,
|
||||
fetchImpl: fake.fetchImpl,
|
||||
});
|
||||
|
||||
await assert.rejects(gateway[Symbol.asyncDispose](), /Sandbox deletion gate failed/);
|
||||
await gateway[Symbol.asyncDispose]();
|
||||
assert.equal(fake.calls.filter((call) => call.args[0] === "delete").length, 2);
|
||||
});
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { server } from "../src/den.ts";
|
||||
import { resolvePlace } from "../src/place.ts";
|
||||
|
||||
test("an attached server refuses the local-only demo seed", async () => {
|
||||
await assert.rejects(
|
||||
server({
|
||||
place: resolvePlace({}),
|
||||
reuse: { apiUrl: "https://den.example.test" },
|
||||
seedProfile: "demo-org",
|
||||
}),
|
||||
/seedProfile "demo-org" is local-only and cannot seed an attached Den/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user