源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/specs/automation-model-needs-attention.e2e.test.ts

@@ -0,0 +1,349 @@
+import { expect } from "vitest";
+import { screenshot, validate } from "@openwork/test-evidence";
+import {
+  clickButton,
+  denFetch,
+  evalIn,
+  go,
+  visibleText,
+  waitFor,
+  waitForText,
+} from "@openwork/behaviors";
+import type { DenSession } from "@openwork/behaviors";
+import type { Surface } from "@openwork/cdp";
+import { app, needs, server, test } from "@openwork/testkit";
+
+/**
+ * CORE JOURNEY: an active Automation keeps its instructions, cadence, and
+ * durable history when its pinned model disappears. Den records the first due
+ * occurrence as skipped, pauses future dispatches in `needs_attention`, and
+ * the app points the owner directly to a supported-model picker. Choosing a
+ * replacement creates a new immutable revision, clears the warning, and
+ * resumes from the next scheduled occurrence without replaying the skipped
+ * one.
+ */
+
+const REQUEST_TIMEOUT_MS = 10_000;
+const SCHEDULE_TIMEOUT_MS = 120_000;
+const LEGACY_PROVIDER_NAME = "Legacy Automation Models";
+const LEGACY_MODEL_ID = "legacy-automation-model";
+const REPLACEMENT_PROVIDER_NAME = "Replacement Automation Models";
+const REPLACEMENT_MODEL_ID = "replacement-automation-model";
+const REPLACEMENT_MODEL_NAME = "Replacement Automation Model";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function auth(session: DenSession): Record<string, string> {
+  return { authorization: `Bearer ${session.token}` };
+}
+
+async function activeOrganizationId(session: DenSession): Promise<string> {
+  const result = await denFetch(session, "/v1/me/orgs", {
+    headers: auth(session),
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  const organizations = isRecord(result.body) && Array.isArray(result.body.orgs)
+    ? result.body.orgs.filter(isRecord)
+    : [];
+  const activeId = isRecord(result.body) && typeof result.body.activeOrgId === "string"
+    ? result.body.activeOrgId
+    : "";
+  const active = organizations.find((organization) => organization.isActive === true) ?? organizations[0];
+  const orgId = activeId || (active && typeof active.id === "string" ? active.id : "");
+  if (!result.response.ok || !orgId) {
+    throw new Error(`Finding the active organization failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
+  }
+  return orgId;
+}
+
+async function createProvider(input: {
+  admin: DenSession;
+  orgId: string;
+  providerKey: string;
+  providerName: string;
+  modelId: string;
+  modelName: string;
+}): Promise<string> {
+  const result = await denFetch(input.admin, "/v1/llm-providers", {
+    method: "POST",
+    headers: { ...auth(input.admin), "x-openwork-org-id": input.orgId },
+    body: JSON.stringify({
+      name: input.providerName,
+      source: "custom",
+      customConfig: {
+        id: input.providerKey,
+        name: input.providerName,
+        npm: "@ai-sdk/openai-compatible",
+        env: [`${input.providerKey.toUpperCase().replaceAll("-", "_")}_API_KEY`],
+        models: [{ id: input.modelId, name: input.modelName }],
+      },
+      apiKey: "sk-openwork-automation-transition-eval-only",
+      allMembers: true,
+    }),
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  const provider = isRecord(result.body) && isRecord(result.body.llmProvider)
+    ? result.body.llmProvider
+    : null;
+  const providerId = provider && typeof provider.id === "string" ? provider.id : "";
+  if (result.response.status !== 201 || !providerId) {
+    throw new Error(`Creating ${input.providerName} failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
+  }
+  return providerId;
+}
+
+async function createDueAutomation(input: {
+  admin: DenSession;
+  orgId: string;
+  providerId: string;
+  name: string;
+  instructions: string;
+}): Promise<string> {
+  const nextMinute = new Date(Date.now() + 90_000);
+  const result = await denFetch(input.admin, "/v1/automations", {
+    method: "POST",
+    headers: { ...auth(input.admin), "x-openwork-org-id": input.orgId },
+    body: JSON.stringify({
+      name: input.name,
+      instructions: input.instructions,
+      schedule: {
+        kind: "daily",
+        timezone: "UTC",
+        hour: nextMinute.getUTCHours(),
+        minute: nextMinute.getUTCMinutes(),
+      },
+      model: { providerId: input.providerId, modelId: LEGACY_MODEL_ID, variant: null },
+    }),
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  const automation = isRecord(result.body) && isRecord(result.body.automation)
+    ? result.body.automation
+    : null;
+  const automationId = automation && typeof automation.id === "string" ? automation.id : "";
+  if (result.response.status !== 201 || !automationId) {
+    throw new Error(`Creating the Automation failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
+  }
+  const nextDueAt = automation && typeof automation.nextDueAt === "number" ? automation.nextDueAt : 0;
+  if (nextDueAt <= Date.now() || nextDueAt > Date.now() + SCHEDULE_TIMEOUT_MS) {
+    throw new Error(`The Automation was not scheduled within the test window: ${result.text.slice(0, 1_000)}`);
+  }
+  return automationId;
+}
+
+async function deleteProvider(admin: DenSession, orgId: string, providerId: string): Promise<void> {
+  const result = await denFetch(admin, `/v1/llm-providers/${encodeURIComponent(providerId)}`, {
+    method: "DELETE",
+    headers: { ...auth(admin), "x-openwork-org-id": orgId },
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  expect(result.response.status).toBe(204);
+}
+
+async function skippedModelReceipt(admin: DenSession, orgId: string, automationId: string): Promise<Record<string, unknown>> {
+  const deadline = Date.now() + SCHEDULE_TIMEOUT_MS;
+  let lastResponse = "not requested";
+  while (Date.now() < deadline) {
+    const result = await denFetch(admin, `/v1/automations/${encodeURIComponent(automationId)}/runs`, {
+      headers: { ...auth(admin), "x-openwork-org-id": orgId },
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+    });
+    lastResponse = `HTTP ${result.response.status} ${result.text.slice(0, 1_000)}`;
+    const items = isRecord(result.body) && Array.isArray(result.body.items)
+      ? result.body.items.filter(isRecord)
+      : [];
+    const skipped = items.find((item) =>
+      item.status === "skipped"
+      && isRecord(item.error)
+      && (item.error.code === "provider_unavailable" || item.error.code === "model_access_lost")
+    );
+    if (skipped) return skipped;
+    await new Promise((resolve) => setTimeout(resolve, 1_000));
+  }
+  throw new Error(`The due occurrence was not recorded as skipped after its model disappeared. Last response: ${lastResponse}`);
+}
+
+async function clickAutomationCard(surface: Surface, name: string): Promise<void> {
+  const clicked = await evalIn(surface, `(() => {
+    const card = [...document.querySelectorAll('button')]
+      .find((candidate) => (candidate.textContent ?? '').includes(${JSON.stringify(name)}));
+    if (!(card instanceof HTMLButtonElement)) return false;
+    card.click();
+    return true;
+  })()`);
+  expect(clicked).toBe(true);
+}
+
+test("an unavailable Automation model needs attention until the owner selects a supported replacement", async ({ evidence, place }) => {
+  needs({ optIn: ["OPENWORK_EVAL_E2E_TESTS", "OPENWORK_EVAL_AUTOMATIONS_E2E_TEST"] });
+
+  await using den = await server({ place });
+  const orgId = await activeOrganizationId(den.admin);
+  const stamp = Date.now();
+  const automationName = `CRM transition check ${stamp}`;
+  const instructions = `Keep the CRM transition marker ${stamp} in every summary.`;
+  const legacyProviderId = await createProvider({
+    admin: den.admin,
+    orgId,
+    providerKey: `legacy-automation-${stamp}`,
+    providerName: LEGACY_PROVIDER_NAME,
+    modelId: LEGACY_MODEL_ID,
+    modelName: "Legacy Automation Model",
+  });
+  await createProvider({
+    admin: den.admin,
+    orgId,
+    providerKey: `replacement-automation-${stamp}`,
+    providerName: REPLACEMENT_PROVIDER_NAME,
+    modelId: REPLACEMENT_MODEL_ID,
+    modelName: REPLACEMENT_MODEL_NAME,
+  });
+  const automationId = await createDueAutomation({
+    admin: den.admin,
+    orgId,
+    providerId: legacyProviderId,
+    name: automationName,
+    instructions,
+  });
+
+  // The model disappears after the revision is durable but before dispatch.
+  await deleteProvider(den.admin, orgId, legacyProviderId);
+  const skippedReceipt = await skippedModelReceipt(den.admin, orgId, automationId);
+  expect(skippedReceipt.trigger).toBe("scheduled");
+  evidence.recordAssertionEvidence(
+    "The first unavailable-model occurrence is retained once",
+    "Den recorded one scheduled receipt as skipped before pausing the Automation.",
+    true,
+  );
+
+  await using desktop = await app({ den, as: "admin", place });
+  await go(desktop, "/automations");
+  await waitForText(desktop, automationName, { timeoutMs: 60_000 });
+  await waitFor(desktop, `(() => {
+    const card = document.querySelector('[data-automation-needs-attention]');
+    return Boolean(card && (card.textContent ?? '').includes(${JSON.stringify(automationName)}));
+  })()`, { timeoutMs: 60_000, label: "Automation card warning" });
+  expect(await evalIn(desktop, "Boolean(document.querySelector('[data-automations-attention-indicator]'))")).toBe(true);
+  expect(await visibleText(desktop)).toContain("Needs attention");
+  evidence.recordAssertionEvidence(
+    "The Automations list calls out the broken model",
+    "The sidebar and affected card have warning icons, and the card reports Needs attention while healthy cards remain unchanged.",
+    true,
+  );
+
+  await clickAutomationCard(desktop, automationName);
+  await waitForText(desktop, "Model needs attention", { timeoutMs: 30_000 });
+  const pausedDetail = await visibleText(desktop);
+  expect(pausedDetail).toContain(instructions);
+  expect(pausedDetail).toContain(`${legacyProviderId}/${LEGACY_MODEL_ID}`);
+  expect(pausedDetail).toContain("No future run scheduled");
+  expect(pausedDetail).toContain("Skipped — model unavailable");
+  const runNowDisabled = await evalIn(desktop, `(() => {
+    const button = [...document.querySelectorAll('button')]
+      .find((candidate) => (candidate.textContent ?? '').trim() === 'Run now');
+    return button instanceof HTMLButtonElement && button.disabled;
+  })()`);
+  expect(runNowDisabled).toBe(true);
+  evidence.recordAssertionEvidence(
+    "The detail page preserves context and pauses execution",
+    "Instructions, schedule, old model identity, and skipped history remain visible; Run now and future scheduling are paused.",
+    true,
+  );
+
+  await clickButton(desktop, "Select a supported model");
+  await waitForText(desktop, "Current model is no longer available", { timeoutMs: 30_000 });
+  await waitFor(desktop, `(() => {
+    const dialog = document.querySelector('[role=dialog]');
+    return Boolean(dialog && (dialog.textContent ?? '').includes('Replacement Automation Model'));
+  })()`, { timeoutMs: 30_000, label: "supported replacement model picker" });
+  const pickerText = await evalIn(desktop, `document.querySelector('[role=dialog]')?.textContent ?? ''`);
+  expect(String(pickerText)).toContain("Replacement Automation Model");
+  expect(String(pickerText)).not.toContain("Legacy Automation Model");
+  // Provider groups start collapsed unless they hold the current model, a
+  // cloud-sourced group, or OpenWork models — none apply here, so the model
+  // rows are not in the DOM yet. The shipped interaction is: expand the
+  // provider group via its header button, then click the model row (a
+  // role="button" div whose label span is the model's display name).
+  const expandedReplacementGroup = await evalIn(desktop, `(() => {
+    const dialog = document.querySelector('[role=dialog]');
+    if (!dialog) return false;
+    const header = [...dialog.querySelectorAll('button')]
+      .find((candidate) => [...candidate.querySelectorAll('span')]
+        .some((label) => (label.textContent ?? '').trim() === ${JSON.stringify(REPLACEMENT_PROVIDER_NAME)}));
+    if (!(header instanceof HTMLElement)) return false;
+    header.click();
+    return true;
+  })()`);
+  expect(expandedReplacementGroup).toBe(true);
+  await waitFor(desktop, `(() => {
+    const dialog = document.querySelector('[role=dialog]');
+    return Boolean(dialog && [...dialog.querySelectorAll('span')]
+      .some((label) => (label.textContent ?? '').trim() === ${JSON.stringify(REPLACEMENT_MODEL_NAME)}));
+  })()`, { timeoutMs: 15_000, label: "replacement model row rendered" });
+  // The expanded row is a native button whose label spans carry the model's
+  // display name and raw id (the plural group header never does). Click the
+  // candidate with an exact label match, and surface the visible spans when
+  // nothing is clickable so a red run explains itself.
+  const selectedReplacement = await evalIn(desktop, `(() => {
+    const dialogs = [...document.querySelectorAll('[role=dialog]')];
+    if (dialogs.length === 0) return "no dialog";
+    for (const dialog of dialogs.slice().reverse()) {
+      const items = [...dialog.querySelectorAll('button, [role=button], [role=option], [cmdk-item]')];
+      const item = items.find((candidate) => [...candidate.querySelectorAll('span')]
+        .some((label) => {
+          const text = (label.textContent ?? '').trim();
+          return text === ${JSON.stringify(REPLACEMENT_MODEL_NAME)} || text === ${JSON.stringify(REPLACEMENT_MODEL_ID)};
+        }));
+      if (item instanceof HTMLElement) {
+        item.click();
+        return "ok";
+      }
+    }
+    const spans = [...dialogs[dialogs.length - 1].querySelectorAll('span')]
+      .map((label) => (label.textContent ?? '').trim())
+      .filter((text) => text.length > 0)
+      .slice(0, 40);
+    return "no clickable row; dialogs=" + dialogs.length + " spans=" + JSON.stringify(spans);
+  })()`);
+  expect(selectedReplacement).toBe("ok");
+  await clickButton(desktop, "Save changes");
+
+  await waitForText(desktop, "Active", { timeoutMs: 60_000 });
+  await waitForText(desktop, "Revision 2", { timeoutMs: 30_000 });
+  const repairedDetail = await visibleText(desktop);
+  expect(repairedDetail).toContain(`${REPLACEMENT_PROVIDER_NAME} · Replacement Automation Model`);
+  expect(repairedDetail).not.toContain("Model needs attention");
+  expect(repairedDetail).not.toContain("No future run scheduled");
+  expect(repairedDetail).toContain("Skipped — model unavailable");
+  const finalRunNowDisabled = await evalIn(desktop, `(() => {
+    const button = [...document.querySelectorAll('button')]
+      .find((candidate) => (candidate.textContent ?? '').trim() === 'Run now');
+    return button instanceof HTMLButtonElement ? button.disabled : null;
+  })()`);
+  expect(finalRunNowDisabled).toBe(false);
+  await waitFor(desktop, "!document.querySelector('[data-automations-attention-indicator]')", {
+    timeoutMs: 30_000,
+    label: "sidebar warning clears after model replacement",
+  });
+
+  const runs = await denFetch(den.admin, `/v1/automations/${encodeURIComponent(automationId)}/runs`, {
+    headers: { ...auth(den.admin), "x-openwork-org-id": orgId },
+  });
+  const runItems = isRecord(runs.body) && Array.isArray(runs.body.items) ? runs.body.items : [];
+  expect(runItems).toHaveLength(1);
+  evidence.recordAssertionEvidence(
+    "Explicit replacement resumes without replay",
+    "Saving the supported model created revision 2, restored Active with a next run, retained the skipped receipt, and did not replay it.",
+    true,
+  );
+
+  const shot = await screenshot(desktop);
+  const seen = await validate(shot, [
+    "The repaired Automation is Active and shows revision 2",
+    "The replacement model is visible in Desktop execution",
+    "The earlier skipped run remains visible in Run history",
+    "No model-needs-attention warning or error stack is visible",
+  ]);
+  expect(seen.ok, seen.why).toBe(true);
+});