源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/specs/cloud-mcp-provider-capability-submit.e2e.test.ts

@@ -0,0 +1,159 @@
+import { expect, onTestFinished } from "vitest";
+import { screenshot, validate } from "@openwork/test-evidence";
+import {
+  clickButton,
+  createOrgConnection,
+  deleteConnection,
+  deleteConnectionsNamed,
+  evalIn,
+  go,
+  openConnectionsSurface,
+  readAvailableModels,
+  readComposerState,
+  revealText,
+  selectModel,
+  waitFor,
+  waitForConnectionCard,
+  waitForText,
+  writeComposerText,
+} from "@openwork/behaviors";
+import { app, mcpMock, needs, server, test, unmetNeeds } from "@openwork/testkit";
+import type { TestNeeds } from "@openwork/testkit";
+
+const requirements: TestNeeds = {
+  model: "tool-capable",
+  env: ["OPENWORK_EVAL_DEN_API_URL"],
+  optIn: ["OPENWORK_EVAL_E2E_TESTS", "OPENWORK_EVAL_CONNECTOR_E2E_TEST"],
+};
+const missingRequirements = unmetNeeds(requirements, process.env);
+const title = missingRequirements.length > 0
+  ? `Cloud MCP provider capability submission skipped — needs: ${missingRequirements.join(", ")}`
+  : "bundled engine provider capability proof allows an organization connector task to submit";
+const modelId = process.env.OPENWORK_EVAL_MODEL?.trim() || "";
+
+test(title, async ({ evidence, place }) => {
+  needs(requirements);
+  await using den = await server({
+    place,
+    mocks: {
+      connector: mcpMock({
+        port: Number(process.env.OPENWORK_EVAL_CONNECTOR_MOCK_PORT ?? 3979),
+        publicUrl: process.env.OPENWORK_EVAL_CONNECTOR_MOCK_PUBLIC_URL?.trim() || undefined,
+      }),
+    },
+  });
+  const connector = den.mocks.connector;
+  await deleteConnectionsNamed(den.admin, "Readiness Probe ");
+  const connection = await createOrgConnection(den.admin, {
+    name: `Readiness Probe ${Date.now()}`,
+    url: connector.mcpUrl,
+    authType: "oauth",
+    credentialMode: "per_member",
+    access: { orgWide: true },
+  });
+  onTestFinished(async () => deleteConnection(den.admin, connection.id));
+
+  await using desktopApp = await app({ den, as: "admin", place });
+  await openConnectionsSurface(desktopApp, desktopApp.workspaceId);
+  await waitForConnectionCard(desktopApp, connection.name, desktopApp.workspaceId);
+  await waitFor(desktopApp, `(() => {
+    const card = [...document.querySelectorAll('button')]
+      .find((button) => (button.textContent ?? '').includes(${JSON.stringify(connection.name)}));
+    card?.click();
+    return Boolean(card);
+  })()`, { timeoutMs: 30_000, label: "opened readiness probe connection" });
+  await waitForText(desktopApp, "OAuth required", { timeoutMs: 30_000 });
+  await revealText(desktopApp, "Connect your account");
+  const connectClickedAt = new Date().toISOString();
+  await clickButton(desktopApp, "Connect your account");
+  await connector.authorizeRequestSince(connectClickedAt);
+  await waitForText(desktopApp, "Connected with your own account.", { timeoutMs: 120_000 });
+
+  await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
+  if (modelId) {
+    const models = await readAvailableModels(desktopApp);
+    expect(models.some((model) => model.id === modelId && model.selectable)).toBe(true);
+    await selectModel(desktopApp, modelId);
+  }
+
+  const marker = `provider-capability-${Date.now()}`;
+  const prompt = `Call the mock_echo tool with text exactly "${marker}" and reply with only its result.`;
+  const before = await readComposerState(desktopApp);
+  await writeComposerText(desktopApp, prompt);
+  await evalIn(desktopApp, `(() => {
+    const originalFetch = window.fetch.bind(window);
+    const probe = { originalFetch, health: null, probeRequested: false };
+    window.__cloudMcpProviderCapabilityProbe = probe;
+    window.fetch = async (input, init) => {
+      const response = await originalFetch(input, init);
+      const url = typeof input === "string" ? input : input?.url ?? String(input);
+      if (url.includes("/mcp/openwork-cloud/health")) {
+        probe.probeRequested ||= url.includes("probe=1");
+        const body = await response.clone().json().catch(() => null);
+        if (body?.tools?.providerProjection) {
+          probe.health = {
+            source: body.tools.providerProjection.source,
+            usableByCurrentModel: body.usableByCurrentModel,
+            directChecked: body.tools.direct?.checked,
+          };
+        }
+      }
+      return response;
+    };
+    return true;
+  })()`);
+  const submittedAt = new Date().toISOString();
+  await clickButton(desktopApp, "Run task");
+  await waitFor(desktopApp, `(() => {
+    const failed = document.querySelector('[data-testid="cloud-mcp-submission-failure"]');
+    const sent = document.querySelectorAll('[data-message-role="user"]').length > ${before.userMessageCount};
+    return Boolean(failed || sent);
+  })()`, { timeoutMs: 60_000, label: "Cloud MCP gate completed" });
+  const gate = await evalIn(desktopApp, `({
+    failure: document.querySelector('[data-testid="cloud-mcp-submission-failure"]')?.textContent ?? null,
+    userMessages: document.querySelectorAll('[data-message-role="user"]').length,
+  })()`);
+  const failure = typeof gate === "object" && gate !== null && "failure" in gate ? gate.failure : "unknown";
+  const userMessages = typeof gate === "object" && gate !== null && "userMessages" in gate ? gate.userMessages : 0;
+  evidence.recordAssertionEvidence(
+    "The Cloud MCP submission gate accepted the bundled engine's provider capability proof",
+    `Gate state: ${JSON.stringify(gate)}`,
+    failure === null && typeof userMessages === "number" && userMessages > before.userMessageCount,
+  );
+  expect(failure).toBeNull();
+  expect(userMessages).toBeGreaterThan(before.userMessageCount);
+
+  const projectionJson = await evalIn(desktopApp, `(() => {
+    const probe = window.__cloudMcpProviderCapabilityProbe;
+    if (probe?.originalFetch) window.fetch = probe.originalFetch;
+    delete window.__cloudMcpProviderCapabilityProbe;
+    return JSON.stringify({ health: probe?.health ?? null, probeRequested: probe?.probeRequested === true });
+  })()`);
+  const providerCapabilityObserved = typeof projectionJson === "string"
+    && projectionJson.includes('"source":"provider_capability"')
+    && projectionJson.includes('"usableByCurrentModel":true')
+    && projectionJson.includes('"directChecked":true')
+    && projectionJson.includes('"probeRequested":true');
+  evidence.recordAssertionEvidence(
+    "The first-attempt health probe combined direct tools with provider tool-call capability",
+    `Captured health proof: ${String(projectionJson)}`,
+    providerCapabilityObserved,
+  );
+  expect(providerCapabilityObserved).toBe(true);
+
+  const calls = await connector.toolCalls({ name: "mock_echo", atLeast: 1, timeoutMs: 240_000, sinceIso: submittedAt });
+  const markerReachedConnector = calls.some((call) => String(call.args.text ?? "").includes(marker));
+  evidence.recordAssertionEvidence(
+    "The submitted organization connector task reached the provider witness",
+    `Served mock_echo calls: ${JSON.stringify(calls)}`,
+    markerReachedConnector,
+  );
+  expect(markerReachedConnector).toBe(true);
+
+  const shot = await screenshot(desktopApp);
+  const seen = await validate(shot, [
+    "An OpenWork session shows a submitted task that used a connected organization tool",
+    "No connected service preparation failure or crash message is visible",
+  ]);
+  expect(seen.ok, seen.why).toBe(true);
+});