源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

evals/specs/github-connector-import.e2e.test.ts

@@ -0,0 +1,417 @@
+import { generateKeyPairSync } from "node:crypto";
+import { expect } from "vitest";
+import { denFetch } from "@openwork/behaviors";
+import type { DenSession } from "@openwork/behaviors";
+import { startMockGithub } from "@openwork/labs";
+import type { MockGithubRepository } from "@openwork/labs";
+import { localMysqlIsRunning, needs, server, test } from "@openwork/testkit";
+
+/**
+ * CLAIMS:
+ *  - The GitHub repository picker returns all 130 installation repositories in
+ *    provider order across its cursor pages, including private repositories.
+ *  - Repository enumeration requests only GitHub pages 1 and 2 at 100 items per
+ *    page, and repository traffic uses the installation token rather than the
+ *    GitHub App JWT.
+ *  - Discovery recognizes the private Claude marketplace repository, its one
+ *    supported plugin, and its skill directory without warnings.
+ *  - Applying that discovery succeeds and derives the imported skill title and
+ *    description from intact YAML frontmatter.
+ *  - A companion markdown file below the skill is not imported as another skill.
+ *  - The stored skill source strictly equals the provider's full SKILL.md after
+ *    product-wide surrounding-whitespace canonicalization, with frontmatter and
+ *    body bytes intact.
+ *  - Re-applying an unchanged revision reuses the same config object, creates no
+ *    duplicate version, and does not fetch the skill file again.
+ */
+
+const localPlacement = process.env.OPENWORK_EVAL_DAYTONA !== "1"
+  && !process.env.OPENWORK_EVAL_DEN_API_URL?.trim();
+const mysqlOpen = await localMysqlIsRunning();
+const title = !localPlacement
+  ? "github connector import skipped — needs: local placement without OPENWORK_EVAL_DEN_API_URL"
+  : !mysqlOpen
+    ? "github connector import skipped — needs: MySQL on 127.0.0.1:3306"
+    : "the GitHub connector lists every installation repository and imports marketplace skills end-to-end";
+
+const REQUEST_TIMEOUT_MS = 120_000;
+const SKILL_MARKDOWN = "---\nname: probe\ndescription: Probe connector import\n---\n\nRun the probe checklist.\n";
+const SKILL_PATH = "probe-plugin/skills/probe/SKILL.md";
+const SKILL_CONTENT_REQUEST_PATH = `/repos/acme-eval/probe-plugins/contents/${SKILL_PATH}`;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function requireRecord(value: unknown, label: string): Record<string, unknown> {
+  if (!isRecord(value)) throw new Error(`${label} was not an object: ${JSON.stringify(value).slice(0, 500)}`);
+  return value;
+}
+
+function requireRecords(value: unknown, label: string): Record<string, unknown>[] {
+  if (!Array.isArray(value) || !value.every(isRecord)) {
+    throw new Error(`${label} was not an object array: ${JSON.stringify(value).slice(0, 500)}`);
+  }
+  return value.filter(isRecord);
+}
+
+function requireString(value: unknown, label: string): string {
+  if (typeof value !== "string" || !value) throw new Error(`${label} was not a non-empty string.`);
+  return value;
+}
+
+function requireNumber(value: unknown, label: string): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${label} was not a finite number.`);
+  return value;
+}
+
+function responseItem(body: unknown, label: string): Record<string, unknown> {
+  return requireRecord(requireRecord(body, `${label} response`).item, `${label} item`);
+}
+
+function installEnvironment(values: Record<string, string>): () => void {
+  const previous = new Map<string, string | undefined>();
+  for (const [name, value] of Object.entries(values)) {
+    previous.set(name, process.env[name]);
+    process.env[name] = value;
+  }
+  return () => {
+    for (const [name, value] of previous) {
+      if (value === undefined) delete process.env[name];
+      else process.env[name] = value;
+    }
+  };
+}
+
+function githubRepositories(): MockGithubRepository[] {
+  return Array.from({ length: 130 }, (_, index) => {
+    const id = index + 1;
+    return id === 42
+      ? { id, fullName: "acme-eval/probe-plugins", defaultBranch: "develop", private: true }
+      : { id, fullName: `acme-eval/repo-${id}`, defaultBranch: "main", private: false };
+  });
+}
+
+interface ApiResult {
+  body: unknown;
+  status: number;
+  text: string;
+}
+
+async function apiRequest(
+  session: DenSession,
+  path: string,
+  input: { body?: Record<string, unknown>; method?: "GET" | "POST"; orgId?: string } = {},
+): Promise<ApiResult> {
+  const headers: Record<string, string> = {
+    authorization: `Bearer ${session.token}`,
+  };
+  if (input.orgId) headers["x-openwork-org-id"] = input.orgId;
+  if (input.body) headers["content-type"] = "application/json";
+  const result = await denFetch(session, path, {
+    method: input.method ?? "GET",
+    headers,
+    body: input.body ? JSON.stringify(input.body) : undefined,
+    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+  });
+  if (!result.response.ok) {
+    throw new Error(`${input.method ?? "GET"} ${path} failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
+  }
+  return { body: result.body, status: result.response.status, text: result.text };
+}
+
+async function resolveOrganizationId(session: DenSession): Promise<string> {
+  const result = await apiRequest(session, "/v1/me/orgs");
+  const orgs = requireRecords(requireRecord(result.body, "organization list response").orgs, "organizations");
+  expect(orgs).toHaveLength(1);
+  return requireString(orgs[0]?.id, "organization id");
+}
+
+test.skipIf(!localPlacement || !mysqlOpen)(title, async ({ evidence, place }) => {
+  needs({});
+  const repositories = githubRepositories();
+  const marketplaceManifest = JSON.stringify({
+    name: "Probe marketplace",
+    description: "Private GitHub connector marketplace fixture",
+    owner: { name: "acme-eval" },
+    plugins: [{
+      name: "probe-plugin",
+      description: "Probe connector plugin",
+      source: "./probe-plugin",
+    }],
+  });
+  const pluginManifest = JSON.stringify({
+    name: "probe-plugin",
+    description: "Probe connector plugin",
+    skills: ["./skills"],
+  });
+  const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
+  const privateKeyPem = String(privateKey.export({ type: "pkcs8", format: "pem" }));
+
+  await using github = await startMockGithub({
+    installationId: 777,
+    accountLogin: "acme-eval",
+    repositories,
+    repoFiles: {
+      "acme-eval/probe-plugins": {
+        branch: "develop",
+        files: {
+          ".claude-plugin/marketplace.json": marketplaceManifest,
+          "probe-plugin/.claude-plugin/plugin.json": pluginManifest,
+          [SKILL_PATH]: SKILL_MARKDOWN,
+          "probe-plugin/skills/probe/references/notes.md": "# Notes\n\nCompanion file that must not be imported as a skill.\n",
+        },
+      },
+    },
+  });
+  const restoreEnvironment = installEnvironment({
+    GITHUB_CONNECTOR_API_BASE: github.apiUrl,
+    GITHUB_CONNECTOR_APP_ID: "123456",
+    GITHUB_CONNECTOR_APP_PRIVATE_KEY: privateKeyPem,
+  });
+
+  try {
+    await using den = await server({ place });
+    const orgId = await resolveOrganizationId(den.admin);
+    const selectedOrganization = await apiRequest(den.admin, "/v1/me/active-organization", {
+      method: "POST",
+      orgId,
+      body: { organizationId: orgId },
+    });
+    expect(selectedOrganization.status).toBe(200);
+
+    const accountResult = await apiRequest(den.admin, "/v1/connectors/github/accounts", {
+      method: "POST",
+      orgId,
+      body: {
+        installationId: 777,
+        accountLogin: "acme-eval",
+        accountType: "Organization",
+        displayName: "acme-eval",
+      },
+    });
+    expect(accountResult.status).toBe(201);
+    const connectorAccountId = requireString(responseItem(accountResult.body, "GitHub account").id, "connector account id");
+
+    const repositoryListStartedAt = new Date().toISOString();
+    const firstRepositoryPage = await apiRequest(
+      den.admin,
+      `/v1/connectors/github/accounts/${encodeURIComponent(connectorAccountId)}/repositories?limit=100`,
+      { orgId },
+    );
+    const firstRepositoryBody = requireRecord(firstRepositoryPage.body, "first repository page");
+    const firstRepositories = requireRecords(firstRepositoryBody.items, "first repository page items");
+    const nextCursor = requireString(firstRepositoryBody.nextCursor, "first repository page next cursor");
+    expect(firstRepositories).toHaveLength(100);
+
+    const secondRepositoryPage = await apiRequest(
+      den.admin,
+      `/v1/connectors/github/accounts/${encodeURIComponent(connectorAccountId)}/repositories?limit=100&cursor=${encodeURIComponent(nextCursor)}`,
+      { orgId },
+    );
+    const secondRepositoryBody = requireRecord(secondRepositoryPage.body, "second repository page");
+    const secondRepositories = requireRecords(secondRepositoryBody.items, "second repository page items");
+    expect(secondRepositories).toHaveLength(30);
+    expect(secondRepositoryBody.nextCursor).toBeNull();
+
+    const listedRepositories = [...firstRepositories, ...secondRepositories];
+    const repositoryIds = listedRepositories.map((repository) => requireNumber(repository.id, "repository id"));
+    const orderedIds = Array.from({ length: 130 }, (_, index) => index + 1);
+    const repositoriesCompleteAndOrdered = new Set(repositoryIds).size === 130
+      && repositoryIds.every((id, index) => id === orderedIds[index]);
+    expect(repositoryIds).toEqual(orderedIds);
+    expect(new Set(repositoryIds).size).toBe(130);
+    const marketplaceRepository = listedRepositories.find((repository) => repository.id === 42);
+    expect(marketplaceRepository).toMatchObject({
+      fullName: "acme-eval/probe-plugins",
+      private: true,
+      hasPluginManifest: true,
+      manifestKind: "marketplace",
+    });
+    evidence.recordAssertionEvidence(
+      "The repository picker returns all 130 installation repositories across cursor pages",
+      `The first page contained ${firstRepositories.length}, the second contained ${secondRepositories.length}, and the concatenated IDs were ${repositoryIds[0]} through ${repositoryIds.at(-1)} in provider order (regression: only the first 30 before #3524).`,
+      repositoriesCompleteAndOrdered,
+    );
+
+    const pickerRequests = (await github.requests()).filter((request) => request.at >= repositoryListStartedAt);
+    const installationRepositoryRequests = pickerRequests.filter((request) => request.path === "/installation/repositories");
+    const providerPages = installationRepositoryRequests.map((request) => {
+      const url = new URL(request.url, github.apiUrl);
+      return {
+        page: url.searchParams.get("page"),
+        perPage: url.searchParams.get("per_page"),
+      };
+    });
+    const uniqueProviderPages = [...new Set(providerPages.map((request) => request.page))];
+    const providerPaginationCorrect = providerPages.length >= 2
+      && providerPages.every((request) => request.perPage === "100" && (request.page === "1" || request.page === "2"))
+      && uniqueProviderPages.length === 2
+      && uniqueProviderPages[0] === "1"
+      && uniqueProviderPages[1] === "2";
+    expect(uniqueProviderPages).toEqual(["1", "2"]);
+    expect(providerPages.every((request) => request.perPage === "100")).toBe(true);
+    evidence.recordAssertionEvidence(
+      "GitHub repository enumeration requests provider pages 1 and 2 at 100 items per page",
+      `Provider pagination requests were ${JSON.stringify(providerPages)}; there was no unpaged request or page 3 request.`,
+      providerPaginationCorrect,
+    );
+
+    const repositoryProviderRequests = pickerRequests.filter((request) => (
+      request.path === "/installation/repositories" || request.path.startsWith("/repos/")
+    ));
+    const installationToken = "Bearer mock-installation-token-777";
+    const installationTokenIdentityCorrect = repositoryProviderRequests.length > 0
+      && repositoryProviderRequests.every((request) => request.authorization === installationToken);
+    expect(repositoryProviderRequests.length).toBeGreaterThan(0);
+    expect([...new Set(repositoryProviderRequests.map((request) => request.authorization))]).toEqual([installationToken]);
+    evidence.recordAssertionEvidence(
+      "Repository discovery uses the GitHub installation access token",
+      `${repositoryProviderRequests.length} repository and content requests carried ${installationToken}, not the app JWT.`,
+      installationTokenIdentityCorrect,
+    );
+
+    const setupResult = await apiRequest(den.admin, "/v1/connectors/github/setup", {
+      method: "POST",
+      orgId,
+      body: {
+        installationId: 777,
+        connectorAccountId,
+        connectorInstanceName: "Probe plugins",
+        repositoryId: 42,
+        repositoryFullName: "acme-eval/probe-plugins",
+        branch: "develop",
+        ref: "refs/heads/develop",
+        mappings: [],
+      },
+    });
+    expect([200, 201]).toContain(setupResult.status);
+    const setupItem = responseItem(setupResult.body, "GitHub setup");
+    const connectorInstanceId = requireString(
+      requireRecord(setupItem.connectorInstance, "setup connector instance").id,
+      "connector instance id",
+    );
+
+    const discoveryResult = await apiRequest(
+      den.admin,
+      `/v1/connector-instances/${encodeURIComponent(connectorInstanceId)}/discovery`,
+      { orgId },
+    );
+    const discoveryItem = responseItem(discoveryResult.body, "GitHub discovery");
+    const discoveredPlugins = requireRecords(discoveryItem.discoveredPlugins, "discovered plugins");
+    expect(discoveryItem.classification).toBe("claude_marketplace_repo");
+    expect(discoveredPlugins).toHaveLength(1);
+    const discoveredPlugin = discoveredPlugins[0];
+    expect(discoveredPlugin).toMatchObject({ supported: true, warnings: [] });
+    expect(requireRecord(discoveredPlugin?.componentPaths, "discovered plugin component paths").skills).toEqual([
+      "probe-plugin/skills",
+    ]);
+    expect(discoveryItem.warnings).toEqual([]);
+    const discoverySupportedWithoutWarnings = discoveryItem.classification === "claude_marketplace_repo"
+      && discoveredPlugins.length === 1
+      && discoveredPlugin?.supported === true
+      && Array.isArray(discoveredPlugin.warnings)
+      && discoveredPlugin.warnings.length === 0
+      && Array.isArray(discoveryItem.warnings)
+      && discoveryItem.warnings.length === 0;
+    evidence.recordAssertionEvidence(
+      "Discovery classifies the private marketplace repository with one supported plugin and no warnings",
+      `Classification was ${String(discoveryItem.classification)}; plugin support was ${String(discoveredPlugin?.supported)}; discovery and plugin warnings were empty.`,
+      discoverySupportedWithoutWarnings,
+    );
+
+    const selectedKey = requireString(discoveredPlugin?.key, "discovered plugin key");
+    const applyBody = { autoImportNewPlugins: false, selectedKeys: [selectedKey] };
+    const applyStartedAt = new Date().toISOString();
+    const firstApplyResult = await apiRequest(
+      den.admin,
+      `/v1/connector-instances/${encodeURIComponent(connectorInstanceId)}/discovery/apply`,
+      { method: "POST", orgId, body: applyBody },
+    );
+    expect(firstApplyResult.status).toBe(200);
+    expect(requireRecord(firstApplyResult.body, "first apply response").ok).toBe(true);
+    const firstApplyItem = responseItem(firstApplyResult.body, "first discovery apply");
+    requireRecord(firstApplyItem.createdMarketplace, "created marketplace");
+    expect(requireRecords(firstApplyItem.createdPlugins, "created plugins")).toHaveLength(1);
+    const firstMaterializedObjects = requireRecords(firstApplyItem.materializedConfigObjects, "materialized config objects");
+    expect(firstMaterializedObjects).toHaveLength(1);
+    const materializedSkill = firstMaterializedObjects[0];
+    expect(materializedSkill).toMatchObject({
+      objectType: "skill",
+      title: "probe",
+      description: "Probe connector import",
+      currentRelativePath: SKILL_PATH,
+    });
+    const configObjectId = requireString(materializedSkill?.id, "materialized config object id");
+    const frontmatterProjectionCorrect = materializedSkill?.objectType === "skill"
+      && materializedSkill.title === "probe"
+      && materializedSkill.description === "Probe connector import"
+      && materializedSkill.currentRelativePath === SKILL_PATH;
+    evidence.recordAssertionEvidence(
+      "Applying the marketplace import materializes the frontmatter-derived probe skill",
+      `Apply returned HTTP ${firstApplyResult.status}; the only materialized object was ${String(materializedSkill?.currentRelativePath)} titled ${String(materializedSkill?.title)} (regression: invalid_skill_frontmatter before #3523).`,
+      firstApplyResult.status === 200 && frontmatterProjectionCorrect,
+    );
+    evidence.recordAssertionEvidence(
+      "The companion markdown next to the skill is not imported",
+      `The repository tree included references/notes.md, while apply materialized ${firstMaterializedObjects.length} config object at ${String(materializedSkill?.currentRelativePath)}.`,
+      firstMaterializedObjects.length === 1 && materializedSkill?.currentRelativePath === SKILL_PATH,
+    );
+
+    const firstVersionsResult = await apiRequest(
+      den.admin,
+      `/v1/config-objects/${encodeURIComponent(configObjectId)}/versions`,
+      { orgId },
+    );
+    const firstVersions = requireRecords(requireRecord(firstVersionsResult.body, "first version list").items, "first versions");
+    expect(firstVersions).toHaveLength(1);
+    const storedSource = requireString(firstVersions[0]?.rawSourceText, "stored skill source");
+    expect(storedSource).toBe(SKILL_MARKDOWN.trim());
+    expect(storedSource.startsWith("---\nname: probe\ndescription: Probe connector import\n---")).toBe(true);
+    expect(storedSource.endsWith("Run the probe checklist.")).toBe(true);
+    const sourceMatchesCanonicalForm = storedSource === SKILL_MARKDOWN.trim();
+    evidence.recordAssertionEvidence(
+      "The imported SKILL.md keeps frontmatter and body byte-for-byte after canonical whitespace trimming",
+      "The stored version strictly equaled the provider source after surrounding-whitespace canonicalization; normalizeOptionalString trims every config-object version, including manual creates.",
+      firstVersions.length === 1 && sourceMatchesCanonicalForm,
+    );
+
+    const secondApplyResult = await apiRequest(
+      den.admin,
+      `/v1/connector-instances/${encodeURIComponent(connectorInstanceId)}/discovery/apply`,
+      { method: "POST", orgId, body: applyBody },
+    );
+    expect(secondApplyResult.status).toBe(200);
+    expect(requireRecord(secondApplyResult.body, "second apply response").ok).toBe(true);
+    const secondApplyItem = responseItem(secondApplyResult.body, "second discovery apply");
+    const secondMaterializedObjects = requireRecords(secondApplyItem.materializedConfigObjects, "second materialized config objects");
+    expect(secondMaterializedObjects).toHaveLength(1);
+    expect(secondMaterializedObjects[0]?.id).toBe(configObjectId);
+
+    const secondVersionsResult = await apiRequest(
+      den.admin,
+      `/v1/config-objects/${encodeURIComponent(configObjectId)}/versions`,
+      { orgId },
+    );
+    const secondVersions = requireRecords(requireRecord(secondVersionsResult.body, "second version list").items, "second versions");
+    expect(secondVersions).toHaveLength(1);
+    expect(secondVersions[0]?.rawSourceText).toBe(SKILL_MARKDOWN.trim());
+    const skillContentRequests = (await github.requests()).filter((request) => (
+      request.at >= applyStartedAt
+      && request.method === "GET"
+      && request.path === SKILL_CONTENT_REQUEST_PATH
+    ));
+    expect(skillContentRequests).toHaveLength(1);
+    const reapplyIsIdempotent = secondMaterializedObjects.length === 1
+      && secondMaterializedObjects[0]?.id === configObjectId
+      && secondVersions.length === 1
+      && skillContentRequests.length === 1;
+    evidence.recordAssertionEvidence(
+      "Re-applying the same revision creates no duplicate version and refetches no file content",
+      `Both applies returned config object ${configObjectId}; versions remained ${secondVersions.length}; ${SKILL_PATH} was fetched ${skillContentRequests.length} time since ${applyStartedAt}.`,
+      reapplyIsIdempotent,
+    );
+  } finally {
+    restoreEnvironment();
+  }
+});