源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

packages/handsfree/test/e2e/bench.html

@@ -0,0 +1,34 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>CU-BENCH</title>
+<style>
+  body { font-family: -apple-system, sans-serif; margin: 24px; }
+  .grid { display: grid; grid-template-columns: repeat(4, 160px); gap: 12px; margin: 16px 0; }
+  button { padding: 12px; font-size: 14px; }
+  input, textarea { width: 480px; font-size: 14px; padding: 8px; display: block; margin: 8px 0; }
+</style>
+</head>
+<body>
+<h1>Computer Use Benchmark</h1>
+<div class="grid" id="grid"></div>
+<input id="field" placeholder="type here" aria-label="bench-field">
+<textarea id="area" rows="3" aria-label="bench-area" placeholder="textarea here"></textarea>
+<label><input type="checkbox" id="check" aria-label="bench-check"> bench-check</label>
+<script>
+  window.__events = [];
+  const names = ["alpha","bravo","charlie","delta","echo","foxtrot","golf","hotel","india","juliett","kilo","lima"];
+  const grid = document.getElementById("grid");
+  for (const n of names) {
+    const b = document.createElement("button");
+    b.id = "btn-" + n;
+    b.textContent = "btn-" + n;
+    b.addEventListener("click", () => window.__events.push({ t: Date.now(), type: "click", id: b.id }));
+    grid.appendChild(b);
+  }
+  document.getElementById("check").addEventListener("change", (e) =>
+    window.__events.push({ t: Date.now(), type: "check", checked: e.target.checked }));
+</script>
+</body>
+</html>

packages/handsfree/test/e2e/run.mjs

@@ -0,0 +1,224 @@
+// Computer-use e2e quality benchmark.
+//
+// Measures snapshot/click/type accuracy and latency against a deterministic
+// local HTML page in Chrome, with ground truth verified independently via the
+// Chrome DevTools Protocol (so accuracy is not measured by the system under
+// test).
+//
+// Requirements (macOS only):
+// - The signed helper app with Accessibility + Screen Recording granted:
+//   run `node apps/desktop/scripts/prepare-computer-use-helper.mjs` and grant
+//   permissions once via the helper's setup GUI.
+// - Google Chrome installed. The test launches a disposable instance with a
+//   temp profile and closes it afterwards. No accessibility flags are passed;
+//   the runtime is expected to enable renderer AX itself.
+//
+// Usage: node packages/handsfree/test/e2e/run.mjs
+
+import { spawn, execSync } from "node:child_process";
+import { createReadStream, createWriteStream, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = resolve(__dirname, "../../../..");
+const HELPER_APP = process.env.OPENWORK_COMPUTER_USE_HELPER
+  ?? join(repoRoot, "apps/desktop/resources/helpers/OpenWork Computer Use.app");
+const PAGE = `file://${join(__dirname, "bench.html")}`;
+const CDP_PORT = Number(process.env.CU_BENCH_CDP_PORT ?? 9224);
+const CHROME_APP = "Google Chrome";
+
+if (process.platform !== "darwin") {
+  console.log("computer-use e2e benchmark is macOS-only; skipping");
+  process.exit(0);
+}
+
+// --- MCP bridge: launch the helper via Launch Services with FIFO stdio so
+// --- TCC permissions attribute to the helper bundle, not this shell.
+const fifoDir = mkdtempSync(join(tmpdir(), "cu-e2e-"));
+const IN = join(fifoDir, "in.fifo");
+const OUT = join(fifoDir, "out.fifo");
+execSync(`mkfifo ${IN} ${OUT}`);
+try { execSync(`pkill -f 'OpenWork Computer Use.app'`); } catch {}
+spawn("open", ["--stdin", IN, "--stdout", OUT, HELPER_APP, "--args", "mcp"], { stdio: "ignore" });
+
+const mcpOut = createReadStream(OUT, { encoding: "utf8" });
+const mcpIn = createWriteStream(IN);
+let mcpBuf = "";
+const mcpPending = new Map();
+mcpOut.on("data", (chunk) => {
+  mcpBuf += chunk;
+  let idx;
+  while ((idx = mcpBuf.indexOf("\n")) >= 0) {
+    const line = mcpBuf.slice(0, idx).trim();
+    mcpBuf = mcpBuf.slice(idx + 1);
+    if (!line) continue;
+    let msg;
+    try { msg = JSON.parse(line); } catch { continue; }
+    if (msg.id != null && mcpPending.has(msg.id)) {
+      mcpPending.get(msg.id)(msg);
+      mcpPending.delete(msg.id);
+    }
+  }
+});
+
+let rpcId = 0;
+function rpc(method, params, timeoutMs = 30000) {
+  const id = ++rpcId;
+  return new Promise((res, rej) => {
+    const t = setTimeout(() => rej(new Error(`mcp timeout: ${method}`)), timeoutMs);
+    mcpPending.set(id, (msg) => { clearTimeout(t); res(msg); });
+    mcpIn.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
+  });
+}
+
+const results = { latency: {}, accuracy: {}, paths: {} };
+async function cu(name, args = {}) {
+  const t0 = performance.now();
+  const msg = await rpc("tools/call", { name, arguments: args });
+  (results.latency[name] ??= []).push(performance.now() - t0);
+  const text = msg.result?.content?.find((c) => c.type === "text")?.text;
+  const json = JSON.parse(text ?? "{}");
+  if (json.path) (results.paths[json.path] ??= []).push(name);
+  return json;
+}
+
+// --- disposable Chrome with CDP ground truth ---
+const profileDir = mkdtempSync(join(tmpdir(), "cu-e2e-chrome-"));
+spawn("open", ["-na", CHROME_APP, "--args",
+  `--remote-debugging-port=${CDP_PORT}`, `--user-data-dir=${profileDir}`,
+  "--no-first-run", "--no-default-browser-check", "--window-size=1200,900", PAGE,
+], { stdio: "ignore" });
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function cdpTargets() {
+  for (let i = 0; i < 30; i++) {
+    try {
+      const r = await fetch(`http://127.0.0.1:${CDP_PORT}/json`);
+      return await r.json();
+    } catch { await sleep(500); }
+  }
+  throw new Error(`Chrome CDP endpoint did not come up on :${CDP_PORT}`);
+}
+
+await rpc("initialize", { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "cu-e2e", version: "0" } });
+mcpIn.write(JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) + "\n");
+
+const targets = await cdpTargets();
+const tab = targets.find((t) => t.type === "page" && t.url.includes("bench.html"));
+if (!tab) throw new Error("bench.html tab not found in disposable Chrome");
+const ws = new WebSocket(tab.webSocketDebuggerUrl);
+await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
+let wsId = 0;
+const wsPending = new Map();
+ws.onmessage = (ev) => {
+  const m = JSON.parse(ev.data);
+  if (m.id && wsPending.has(m.id)) { wsPending.get(m.id)(m); wsPending.delete(m.id); }
+};
+const evaljs = (expression) => new Promise((res, rej) => {
+  const i = ++wsId;
+  const t = setTimeout(() => rej(new Error("cdp timeout")), 15000);
+  wsPending.set(i, (m) => { clearTimeout(t); res(m.result?.result?.value); });
+  ws.send(JSON.stringify({ id: i, method: "Runtime.evaluate", params: { expression, returnByValue: true } }));
+});
+
+await sleep(2000);
+const findEl = (snap, pred) => snap.elements?.find(pred);
+
+let failures = 0;
+const record = (key, ok, total, note = "") => {
+  results.accuracy[key] = `${ok}/${total}`;
+  if (ok !== total) { failures++; if (note) console.log(note); }
+};
+
+// 1. element recall
+let snap = await cu("snapshot", { app: CHROME_APP });
+const buttons = ["btn-alpha","btn-bravo","btn-charlie","btn-delta","btn-echo","btn-foxtrot","btn-golf","btn-hotel","btn-india","btn-juliett","btn-kilo","btn-lima"];
+const expected = [...buttons, "bench-field", "bench-area", "bench-check"];
+const labels = (snap.elements ?? []).map((e) => `${e.label ?? ""} ${e.value ?? ""}`).join(" | ");
+const found = expected.filter((x) => labels.includes(x));
+record("element_recall", found.length, expected.length, `missing: ${expected.filter((x) => !found.includes(x)).join(", ")}`);
+
+// 2. click accuracy across all three addressing modes, CDP-verified
+async function clickRound(key, makeArgs) {
+  let ok = 0;
+  await evaljs("window.__events = []");
+  for (const name of buttons) {
+    const el = findEl(snap, (e) => e.role === "Button" && (e.label ?? "").includes(name));
+    if (!el) continue;
+    const before = (await evaljs("window.__events.length")) ?? 0;
+    await cu("click", { app: CHROME_APP, ...makeArgs(el) });
+    await sleep(150);
+    const last = await evaljs("window.__events[window.__events.length-1]?.id");
+    const after = await evaljs("window.__events.length");
+    if (after === before + 1 && last === name) ok++;
+    else console.log(`${key} MISS: ${name} -> ${last}`);
+  }
+  record(key, ok, buttons.length);
+}
+await clickRound("click_by_ref", (el) => ({ ref: el.ref }));
+snap = await cu("snapshot", { app: CHROME_APP });
+await clickRound("click_by_image_coord", (el) => ({ x: el.center.imageX, y: el.center.imageY }));
+await clickRound("click_by_screen_coord", (el) => ({ screen_x: el.center.screenX, screen_y: el.center.screenY }));
+
+// 3. pid targeting
+const apps = await cu("list_apps");
+const chrome = (apps.appDetails ?? []).find((a) => a.name === CHROME_APP);
+const pidSnap = chrome ? await cu("snapshot", { pid: chrome.pid }) : {};
+record("snapshot_by_pid", pidSnap.windowTitle?.includes("CU-BENCH") ? 1 : 0, 1);
+
+// 4. typing accuracy (incl. symbols), CDP-verified
+const strings = [
+  "The quick brown fox jumps over the lazy dog 0123456789",
+  "~!@#$%^&*()_+-=[]{}|;':\",./<>?",
+  "CamelCase snake_case kebab-case 42.5%",
+];
+let typeOk = 0;
+for (const text of strings) {
+  await evaljs("document.getElementById('field').value=''; document.getElementById('field').focus()");
+  await sleep(100);
+  await cu("type_text", { app: CHROME_APP, text });
+  await sleep(300);
+  if ((await evaljs("document.getElementById('field').value")) === text) typeOk++;
+}
+record("type_text", typeOk, strings.length);
+
+// 5. set_value with verify + retype fallback
+snap = await cu("snapshot", { app: CHROME_APP });
+const field = findEl(snap, (e) => (e.label ?? "").includes("bench-field"));
+await cu("set_value", { app: CHROME_APP, ref: field.ref, value: "set-value-test-42" });
+await sleep(300);
+record("set_value", (await evaljs("document.getElementById('field').value")) === "set-value-test-42" ? 1 : 0, 1);
+
+// 6. background modifier combo: cmd+a + retype
+await cu("snapshot", { app: CHROME_APP });
+await evaljs("document.getElementById('field').focus()");
+await cu("press_key", { app: CHROME_APP, combo: "command+a" });
+await sleep(100);
+await cu("type_text", { app: CHROME_APP, text: "replaced" });
+await sleep(300);
+record("select_all_replace", (await evaljs("document.getElementById('field').value")) === "replaced" ? 1 : 0, 1);
+
+// --- report ---
+console.log("\n========== RESULTS ==========");
+console.log("\n-- Accuracy --");
+for (const [k, v] of Object.entries(results.accuracy)) console.log(`${k.padEnd(24)} ${v}`);
+console.log("\n-- Latency (ms) --");
+for (const [k, v] of Object.entries(results.latency)) {
+  const s = [...v].sort((a, b) => a - b);
+  const avg = Math.round(s.reduce((a, b) => a + b, 0) / s.length);
+  console.log(`${k.padEnd(24)} n=${String(s.length).padEnd(4)} avg=${String(avg).padEnd(6)} min=${Math.round(s[0]).toString().padEnd(6)} max=${Math.round(s[s.length - 1])}`);
+}
+console.log("\n-- Execution paths --");
+for (const [k, v] of Object.entries(results.paths)) console.log(`${k.padEnd(24)} ${v.length}x`);
+
+// --- cleanup ---
+try { execSync(`pkill -f "user-data-dir=${profileDir}"`); } catch {}
+try { execSync(`pkill -f 'OpenWork Computer Use.app'`); } catch {}
+await sleep(1000);
+try { rmSync(fifoDir, { recursive: true, force: true }); } catch {}
+try { rmSync(profileDir, { recursive: true, force: true }); } catch {}
+
+console.log(failures === 0 ? "\nRESULT: PASS" : `\nRESULT: FAIL (${failures} failing checks)`);
+process.exit(failures === 0 ? 0 : 1);