Publish agent-radar via gitea-publish skill

2026-09-01 15:14:54 +08:00
commit 78b87e8b2f
17 changed files with 2279 additions and 0 deletions
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""抓取被追踪项目的 GitHub 指标，回写档案并累积时间序列快照。
+
+用法
+----
+    python3 scripts/fetch_stats.py                 # 刷新 registry.yaml 里的全部项目
+    python3 scripts/fetch_stats.py --id ipollowork # 只刷新一个
+    python3 scripts/fetch_stats.py --force         # 忽略当日缓存强制重抓
+
+行为
+----
+· 优先读环境变量 GITHUB_TOKEN；没有则匿名（限额 60 次/小时），限速间隔自动放宽。
+· 同一项目一天只抓一次（快照文件里已有当天记录就跳过），--force 可覆盖。
+· 抓取失败时沿用上次快照数值，并把 metrics.stale 置为 true，
+  这样文章里的数字始终有出处，且能被识别出"这是旧数据"。
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _common import (  # noqa: E402
+    DATA,
+    PROJECTS,
+    append_snapshot,
+    classify_license,
+    gh_get,
+    last_snapshot,
+    load_yaml,
+    today,
+    update_block,
+)
+
+
+def human_size(kb: int) -> str:
+    mb = kb / 1024
+    return f"{mb:.0f} MB" if mb >= 1 else f"{kb} KB"
+
+
+def calc_cadence(releases: list) -> str:
+    """从最近若干 release 的发布时间间隔估算版本节奏。"""
+    dates = []
+    for r in releases:
+        d = (r.get("published_at") or "")[:10]
+        if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
+            dates.append(datetime.strptime(d, "%Y-%m-%d"))
+    if len(dates) < 2:
+        return "样本不足"
+    dates.sort(reverse=True)
+    spans = [(dates[i] - dates[i + 1]).days for i in range(len(dates) - 1)]
+    spans = [s for s in spans if s >= 0]
+    if not spans:
+        return "样本不足"
+    avg = sum(spans) / len(spans)
+    if avg < 1:
+        return "一天多版"
+    if avg < 2:
+        return "约 1 天/版本"
+    if avg < 8:
+        return f"约 {avg:.0f} 天/版本"
+    if avg < 40:
+        return f"约 {avg / 7:.1f} 周/版本"
+    return f"约 {avg / 30:.1f} 月/版本"
+
+
+def fetch_one(pid: str, repo: str, force: bool) -> bool:
+    print(f"\n[{pid}] {repo}")
+
+    # ---- 当日缓存 ----
+    if not force:
+        prev = last_snapshot(pid)
+        if prev and prev.get("data_date") == today():
+            print(f"    今日已抓取过（{today()}），跳过。用 --force 强制刷新。")
+            return True
+
+    # ---- 主仓库信息 ----
+    info = gh_get(f"repos/{repo}")
+    if not info:
+        print("    主信息抓取失败，尝试沿用上次快照 …")
+        return fallback(pid)
+
+    lic = info.get("license") or {}
+    spdx = lic.get("spdx_id") or "NOASSERTION"
+    if spdx == "NOASSERTION" and not lic.get("key"):
+        spdx = ""
+
+    langs_raw = gh_get(f"repos/{repo}/languages") or {}
+    total = sum(langs_raw.values()) or 1
+    langs = {k: round(v / total * 100, 1) for k, v in sorted(langs_raw.items(), key=lambda x: -x[1])[:6]}
+
+    rels = gh_get(f"repos/{repo}/releases?per_page=8") or []
+    latest = ""
+    if rels:
+        latest = rels[0].get("tag_name") or ""
+
+    metrics = {
+        "stars": info.get("stargazers_count", 0),
+        "forks": info.get("forks_count", 0),
+        "open_issues": info.get("open_issues_count", 0),
+        "subscribers": info.get("subscribers_count", 0),
+        "created_at": (info.get("created_at") or "")[:10],
+        "pushed_at": (info.get("pushed_at") or "")[:10],
+        "size_kb": info.get("size", 0),
+        "languages": langs,
+        "latest_release": latest,
+        "release_cadence": calc_cadence(rels),
+        "data_date": today(),
+        "stale": False,
+    }
+
+    print(f"    ★ {metrics['stars']}  fork {metrics['forks']}  issues {metrics['open_issues']}")
+    print(f"    最新 {latest}  节奏 {metrics['release_cadence']}  体积 {human_size(metrics['size_kb'])}")
+    print(f"    语言 {langs}")
+    print(f"    许可证 spdx={spdx or '(无)'}")
+
+    # ---- 快照 + 回写 ----
+    append_snapshot(pid, metrics)
+
+    ypath = PROJECTS / f"{pid}.yaml"
+    if ypath.exists():
+        text = ypath.read_text(encoding="utf-8")
+        try:
+            text = update_block(text, "metrics", metrics)
+        except ValueError as e:
+            print(f"    [warn] 回写 metrics 失败：{e}")
+        # 许可证若尚未定级，顺手补上
+        if "openness: ~" in text or "openness: null" in text or 'openness: ""' in text:
+            cls = classify_license(spdx, lic.get("name", ""))
+            text = update_block(text, "license", {
+                "spdx": spdx or "NOASSERTION",
+                "name": lic.get("name") or "自定义许可",
+                "openness": cls["level"],
+                "redistributable": cls["redistributable"],
+            })
+            print(f"    自动定级：{cls['level']} → 镜像可见性 {cls['visibility']}")
+        ypath.write_text(text, encoding="utf-8")
+    else:
+        print(f"    [warn] 档案不存在：{ypath}，跳过回写")
+
+    return True
+
+
+def fallback(pid: str) -> bool:
+    """抓取失败：沿用上次快照并把 stale 标为 true。"""
+    prev = last_snapshot(pid)
+    ypath = PROJECTS / f"{pid}.yaml"
+    if not prev or not ypath.exists():
+        print("    无历史快照可用，本次放弃。")
+        return False
+    prev = dict(prev)
+    prev["stale"] = True
+    text = ypath.read_text(encoding="utf-8")
+    try:
+        text = update_block(text, "metrics", prev)
+        ypath.write_text(text, encoding="utf-8")
+        print(f"    已沿用 {prev.get('data_date')} 的快照并标记 stale=true")
+    except ValueError as e:
+        print(f"    [warn] 回写失败：{e}")
+        return False
+    return True
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--id", help="只刷新指定项目 id")
+    ap.add_argument("--force", action="store_true", help="忽略当日缓存")
+    args = ap.parse_args()
+
+    reg = load_yaml(DATA / "registry.yaml")
+    items = reg.get("projects", [])
+    if not items:
+        print("registry.yaml 里没有登记任何项目。")
+        return 1
+
+    targets = [p for p in items if not args.id or p.get("id") == args.id]
+    if not targets:
+        print(f"未找到 id={args.id}")
+        return 1
+
+    ok = sum(fetch_one(p["id"], p["repo"], args.force) for p in targets)
+    print(f"\n完成 {ok}/{len(targets)}")
+    return 0 if ok == len(targets) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())