agent-radar/scripts/fetch_stats.py

#!/usr/bin/env python3
"""抓取被追踪项目的 GitHub 指标，回写档案并累积时间序列快照。

用法
----
    python3 scripts/fetch_stats.py                 # 刷新 registry.yaml 里的全部项目
    python3 scripts/fetch_stats.py --id ipollowork # 只刷新一个
    python3 scripts/fetch_stats.py --force         # 忽略当日缓存强制重抓

行为
----
· 优先读环境变量 GITHUB_TOKEN；没有则匿名（限额 60 次/小时），限速间隔自动放宽。
· 同一项目一天只抓一次（快照文件里已有当天记录就跳过），--force 可覆盖。
· 抓取失败时沿用上次快照数值，并把 metrics.stale 置为 true，
  这样文章里的数字始终有出处，且能被识别出"这是旧数据"。
"""

from __future__ import annotations

import argparse
import re
import sys
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _common import (  # noqa: E402
    DATA,
    PROJECTS,
    append_snapshot,
    classify_license,
    gh_get,
    last_snapshot,
    load_yaml,
    today,
    update_block,
    upsert_block,
)


def human_size(kb: int) -> str:
    mb = kb / 1024
    return f"{mb:.0f} MB" if mb >= 1 else f"{kb} KB"


def calc_cadence(releases: list) -> str:
    """从最近若干 release 的发布时间间隔估算版本节奏。"""
    dates = []
    for r in releases:
        d = (r.get("published_at") or "")[:10]
        if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
            dates.append(datetime.strptime(d, "%Y-%m-%d"))
    if len(dates) < 2:
        return "样本不足"
    dates.sort(reverse=True)
    spans = [(dates[i] - dates[i + 1]).days for i in range(len(dates) - 1)]
    spans = [s for s in spans if s >= 0]
    if not spans:
        return "样本不足"
    avg = sum(spans) / len(spans)
    if avg < 1:
        return "一天多版"
    if avg < 2:
        return "约 1 天/版本"
    if avg < 8:
        return f"约 {avg:.0f} 天/版本"
    if avg < 40:
        return f"约 {avg / 7:.1f} 周/版本"
    return f"约 {avg / 30:.1f} 月/版本"


def fetch_one(pid: str, repo: str, force: bool) -> bool:
    print(f"\n[{pid}] {repo}")

    # ---- 当日缓存 ----
    if not force:
        prev = last_snapshot(pid)
        if prev and prev.get("data_date") == today():
            print(f"    今日已抓取过（{today()}），跳过。用 --force 强制刷新。")
            return True

    # ---- 主仓库信息 ----
    info = gh_get(f"repos/{repo}")
    if not info:
        print("    主信息抓取失败，尝试沿用上次快照 …")
        return fallback(pid)

    lic = info.get("license") or {}
    spdx = lic.get("spdx_id") or "NOASSERTION"
    if spdx == "NOASSERTION" and not lic.get("key"):
        spdx = ""

    langs_raw = gh_get(f"repos/{repo}/languages") or {}
    total = sum(langs_raw.values()) or 1
    langs = {k: round(v / total * 100, 1) for k, v in sorted(langs_raw.items(), key=lambda x: -x[1])[:6]}

    rels = gh_get(f"repos/{repo}/releases?per_page=8") or []
    latest = ""
    if rels:
        latest = rels[0].get("tag_name") or ""

    metrics = {
        "stars": info.get("stargazers_count", 0),
        "forks": info.get("forks_count", 0),
        "open_issues": info.get("open_issues_count", 0),
        "subscribers": info.get("subscribers_count", 0),
        "created_at": (info.get("created_at") or "")[:10],
        "pushed_at": (info.get("pushed_at") or "")[:10],
        "size_kb": info.get("size", 0),
        "languages": langs,
        "latest_release": latest,
        "release_cadence": calc_cadence(rels),
        "data_date": today(),
        "stale": False,
    }

    print(f"    ★ {metrics['stars']}  fork {metrics['forks']}  issues {metrics['open_issues']}")
    print(f"    最新 {latest}  节奏 {metrics['release_cadence']}  体积 {human_size(metrics['size_kb'])}")
    print(f"    语言 {langs}")
    print(f"    许可证 spdx={spdx or '(无)'}")

    # ---- 快照 + 回写 ----
    append_snapshot(pid, metrics)

    ypath = PROJECTS / f"{pid}.yaml"
    if ypath.exists():
        text = ypath.read_text(encoding="utf-8")
        try:
            # 用 upsert 而非 update：手工补完档案时若整块覆盖了 ecosystem，
            # metrics 子块会消失，upsert 能把它重建回来。
            text = upsert_block(text, "metrics", metrics, parent="ecosystem")
        except ValueError as e:
            print(f"    [warn] 回写 metrics 失败：{e}")
        # 许可证若尚未定级，顺手补上
        if "openness: ~" in text or "openness: null" in text or 'openness: ""' in text:
            cls = classify_license(spdx, lic.get("name", ""))
            text = update_block(text, "license", {
                "spdx": spdx or "NOASSERTION",
                "name": lic.get("name") or "自定义许可",
                "openness": cls["level"],
                "redistributable": cls["redistributable"],
            })
            print(f"    自动定级：{cls['level']} → 镜像可见性 {cls['visibility']}")
        ypath.write_text(text, encoding="utf-8")
    else:
        print(f"    [warn] 档案不存在：{ypath}，跳过回写")

    return True


def fallback(pid: str) -> bool:
    """抓取失败：沿用上次快照并把 stale 标为 true。"""
    prev = last_snapshot(pid)
    ypath = PROJECTS / f"{pid}.yaml"
    if not prev or not ypath.exists():
        print("    无历史快照可用，本次放弃。")
        return False
    prev = dict(prev)
    prev["stale"] = True
    text = ypath.read_text(encoding="utf-8")
    try:
        text = update_block(text, "metrics", prev)
        ypath.write_text(text, encoding="utf-8")
        print(f"    已沿用 {prev.get('data_date')} 的快照并标记 stale=true")
    except ValueError as e:
        print(f"    [warn] 回写失败：{e}")
        return False
    return True


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--id", help="只刷新指定项目 id")
    ap.add_argument("--force", action="store_true", help="忽略当日缓存")
    args = ap.parse_args()

    reg = load_yaml(DATA / "registry.yaml")
    items = reg.get("projects", [])
    if not items:
        print("registry.yaml 里没有登记任何项目。")
        return 1

    targets = [p for p in items if not args.id or p.get("id") == args.id]
    if not targets:
        print(f"未找到 id={args.id}")
        return 1

    ok = sum(fetch_one(p["id"], p["repo"], args.force) for p in targets)
    print(f"\n完成 {ok}/{len(targets)}")
    return 0 if ok == len(targets) else 1


if __name__ == "__main__":
    raise SystemExit(main())