192 lines
6.3 KiB
Python
Executable File
192 lines
6.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""抓取被追踪项目的 GitHub 指标，回写档案并累积时间序列快照。
|
||
|
||
用法
|
||
----
|
||
python3 scripts/fetch_stats.py # 刷新 registry.yaml 里的全部项目
|
||
python3 scripts/fetch_stats.py --id ipollowork # 只刷新一个
|
||
python3 scripts/fetch_stats.py --force # 忽略当日缓存强制重抓
|
||
|
||
行为
|
||
----
|
||
· 优先读环境变量 GITHUB_TOKEN；没有则匿名（限额 60 次/小时），限速间隔自动放宽。
|
||
· 同一项目一天只抓一次（快照文件里已有当天记录就跳过），--force 可覆盖。
|
||
· 抓取失败时沿用上次快照数值，并把 metrics.stale 置为 true，
|
||
这样文章里的数字始终有出处，且能被识别出"这是旧数据"。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import re
|
||
import sys
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
from _common import ( # noqa: E402
|
||
DATA,
|
||
PROJECTS,
|
||
append_snapshot,
|
||
classify_license,
|
||
gh_get,
|
||
last_snapshot,
|
||
load_yaml,
|
||
today,
|
||
update_block,
|
||
)
|
||
|
||
|
||
def human_size(kb: int) -> str:
|
||
mb = kb / 1024
|
||
return f"{mb:.0f} MB" if mb >= 1 else f"{kb} KB"
|
||
|
||
|
||
def calc_cadence(releases: list) -> str:
|
||
"""从最近若干 release 的发布时间间隔估算版本节奏。"""
|
||
dates = []
|
||
for r in releases:
|
||
d = (r.get("published_at") or "")[:10]
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
|
||
dates.append(datetime.strptime(d, "%Y-%m-%d"))
|
||
if len(dates) < 2:
|
||
return "样本不足"
|
||
dates.sort(reverse=True)
|
||
spans = [(dates[i] - dates[i + 1]).days for i in range(len(dates) - 1)]
|
||
spans = [s for s in spans if s >= 0]
|
||
if not spans:
|
||
return "样本不足"
|
||
avg = sum(spans) / len(spans)
|
||
if avg < 1:
|
||
return "一天多版"
|
||
if avg < 2:
|
||
return "约 1 天/版本"
|
||
if avg < 8:
|
||
return f"约 {avg:.0f} 天/版本"
|
||
if avg < 40:
|
||
return f"约 {avg / 7:.1f} 周/版本"
|
||
return f"约 {avg / 30:.1f} 月/版本"
|
||
|
||
|
||
def fetch_one(pid: str, repo: str, force: bool) -> bool:
|
||
print(f"\n[{pid}] {repo}")
|
||
|
||
# ---- 当日缓存 ----
|
||
if not force:
|
||
prev = last_snapshot(pid)
|
||
if prev and prev.get("data_date") == today():
|
||
print(f" 今日已抓取过（{today()}），跳过。用 --force 强制刷新。")
|
||
return True
|
||
|
||
# ---- 主仓库信息 ----
|
||
info = gh_get(f"repos/{repo}")
|
||
if not info:
|
||
print(" 主信息抓取失败，尝试沿用上次快照 …")
|
||
return fallback(pid)
|
||
|
||
lic = info.get("license") or {}
|
||
spdx = lic.get("spdx_id") or "NOASSERTION"
|
||
if spdx == "NOASSERTION" and not lic.get("key"):
|
||
spdx = ""
|
||
|
||
langs_raw = gh_get(f"repos/{repo}/languages") or {}
|
||
total = sum(langs_raw.values()) or 1
|
||
langs = {k: round(v / total * 100, 1) for k, v in sorted(langs_raw.items(), key=lambda x: -x[1])[:6]}
|
||
|
||
rels = gh_get(f"repos/{repo}/releases?per_page=8") or []
|
||
latest = ""
|
||
if rels:
|
||
latest = rels[0].get("tag_name") or ""
|
||
|
||
metrics = {
|
||
"stars": info.get("stargazers_count", 0),
|
||
"forks": info.get("forks_count", 0),
|
||
"open_issues": info.get("open_issues_count", 0),
|
||
"subscribers": info.get("subscribers_count", 0),
|
||
"created_at": (info.get("created_at") or "")[:10],
|
||
"pushed_at": (info.get("pushed_at") or "")[:10],
|
||
"size_kb": info.get("size", 0),
|
||
"languages": langs,
|
||
"latest_release": latest,
|
||
"release_cadence": calc_cadence(rels),
|
||
"data_date": today(),
|
||
"stale": False,
|
||
}
|
||
|
||
print(f" ★ {metrics['stars']} fork {metrics['forks']} issues {metrics['open_issues']}")
|
||
print(f" 最新 {latest} 节奏 {metrics['release_cadence']} 体积 {human_size(metrics['size_kb'])}")
|
||
print(f" 语言 {langs}")
|
||
print(f" 许可证 spdx={spdx or '(无)'}")
|
||
|
||
# ---- 快照 + 回写 ----
|
||
append_snapshot(pid, metrics)
|
||
|
||
ypath = PROJECTS / f"{pid}.yaml"
|
||
if ypath.exists():
|
||
text = ypath.read_text(encoding="utf-8")
|
||
try:
|
||
text = update_block(text, "metrics", metrics)
|
||
except ValueError as e:
|
||
print(f" [warn] 回写 metrics 失败：{e}")
|
||
# 许可证若尚未定级，顺手补上
|
||
if "openness: ~" in text or "openness: null" in text or 'openness: ""' in text:
|
||
cls = classify_license(spdx, lic.get("name", ""))
|
||
text = update_block(text, "license", {
|
||
"spdx": spdx or "NOASSERTION",
|
||
"name": lic.get("name") or "自定义许可",
|
||
"openness": cls["level"],
|
||
"redistributable": cls["redistributable"],
|
||
})
|
||
print(f" 自动定级：{cls['level']} → 镜像可见性 {cls['visibility']}")
|
||
ypath.write_text(text, encoding="utf-8")
|
||
else:
|
||
print(f" [warn] 档案不存在：{ypath}，跳过回写")
|
||
|
||
return True
|
||
|
||
|
||
def fallback(pid: str) -> bool:
|
||
"""抓取失败：沿用上次快照并把 stale 标为 true。"""
|
||
prev = last_snapshot(pid)
|
||
ypath = PROJECTS / f"{pid}.yaml"
|
||
if not prev or not ypath.exists():
|
||
print(" 无历史快照可用，本次放弃。")
|
||
return False
|
||
prev = dict(prev)
|
||
prev["stale"] = True
|
||
text = ypath.read_text(encoding="utf-8")
|
||
try:
|
||
text = update_block(text, "metrics", prev)
|
||
ypath.write_text(text, encoding="utf-8")
|
||
print(f" 已沿用 {prev.get('data_date')} 的快照并标记 stale=true")
|
||
except ValueError as e:
|
||
print(f" [warn] 回写失败：{e}")
|
||
return False
|
||
return True
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--id", help="只刷新指定项目 id")
|
||
ap.add_argument("--force", action="store_true", help="忽略当日缓存")
|
||
args = ap.parse_args()
|
||
|
||
reg = load_yaml(DATA / "registry.yaml")
|
||
items = reg.get("projects", [])
|
||
if not items:
|
||
print("registry.yaml 里没有登记任何项目。")
|
||
return 1
|
||
|
||
targets = [p for p in items if not args.id or p.get("id") == args.id]
|
||
if not targets:
|
||
print(f"未找到 id={args.id}")
|
||
return 1
|
||
|
||
ok = sum(fetch_one(p["id"], p["repo"], args.force) for p in targets)
|
||
print(f"\n完成 {ok}/{len(targets)}")
|
||
return 0 if ok == len(targets) else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|