"""agent-radar 共享基础库:GitHub API 封装 + YAML 精确回写。 设计要点 -------- 1. GitHub API 优先使用 GITHUB_TOKEN,缺失时匿名降级(限额 60 次/小时)。 2. YAML 回写采用**行级精确替换**而不是 yaml.dump —— dump 会冲掉档案里的 所有注释和字段顺序,而档案是靠注释承载"为什么这么填"的。 """ from __future__ import annotations import json import os import re import time import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path import yaml ROOT = Path(__file__).resolve().parent.parent DATA = ROOT / "data" PROJECTS = DATA / "projects" SNAPSHOTS = DATA / "snapshots" ARTICLES = ROOT / "articles" SCHEMA = ROOT / "schema" GH_API = "https://api.github.com" TOKEN = os.environ.get("GITHUB_TOKEN", "").strip() _last_call = 0.0 # ---------------------------------------------------------------- GitHub API def gh_get(path: str, timeout: int = 25, retries: int = 2) -> dict | list | None: """GET GitHub API。失败返回 None(调用方负责降级)。 内置限速:匿名 1.5s 间隔、带 token 0.3s 间隔,避免触发 403 限流。 retries 是必要的:releases 这类接口响应体可达数百 KB, 偶发 IncompleteRead(连接提前关闭)。不加重试会静默丢掉版本节奏数据, 而"样本不足"和"真的没发版"在结果上无法区分。 """ global _last_call url = f"{GH_API}/{path.lstrip('/')}" req = urllib.request.Request( url, headers={ "Accept": "application/vnd.github+json", "User-Agent": "agent-radar", **({"Authorization": f"Bearer {TOKEN}"} if TOKEN else {}), }, ) last_err = None for attempt in range(retries + 1): wait = 0.3 if TOKEN else 1.5 gap = time.time() - _last_call if gap < wait: time.sleep(wait - gap) _last_call = time.time() try: with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read().decode("utf-8")) except urllib.error.HTTPError as e: # 4xx 是确定性失败,重试无意义 if e.code == 403: print(f" [warn] GitHub API 限流或禁止访问 (403): {path}") elif e.code == 404: print(f" [warn] 资源不存在 (404): {path}") else: print(f" [warn] HTTP {e.code}: {path}") return None except Exception as e: # IncompleteRead / 超时 / 连接重置 last_err = e if attempt < retries: time.sleep(1.5 * (attempt + 1)) continue print(f" [warn] 请求失败(已重试 {retries} 次): {path} -> {last_err}") return None # ---------------------------------------------------------------- YAML 读写 def load_yaml(path) -> dict: """接受 str 或 Path —— 调用方传字符串时不应该炸。""" p = Path(path) return yaml.safe_load(p.read_text(encoding="utf-8")) or {} def today() -> str: return datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d") def _fmt(v) -> str: """把 Python 值渲染成 YAML 标量文本。""" if isinstance(v, bool): return "true" if v else "false" if v is None: return '""' if isinstance(v, (int, float)): return str(v) if isinstance(v, dict): inner = ", ".join(f"{k}: {_fmt(x)}" for k, x in v.items()) return "{" + inner + "}" if isinstance(v, list): inner = ", ".join(_fmt(x) for x in v) return "[" + inner + "]" s = str(v) # 含冒号空格、井号、引号或为空的字符串需要加引号 if s == "" or re.search(r'[:#\n"]|^\s|\s$', s) or re.match(r"^[-?*&!|>%@`]", s): return json.dumps(s, ensure_ascii=False) return s def update_block(text: str, block_key: str, values: dict, indent: int | None = None) -> str: """在 YAML 文本里定位 `:` 块,替换其中的键值。 只改块内已有键的值;块内没有而 values 里有的,追加到块末尾。 块外内容(含所有注释)原样保留。 indent 传 None 时**自动探测**第一个同名块 —— 推荐用这个。 硬编码缩进是很脆的:schema 一改层级,块的实际缩进就变了, 而失败表现只是"静默没写进去",很容易被忽略。 """ lines = text.split("\n") # `(\s*)` 而不是 `(\s+)`:顶层块的缩进是 0,用 `+` 会匹配不到 # (article: 就是顶层块,这个坑踩过一次) pat = re.compile(rf"^(\s*){re.escape(block_key)}:\s*$") start = None base = None for i, ln in enumerate(lines): m = pat.match(ln) if m and (indent is None or len(m.group(1)) == indent): start, base = i, len(m.group(1)) break if start is None: raise ValueError(f"未找到 `{block_key}:` 块(indent={indent})") # 块的结束:下一个缩进 <= base 的非空、非注释行 end = len(lines) for j in range(start + 1, len(lines)): ln = lines[j] if not ln.strip() or ln.lstrip().startswith("#"): continue if len(ln) - len(ln.lstrip()) <= base: end = j break child = base + 2 seen: set[str] = set() for k in range(start + 1, end): m = re.match(rf"^(\s{{{child},}})([A-Za-z_][\w-]*):\s*(.*)$", lines[k]) if not m: continue key = m.group(2) if key in values: lines[k] = f"{' ' * child}{key}: {_fmt(values[key])}" seen.add(key) missing = [f"{' ' * child}{k}: {_fmt(v)}" for k, v in values.items() if k not in seen] if missing: lines[end:end] = missing return "\n".join(lines) # ---------------------------------------------------------------- 快照 def snapshot_path(pid: str) -> Path: return SNAPSHOTS / f"{pid}.jsonl" def append_snapshot(pid: str, metrics: dict) -> None: """追加一行快照。同一天重复运行会覆盖当天那行(保持"一天一个点")。""" SNAPSHOTS.mkdir(parents=True, exist_ok=True) p = snapshot_path(pid) day = metrics.get("data_date") or today() rows = [] if p.exists(): rows = [r for r in p.read_text(encoding="utf-8").splitlines() if r.strip()] rows = [r for r in rows if json.loads(r).get("data_date") != day] rows.append(json.dumps(metrics, ensure_ascii=False)) p.write_text("\n".join(rows) + "\n", encoding="utf-8") def last_snapshot(pid: str) -> dict | None: p = snapshot_path(pid) if not p.exists(): return None rows = [r for r in p.read_text(encoding="utf-8").splitlines() if r.strip()] return json.loads(rows[-1]) if rows else None # ---------------------------------------------------------------- 许可证分级 def classify_license(spdx: str, license_name: str = "") -> dict: """按 schema/license-tiers.yaml 判定成色分级与镜像可见性。 返回 {level, redistributable, visibility}。 无法识别时返回 L3(保守策略:宁可私有,不可误公开)。 """ tiers = load_yaml(SCHEMA / "license-tiers.yaml").get("tiers", []) key = (spdx or "").strip() for t in tiers: match = [str(x) for x in t.get("match_spdx", []) if x is not None] if key in match: return { "level": t["level"], "redistributable": bool(t.get("redistributable")), "visibility": t.get("visibility", "private"), } # 未命中:按 L3 处理(源码可得但许可不明 → 不公开) return {"level": "L3", "redistributable": False, "visibility": "private"}