Publish agent-radar via gitea-publish skill

This commit is contained in:
gitea-publish
2026-09-01 15:14:54 +08:00
commit 78b87e8b2f
17 changed files with 2279 additions and 0 deletions
+215
View File
@@ -0,0 +1,215 @@
"""agent-radar 共享基础库:GitHub API 封装 + YAML 精确回写。
设计要点
--------
1. GitHub API 优先使用 GITHUB_TOKEN,缺失时匿名降级(限额 60 次/小时)。
2. YAML 回写采用**行级精确替换**而不是 yaml.dump —— dump 会冲掉档案里的
所有注释和字段顺序,而档案是靠注释承载"为什么这么填"的。
"""
from __future__ import annotations
import json
import os
import re
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parent.parent
DATA = ROOT / "data"
PROJECTS = DATA / "projects"
SNAPSHOTS = DATA / "snapshots"
ARTICLES = ROOT / "articles"
SCHEMA = ROOT / "schema"
GH_API = "https://api.github.com"
TOKEN = os.environ.get("GITHUB_TOKEN", "").strip()
_last_call = 0.0
# ---------------------------------------------------------------- GitHub API
def gh_get(path: str, timeout: int = 25, retries: int = 2) -> dict | list | None:
"""GET GitHub API。失败返回 None(调用方负责降级)。
内置限速:匿名 1.5s 间隔、带 token 0.3s 间隔,避免触发 403 限流。
retries 是必要的:releases 这类接口响应体可达数百 KB,
偶发 IncompleteRead(连接提前关闭)。不加重试会静默丢掉版本节奏数据,
而"样本不足"和"真的没发版"在结果上无法区分。
"""
global _last_call
url = f"{GH_API}/{path.lstrip('/')}"
req = urllib.request.Request(
url,
headers={
"Accept": "application/vnd.github+json",
"User-Agent": "agent-radar",
**({"Authorization": f"Bearer {TOKEN}"} if TOKEN else {}),
},
)
last_err = None
for attempt in range(retries + 1):
wait = 0.3 if TOKEN else 1.5
gap = time.time() - _last_call
if gap < wait:
time.sleep(wait - gap)
_last_call = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
# 4xx 是确定性失败,重试无意义
if e.code == 403:
print(f" [warn] GitHub API 限流或禁止访问 (403): {path}")
elif e.code == 404:
print(f" [warn] 资源不存在 (404): {path}")
else:
print(f" [warn] HTTP {e.code}: {path}")
return None
except Exception as e: # IncompleteRead / 超时 / 连接重置
last_err = e
if attempt < retries:
time.sleep(1.5 * (attempt + 1))
continue
print(f" [warn] 请求失败(已重试 {retries} 次): {path} -> {last_err}")
return None
# ---------------------------------------------------------------- YAML 读写
def load_yaml(path) -> dict:
"""接受 str 或 Path —— 调用方传字符串时不应该炸。"""
p = Path(path)
return yaml.safe_load(p.read_text(encoding="utf-8")) or {}
def today() -> str:
return datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d")
def _fmt(v) -> str:
"""把 Python 值渲染成 YAML 标量文本。"""
if isinstance(v, bool):
return "true" if v else "false"
if v is None:
return '""'
if isinstance(v, (int, float)):
return str(v)
if isinstance(v, dict):
inner = ", ".join(f"{k}: {_fmt(x)}" for k, x in v.items())
return "{" + inner + "}"
if isinstance(v, list):
inner = ", ".join(_fmt(x) for x in v)
return "[" + inner + "]"
s = str(v)
# 含冒号空格、井号、引号或为空的字符串需要加引号
if s == "" or re.search(r'[:#\n"]|^\s|\s$', s) or re.match(r"^[-?*&!|>%@`]", s):
return json.dumps(s, ensure_ascii=False)
return s
def update_block(text: str, block_key: str, values: dict, indent: int | None = None) -> str:
"""在 YAML 文本里定位 `<block_key>:` 块,替换其中的键值。
只改块内已有键的值;块内没有而 values 里有的,追加到块末尾。
块外内容(含所有注释)原样保留。
indent 传 None 时**自动探测**第一个同名块 —— 推荐用这个。
硬编码缩进是很脆的:schema 一改层级,块的实际缩进就变了,
而失败表现只是"静默没写进去",很容易被忽略。
"""
lines = text.split("\n")
# `(\s*)` 而不是 `(\s+)`:顶层块的缩进是 0,用 `+` 会匹配不到
# (article: 就是顶层块,这个坑踩过一次)
pat = re.compile(rf"^(\s*){re.escape(block_key)}:\s*$")
start = None
base = None
for i, ln in enumerate(lines):
m = pat.match(ln)
if m and (indent is None or len(m.group(1)) == indent):
start, base = i, len(m.group(1))
break
if start is None:
raise ValueError(f"未找到 `{block_key}:` 块(indent={indent})")
# 块的结束:下一个缩进 <= base 的非空、非注释行
end = len(lines)
for j in range(start + 1, len(lines)):
ln = lines[j]
if not ln.strip() or ln.lstrip().startswith("#"):
continue
if len(ln) - len(ln.lstrip()) <= base:
end = j
break
child = base + 2
seen: set[str] = set()
for k in range(start + 1, end):
m = re.match(rf"^(\s{{{child},}})([A-Za-z_][\w-]*):\s*(.*)$", lines[k])
if not m:
continue
key = m.group(2)
if key in values:
lines[k] = f"{' ' * child}{key}: {_fmt(values[key])}"
seen.add(key)
missing = [f"{' ' * child}{k}: {_fmt(v)}" for k, v in values.items() if k not in seen]
if missing:
lines[end:end] = missing
return "\n".join(lines)
# ---------------------------------------------------------------- 快照
def snapshot_path(pid: str) -> Path:
return SNAPSHOTS / f"{pid}.jsonl"
def append_snapshot(pid: str, metrics: dict) -> None:
"""追加一行快照。同一天重复运行会覆盖当天那行(保持"一天一个点")。"""
SNAPSHOTS.mkdir(parents=True, exist_ok=True)
p = snapshot_path(pid)
day = metrics.get("data_date") or today()
rows = []
if p.exists():
rows = [r for r in p.read_text(encoding="utf-8").splitlines() if r.strip()]
rows = [r for r in rows if json.loads(r).get("data_date") != day]
rows.append(json.dumps(metrics, ensure_ascii=False))
p.write_text("\n".join(rows) + "\n", encoding="utf-8")
def last_snapshot(pid: str) -> dict | None:
p = snapshot_path(pid)
if not p.exists():
return None
rows = [r for r in p.read_text(encoding="utf-8").splitlines() if r.strip()]
return json.loads(rows[-1]) if rows else None
# ---------------------------------------------------------------- 许可证分级
def classify_license(spdx: str, license_name: str = "") -> dict:
"""按 schema/license-tiers.yaml 判定成色分级与镜像可见性。
返回 {level, redistributable, visibility}。
无法识别时返回 L3(保守策略:宁可私有,不可误公开)。
"""
tiers = load_yaml(SCHEMA / "license-tiers.yaml").get("tiers", [])
key = (spdx or "").strip()
for t in tiers:
match = [str(x) for x in t.get("match_spdx", []) if x is not None]
if key in match:
return {
"level": t["level"],
"redistributable": bool(t.get("redistributable")),
"visibility": t.get("visibility", "private"),
}
# 未命中:按 L3 处理(源码可得但许可不明 → 不公开)
return {"level": "L3", "redistributable": False, "visibility": "private"}
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
"""从所有项目档案生成横向对比矩阵与赛道图,回填到总纲文。
用法
----
python3 scripts/build_matrix.py
python3 scripts/build_matrix.py --stdout # 只打印,不写文件
输出的两张图
------------
视图 A:形态纵深(x:单点工具 → 统一工作台)× 引擎立场(y:绑定单引擎 → 多引擎中立)
视图 B:本地优先(x:强云依赖 → 完全本地)× 交付物纵深(y:只出代码 → 多模态可编辑)
阈值:项目数 <4 时**不出象限图**,改出赛道分层图。
原因:象限图的意义来自点的相对分布,只有一两个点时它传达的信息量
还不如一张分层图,却会让读者误以为已经建立了坐标系。
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _common import ARTICLES, PROJECTS, load_yaml # noqa: E402
QUAD_THRESHOLD = 4
BEGIN, END = "<!-- MATRIX:BEGIN -->", "<!-- MATRIX:END -->"
def cell(v, empty="-") -> str:
if v is None or v == "" or v == [] or v == {}:
return empty
if isinstance(v, bool):
return "是" if v else "否"
if isinstance(v, list):
return "、".join(str(x) for x in v) if v else empty
if isinstance(v, dict):
return "、".join(f"{k} {x}" for k, x in v.items()) if v else empty
return str(v).replace("|", "\\|").replace("\n", " ")
def load_projects() -> list[dict]:
out = []
for p in sorted(PROJECTS.glob("*.yaml")):
d = load_yaml(p)
if d:
d["_id"] = p.stem
out.append(d)
return out
# ---------------------------------------------------------------- 矩阵
def build_tables(items: list[dict]) -> str:
L = []
add = L.append
add("### 业务维度\n")
add("| 项目 | 定位 | 目标用户 | 变现 | 许可证 | 成色 |")
add("| --- | --- | --- | --- | --- | --- |")
for d in items:
b = d.get("business") or {}
lic = b.get("license") or {}
add(f"| **{d.get('name','')}** | {cell(b.get('positioning'))} | "
f"{cell(b.get('target_users'))} | {cell(b.get('monetization'))} | "
f"{cell(lic.get('name'))} | {cell(lic.get('openness'))} |")
add("\n### 场景维度\n")
add("| 项目 | 交互形态 | 人机协作 | 交付物 |")
add("| --- | --- | --- | --- |")
for d in items:
s = d.get("scene") or d.get("scenario") or {}
add(f"| **{d.get('name','')}** | {cell(s.get('interaction_mode'))} | "
f"{cell(s.get('human_in_the_loop'))} | {cell(s.get('deliverables'))} |")
add("\n### 技术维度\n")
add("| 项目 | 形态 | 引擎抽象 | 接入深度 | 模型绑定 | 本地优先 |")
add("| --- | --- | --- | --- | --- | --- |")
for d in items:
t = d.get("tech") or {}
ea = t.get("engine_abstraction") or {}
add(f"| **{d.get('name','')}** | {cell(t.get('form_factor'))} | "
f"{cell(ea.get('abstraction_layer'))} | {cell(ea.get('depth_notes'))} | "
f"{cell(t.get('model_binding'))} | {cell(t.get('local_first'))} |")
add("\n### 生态与风险\n")
add("| 项目 | Star | Fork | Issues | 最新 release | 节奏 | 成熟度 | 数据日期 |")
add("| --- | --- | --- | --- | --- | --- | --- | --- |")
for d in items:
m = (d.get("ecosystem") or {}).get("metrics") or {}
stale = " ⚠" if m.get("stale") else ""
add(f"| **{d.get('name','')}** | {cell(m.get('stars'))} | {cell(m.get('forks'))} | "
f"{cell(m.get('open_issues'))} | {cell(m.get('latest_release'))} | "
f"{cell(m.get('release_cadence'))} | {cell((d.get('risk') or {}).get('maturity'))} | "
f"{cell(m.get('data_date'))}{stale} |")
# 宣称 vs 实测:全套矩阵里信息密度最高的一张表
rows = []
for d in items:
for c in (d.get("risk") or {}).get("claim_vs_reality") or []:
if isinstance(c, dict):
rows.append((d.get("name", ""), c))
if rows:
add("\n### 宣称 vs 实测\n")
add("| 项目 | 宣称 | 实际 | 证据来源 |")
add("| --- | --- | --- | --- |")
for name, c in rows:
add(f"| **{name}** | {cell(c.get('claim'))} | {cell(c.get('actual'))} | "
f"{cell(c.get('source'))} |")
add("\n### 一句话结论\n")
add("| 项目 | 结论 | 适合 | 不适合 |")
add("| --- | --- | --- | --- |")
for d in items:
v = d.get("verdict") or {}
add(f"| **{d.get('name','')}** | {cell(v.get('one_liner'))} | "
f"{cell(v.get('who_should_use'))} | {cell(v.get('who_should_not'))} |")
return "\n".join(L)
# ---------------------------------------------------------------- 图
def stack_diagram(items: list[dict]) -> str:
"""项目 <4 个时的赛道分层图:按交互形态把项目归到三层栈。"""
layers = [
("执行引擎层", "真正跑 agent loop 的地方", ["cli"]),
("编辑器内 Agent 层", "嵌在 IDE / 编辑器里的交互层", ["ide-plugin", "editor-fork"]),
("桌面工作台层", "跨引擎、跨项目、跨交付物的统一界面", ["desktop-app", "web-app"]),
]
L = ["```text"]
L.append("┌─────────────────────────────────────────────────────────┐")
for name, desc, modes in layers:
hits = []
for d in items:
modes_ = (d.get("scenario") or {}).get("interaction_mode") or []
if any(m in modes_ for m in modes):
hits.append(f"{d.get('name')}★{(d.get('ecosystem',{}).get('metrics',{}) or {}).get('stars','-')}")
body = "、".join(hits) if hits else "(暂无收录)"
L.append(f"│ {name:<12} {body}")
L.append(f"│{'':<14}└ {desc}")
if name != layers[-1][0]:
L.append("│" + " " * 56 + "│")
L.append("└─────────────────────────────────────────────────────────┘")
L.append("```")
L.append(f"\n> 当前收录 {len(items)} 个项目,不足 {QUAD_THRESHOLD} 个,此阶段只出分层图。"
f"样本够了之后会自动切换为象限图。")
return "\n".join(L)
def quadrant_svg(items: list[dict]) -> str:
"""项目 >=4 个时的双视图象限图。"""
W, H, PAD = 720, 420, 70
plots = [
("视图 A · 形态纵深 × 引擎立场", "form_depth", "engine_neutral",
"单点工具", "统一工作台", "绑定单引擎", "多引擎中立"),
("视图 B · 本地优先 × 交付物纵深", "local_first_score", "deliverable_depth",
"强云依赖", "完全本地", "只出代码", "多模态可编辑"),
]
svg = []
for i, (title, xk, yk, xlo, xhi, ylo, yhi) in enumerate(plots):
ox = i * (W + 30)
svg.append(f'<svg viewBox="0 0 {W} {H}" width="{W}" height="{H}" '
f'xmlns="http://www.w3.org/2000/svg" style="font-family:sans-serif">')
svg.append(f'<text x="{W//2}" y="26" text-anchor="middle" font-size="15" '
f'font-weight="600" fill="#1f2328">{title}</text>')
px, py, pw, ph = PAD, 55, W - PAD * 2, H - PAD - 55
svg.append(f'<rect x="{px}" y="{py}" width="{pw}" height="{ph}" fill="#fafbfc" '
f'stroke="#d8dee4"/>')
# 网格
for k in range(1, 4):
gx = px + pw * k / 4
gy = py + ph * k / 4
svg.append(f'<line x1="{gx}" y1="{py}" x2="{gx}" y2="{py+ph}" stroke="#eaeef2"/>')
svg.append(f'<line x1="{px}" y1="{gy}" x2="{px+pw}" y2="{gy}" stroke="#eaeef2"/>')
# 轴标签
svg.append(f'<text x="{px}" y="{py+ph+22}" font-size="11" fill="#6a737d">{xlo}</text>')
svg.append(f'<text x="{px+pw}" y="{py+ph+22}" text-anchor="end" font-size="11" fill="#6a737d">{xhi}</text>')
svg.append(f'<text x="{px-8}" y="{py+ph+6}" text-anchor="end" font-size="11" fill="#6a737d">{ylo}</text>')
svg.append(f'<text x="{px-8}" y="{py+12}" text-anchor="end" font-size="11" fill="#6a737d">{yhi}</text>')
# 点
for d in items:
c = d.get("chart") or {}
xv, yv = c.get(xk), c.get(yk)
if xv is None or yv is None:
continue
cx = px + pw * max(0, min(10, float(xv))) / 10
cy = py + ph * (1 - max(0, min(10, float(yv))) / 10)
name = d.get("name", "")
svg.append(f'<circle cx="{cx:.1f}" cy="{cy:.1f}" r="7" fill="#0969da" '
f'fill-opacity="0.75" stroke="#fff" stroke-width="1.5"/>')
svg.append(f'<text x="{cx+11:.1f}" y="{cy+4:.1f}" font-size="12" fill="#1f2328">{name}</text>')
svg.append("</svg>")
return "\n".join(svg)
# ---------------------------------------------------------------- 主流程
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--stdout", action="store_true", help="只打印不写文件")
args = ap.parse_args()
items = load_projects()
if not items:
print("data/projects/ 下没有任何档案。")
return 1
print(f"==> 载入 {len(items)} 个项目:{', '.join(d.get('name','') for d in items)}")
matrix = build_tables(items)
chart = quadrant_svg(items) if len(items) >= QUAD_THRESHOLD else stack_diagram(items)
block = f"{BEGIN}\n{matrix}\n\n{chart}\n{END}"
if args.stdout:
print(block)
return 0
path = ARTICLES / f"{'00-overview'}.md"
if not path.exists():
print(f"总纲文不存在:{path}")
return 1
text = path.read_text(encoding="utf-8")
if BEGIN not in text or END not in text:
print(f"总纲文缺少 {BEGIN} / {END} 占位标记,未回填。")
return 1
text = re.sub(rf"{re.escape(BEGIN)}.*?{re.escape(END)}", lambda _: block, text, flags=re.S)
path.write_text(text, encoding="utf-8")
print(f"==> 已回填总纲文:{path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""抓取被追踪项目的 GitHub 指标,回写档案并累积时间序列快照。
用法
----
python3 scripts/fetch_stats.py # 刷新 registry.yaml 里的全部项目
python3 scripts/fetch_stats.py --id ipollowork # 只刷新一个
python3 scripts/fetch_stats.py --force # 忽略当日缓存强制重抓
行为
----
· 优先读环境变量 GITHUB_TOKEN;没有则匿名(限额 60 次/小时),限速间隔自动放宽。
· 同一项目一天只抓一次(快照文件里已有当天记录就跳过),--force 可覆盖。
· 抓取失败时沿用上次快照数值,并把 metrics.stale 置为 true,
这样文章里的数字始终有出处,且能被识别出"这是旧数据"。
"""
from __future__ import annotations
import argparse
import re
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _common import ( # noqa: E402
DATA,
PROJECTS,
append_snapshot,
classify_license,
gh_get,
last_snapshot,
load_yaml,
today,
update_block,
)
def human_size(kb: int) -> str:
mb = kb / 1024
return f"{mb:.0f} MB" if mb >= 1 else f"{kb} KB"
def calc_cadence(releases: list) -> str:
"""从最近若干 release 的发布时间间隔估算版本节奏。"""
dates = []
for r in releases:
d = (r.get("published_at") or "")[:10]
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
dates.append(datetime.strptime(d, "%Y-%m-%d"))
if len(dates) < 2:
return "样本不足"
dates.sort(reverse=True)
spans = [(dates[i] - dates[i + 1]).days for i in range(len(dates) - 1)]
spans = [s for s in spans if s >= 0]
if not spans:
return "样本不足"
avg = sum(spans) / len(spans)
if avg < 1:
return "一天多版"
if avg < 2:
return "约 1 天/版本"
if avg < 8:
return f"约 {avg:.0f} 天/版本"
if avg < 40:
return f"约 {avg / 7:.1f} 周/版本"
return f"约 {avg / 30:.1f} 月/版本"
def fetch_one(pid: str, repo: str, force: bool) -> bool:
print(f"\n[{pid}] {repo}")
# ---- 当日缓存 ----
if not force:
prev = last_snapshot(pid)
if prev and prev.get("data_date") == today():
print(f" 今日已抓取过({today()}),跳过。用 --force 强制刷新。")
return True
# ---- 主仓库信息 ----
info = gh_get(f"repos/{repo}")
if not info:
print(" 主信息抓取失败,尝试沿用上次快照 …")
return fallback(pid)
lic = info.get("license") or {}
spdx = lic.get("spdx_id") or "NOASSERTION"
if spdx == "NOASSERTION" and not lic.get("key"):
spdx = ""
langs_raw = gh_get(f"repos/{repo}/languages") or {}
total = sum(langs_raw.values()) or 1
langs = {k: round(v / total * 100, 1) for k, v in sorted(langs_raw.items(), key=lambda x: -x[1])[:6]}
rels = gh_get(f"repos/{repo}/releases?per_page=8") or []
latest = ""
if rels:
latest = rels[0].get("tag_name") or ""
metrics = {
"stars": info.get("stargazers_count", 0),
"forks": info.get("forks_count", 0),
"open_issues": info.get("open_issues_count", 0),
"subscribers": info.get("subscribers_count", 0),
"created_at": (info.get("created_at") or "")[:10],
"pushed_at": (info.get("pushed_at") or "")[:10],
"size_kb": info.get("size", 0),
"languages": langs,
"latest_release": latest,
"release_cadence": calc_cadence(rels),
"data_date": today(),
"stale": False,
}
print(f" ★ {metrics['stars']} fork {metrics['forks']} issues {metrics['open_issues']}")
print(f" 最新 {latest} 节奏 {metrics['release_cadence']} 体积 {human_size(metrics['size_kb'])}")
print(f" 语言 {langs}")
print(f" 许可证 spdx={spdx or '(无)'}")
# ---- 快照 + 回写 ----
append_snapshot(pid, metrics)
ypath = PROJECTS / f"{pid}.yaml"
if ypath.exists():
text = ypath.read_text(encoding="utf-8")
try:
text = update_block(text, "metrics", metrics)
except ValueError as e:
print(f" [warn] 回写 metrics 失败:{e}")
# 许可证若尚未定级,顺手补上
if "openness: ~" in text or "openness: null" in text or 'openness: ""' in text:
cls = classify_license(spdx, lic.get("name", ""))
text = update_block(text, "license", {
"spdx": spdx or "NOASSERTION",
"name": lic.get("name") or "自定义许可",
"openness": cls["level"],
"redistributable": cls["redistributable"],
})
print(f" 自动定级:{cls['level']} → 镜像可见性 {cls['visibility']}")
ypath.write_text(text, encoding="utf-8")
else:
print(f" [warn] 档案不存在:{ypath},跳过回写")
return True
def fallback(pid: str) -> bool:
"""抓取失败:沿用上次快照并把 stale 标为 true。"""
prev = last_snapshot(pid)
ypath = PROJECTS / f"{pid}.yaml"
if not prev or not ypath.exists():
print(" 无历史快照可用,本次放弃。")
return False
prev = dict(prev)
prev["stale"] = True
text = ypath.read_text(encoding="utf-8")
try:
text = update_block(text, "metrics", prev)
ypath.write_text(text, encoding="utf-8")
print(f" 已沿用 {prev.get('data_date')} 的快照并标记 stale=true")
except ValueError as e:
print(f" [warn] 回写失败:{e}")
return False
return True
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--id", help="只刷新指定项目 id")
ap.add_argument("--force", action="store_true", help="忽略当日缓存")
args = ap.parse_args()
reg = load_yaml(DATA / "registry.yaml")
items = reg.get("projects", [])
if not items:
print("registry.yaml 里没有登记任何项目。")
return 1
targets = [p for p in items if not args.id or p.get("id") == args.id]
if not targets:
print(f"未找到 id={args.id}")
return 1
ok = sum(fetch_one(p["id"], p["repo"], args.force) for p in targets)
print(f"\n完成 {ok}/{len(targets)}")
return 0 if ok == len(targets) else 1
if __name__ == "__main__":
raise SystemExit(main())
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env bash
# 把被追踪项目的源码镜像到 gitinbox 组织,供代码级追踪。
#
# 用法:
# bash scripts/mirror_sync.sh --id ipollowork
# bash scripts/mirror_sync.sh --id ipollowork --depth 200 # 超大仓只取近期历史
#
# 可见性由档案的 business.license.redistributable 决定(L1/L2 → public,L3 → private)。
# 这条规则不能手工绕过:把 source-available 项目公开镜像,等于替对方做了一次
# 未经授权的公开分发。
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GITEA_URL="${GITEA_URL:-https://gitea.apescale.com}"
DISPLAY_URL="${GITEA_DISPLAY_URL:-https://gitea.apescale.com}"
ORG="${GITEA_ORG:-gitinbox}"
ID=""
DEPTH=""
while [[ $# -gt 0 ]]; do
case "$1" in
--id) ID="$2"; shift 2;;
--org) ORG="$2"; shift 2;;
--url) GITEA_URL="$2"; shift 2;;
--depth) DEPTH="$2"; shift 2;;
*) echo "未知参数: $1" >&2; exit 1;;
esac
done
[[ -z "$ID" ]] && { echo "缺少 --id" >&2; exit 1; }
YAML="$ROOT/data/projects/$ID.yaml"
[[ -f "$YAML" ]] || { echo "档案不存在: $YAML" >&2; exit 1; }
# ---- 从档案读取镜像配置(bash 不适合解析 YAML,借用 python 输出 shell 赋值) ----
# 用"写临时文件再 source"而不是 eval "$(...)":
# eval 会二次解析内容,YAML 里的括号/引号一旦漏转义就会变成难定位的
# syntax error,而且报错行号会偏移到完全无关的位置。
CONF_FILE="$(mktemp)"
trap 'rm -f "$CONF_FILE"' EXIT
/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python - "$YAML" > "$CONF_FILE" <<'PY'
import sys, shlex
from pathlib import Path
sys.path.insert(0, str(Path(sys.argv[1]).resolve().parent.parent.parent / "scripts"))
from _common import load_yaml
d = load_yaml(Path(sys.argv[1]))
m = d.get("mirror") or {}
lic = (d.get("business") or {}).get("license") or {}
print(f"UPSTREAM={shlex.quote(str(m.get('upstream') or ''))}")
print(f"REPO_NAME={shlex.quote(str(m.get('gitea_repo') or '').split('/')[-1] or d.get('id',''))}")
# 可见性以许可证判定为准,不信任档案里手写的 visibility
vis = "public" if lic.get("redistributable") else "private"
print(f"VISIBILITY={shlex.quote(vis)}")
print(f"LIC_LEVEL={shlex.quote(str(lic.get('openness') or '?'))}")
print(f"LIC_NAME={shlex.quote(str(lic.get('name') or ''))}")
PY
# shellcheck disable=SC1090
source "$CONF_FILE"
[[ -z "$UPSTREAM" ]] && { echo "档案缺少 mirror.upstream" >&2; exit 1; }
echo "==> 镜像 $ID"
echo " 上游 : $UPSTREAM"
echo " 目标 : $ORG/$REPO_NAME"
echo " 许可证 : $LIC_LEVEL ($LIC_NAME)"
echo " 可见性 : $VISIBILITY"
if [[ "$VISIBILITY" == "private" ]]; then
echo " ⚠ L3 级 source-available,已强制私有。公开镜像可能构成未经授权的分发。"
fi
# ---- 凭据:优先环境变量,回退到 macOS 钥匙串 ----
if [[ -z "${GITEA_TOKEN:-}" ]]; then
CRED="$(printf "host=%s\nprotocol=https\n" "${DISPLAY_URL#*://}" | git credential-osxkeychain get 2>/dev/null || true)"
GITEA_USER="${GITEA_USER:-$(printf '%s' "$CRED" | sed -n 's/^username=//p')}"
GITEA_TOKEN="$(printf '%s' "$CRED" | sed -n 's/^password=//p')"
fi
[[ -z "${GITEA_TOKEN:-}" || -z "${GITEA_USER:-}" ]] && {
echo "未拿到 Gitea 凭据(设 GITEA_TOKEN/GITEA_USER,或存入钥匙串)" >&2; exit 1; }
# ---- 确保目标仓库存在(组织端点,不是 /user/repos) ----
API="$GITEA_URL/api/v1"
HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -u "$GITEA_USER:$GITEA_TOKEN" "$API/repos/$ORG/$REPO_NAME" || echo 000)"
if [[ "$HTTP" == "200" ]]; then
echo " 仓库已存在"
elif [[ "$HTTP" == "404" ]]; then
echo " 创建仓库 $ORG/$REPO_NAME (private=$([[ $VISIBILITY == private ]] && echo true || echo false)) ..."
CREATE="$(curl -sS -o /tmp/gitea_mirror_create.json -w '%{http_code}' -u "$GITEA_USER:$GITEA_TOKEN" \
-H 'Content-Type: application/json' -X POST "$API/org/$ORG/repos" \
-d "{\"name\":\"$REPO_NAME\",\"private\":$([[ $VISIBILITY == private ]] && echo true || echo false),\"auto_init\":false,\"description\":\"Mirror of $UPSTREAM (license: $LIC_NAME)\"}")"
[[ "$CREATE" =~ ^(200|201)$ ]] || {
echo "创建失败 (HTTP $CREATE):" >&2; cat /tmp/gitea_mirror_create.json >&2; exit 1; }
echo " 已创建"
else
echo "查询仓库失败 (HTTP $HTTP)" >&2; exit 1
fi
# 已存在的仓库若应为私有,显式修正(创建时的 private 对已存在仓库不生效)
if [[ "$VISIBILITY" == "private" ]]; then
curl -sS -o /dev/null -u "$GITEA_USER:$GITEA_TOKEN" -X PATCH \
-H 'Content-Type: application/json' -d '{"private":true}' "$API/repos/$ORG/$REPO_NAME" || true
fi
# ---- 同步 ----
MIRROR_DIR="$ROOT/.mirrors/$REPO_NAME.git"
mkdir -p "$ROOT/.mirrors"
if [[ -d "$MIRROR_DIR" ]]; then
echo " 更新已有裸仓 ..."
git -C "$MIRROR_DIR" remote update --prune
else
echo " 首次全量镜像(大仓请耐心)..."
if [[ -n "$DEPTH" ]]; then
git clone --mirror --depth "$DEPTH" "$UPSTREAM" "$MIRROR_DIR"
else
git clone --mirror "$UPSTREAM" "$MIRROR_DIR"
fi
fi
PROTO="${GITEA_URL%%://*}"
HOST="${GITEA_URL#*//}"
U_ENC="$(/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python -c \
'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1],safe=""))' "$GITEA_USER")"
P_ENC="$(/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python -c \
'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1],safe=""))' "$GITEA_TOKEN")"
echo " 推送到 $ORG/$REPO_NAME ..."
git -C "$MIRROR_DIR" push --mirror "$PROTO://$U_ENC:$P_ENC@$HOST/$ORG/$REPO_NAME.git"
# ---- 回写档案 ----
/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python - "$YAML" "$VISIBILITY" "$REPO_NAME" <<'PY'
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(sys.argv[1]).resolve().parent.parent.parent / "scripts"))
from _common import today, update_block
p = Path(sys.argv[1])
text = p.read_text(encoding="utf-8")
text = update_block(text, "mirror", {
"visibility": sys.argv[2],
"gitea_repo": f"{'gitinbox'}/{sys.argv[3]}",
"last_sync": today(),
})
p.write_text(text, encoding="utf-8")
PY
SIZE="$(du -sh "$MIRROR_DIR" 2>/dev/null | cut -f1 || echo '?')"
echo "==> 完成。$DISPLAY_URL/$ORG/$REPO_NAME (本地裸仓 $SIZE)"
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""从 schema 生成一个项目的空白档案骨架,并自动预填可机器获取的字段。
用法
----
python3 scripts/new_project.py --repo Devin-AXIS/iPolloWork
python3 scripts/new_project.py --repo owner/name --id custom-id --status watch
生成后需要人工补完的内容(脚本不猜这些):
identity / business / scenario / tech / risk / verdict
其中 **risk.claim_vs_reality 是必填**,它承载"宣称 vs 实测"的落差分析,
是整套档案里最不能空着的字段。
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _common import ( # noqa: E402
DATA,
PROJECTS,
SCHEMA,
classify_license,
gh_get,
load_yaml,
today,
)
LEAF_KEYS = {"type", "required", "desc", "enum", "auto", "note"}
def render(node, indent=0, out=None, prefix="", idx=None):
"""把 schema 的 fields 树渲染成带注释的 YAML 文本。
同时返回 {字段路径: 行号} 索引。回填值必须走这个索引按**路径**定位,
不能靠硬编码缩进 —— schema 一旦加减层级,缩进常量就会悄悄错位,
而且像 `name` 这种同名不同层的字段(项目名 vs 许可证名)会互相覆盖。
"""
out = out if out is not None else []
idx = idx if idx is not None else {}
pad = " " * indent
for key, spec in node.items():
if not isinstance(spec, dict):
continue
path = f"{prefix}{key}"
if "type" in spec: # 叶子字段
default = {
"str": '""', "int": "0", "bool": "false",
"list": "[]", "dict": "{}",
}.get(spec["type"], '""')
comment = ""
if spec.get("enum"):
# fields 里的 enum 是 enums 段的键名引用(字符串),不是字面量数组;
# 直接 join 会把 "vendor_type" 拆成 v|e|n|d|o|r|_|t|y|p|e
e = spec["enum"]
comment = f" # {'|'.join(e) if isinstance(e, list) else e}"
elif spec.get("desc"):
d = re.sub(r"\s+", " ", spec["desc"].replace("\n", " "))
comment = f" # {d[:60]}"
idx[path] = len(out)
out.append(f"{pad}{key}: {default}{comment}")
elif spec and not (set(spec) & LEAF_KEYS): # 嵌套 section
out.append(f"{pad}{key}:")
render(spec, indent + 2, out, path + ".", idx)
return out, idx
def slug(repo: str) -> str:
name = repo.rstrip("/").split("/")[-1]
name = re.sub(r"\.git$", "", name)
return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--repo", required=True, help="GitHub 仓库 owner/name")
ap.add_argument("--id", help="档案 id,默认由仓库名生成")
ap.add_argument("--status", default="active", choices=["active", "watch", "archived"])
args = ap.parse_args()
pid = args.id or slug(args.repo)
ypath = PROJECTS / f"{pid}.yaml"
if ypath.exists():
print(f"档案已存在:{ypath}")
return 1
print(f"==> 生成档案 {pid} <- {args.repo}")
# ---- 抓可自动获取的部分 ----
info = gh_get(f"repos/{args.repo}") or {}
lic = info.get("license") or {}
spdx = lic.get("spdx_id") or ""
if spdx == "NOASSERTION":
spdx = "NOASSERTION"
cls = classify_license(spdx, lic.get("name", ""))
print(f" 许可证 spdx={spdx or '(无)'} → {cls['level']} / 可见性 {cls['visibility']}")
schema = load_yaml(SCHEMA / "project.schema.yaml")
lines, idx = render(schema.get("fields", {}))
def setval(path: str, val) -> None:
"""按字段路径回填值,保留该行原有的缩进与键名。"""
i = idx.get(path)
if i is None:
print(f" [warn] schema 中无字段 {path},跳过回填")
return
line = lines[i]
indent = " " * (len(line) - len(line.lstrip()))
key = path.split(".")[-1]
lines[i] = f"{indent}{key}: {val}"
setval("id", pid)
setval("name", f'"{info.get("name", "")}"')
setval("repo", f'"{args.repo}"')
setval("homepage", f'"{info.get("homepage") or ""}"')
setval("added_at", f'"{today()}"')
setval("status", args.status)
if info:
setval("identity.tagline", f'"{info.get("description") or ""}"')
setval("identity.first_seen", f'"{(info.get("created_at") or "")[:10]}"')
setval("business.license.spdx", f'"{spdx or "NOASSERTION"}"')
setval("business.license.name", f'"{lic.get("name") or "自定义许可"}"')
setval("business.license.openness", cls["level"])
setval("business.license.redistributable", "true" if cls["redistributable"] else "false")
setval("mirror.gitea_repo", f'"gitinbox/{info.get("name") or pid}"')
setval("mirror.visibility", cls["visibility"])
setval("mirror.upstream", f'"https://github.com/{args.repo}"')
text = "\n".join(lines)
header = (
f"# {info.get('name') or pid} · 项目档案\n"
f"# 生成于 {today()} · 由 scripts/new_project.py 从 schema 生成\n"
"# 人工补完重点:business / scenario / tech / risk.claim_vs_reality / verdict\n"
"# 字段规范见 schema/project.schema.yaml\n"
"\n"
)
PROJECTS.mkdir(parents=True, exist_ok=True)
ypath.write_text(header + text + "\n", encoding="utf-8")
print(f" 已写入 {ypath.relative_to(PROJECTS.parent.parent)}")
# ---- 登记到 registry ----
reg_path = DATA / "registry.yaml"
reg_text = reg_path.read_text(encoding="utf-8")
if f"id: {pid}" not in reg_text:
entry = (
f" - id: {pid}\n"
f" repo: {args.repo}\n"
f" status: {args.status}\n"
f" added_at: {today()}\n"
f" license_level: {cls['level']}"
+ (" # source-available,镜像私有\n" if cls["level"] == "L3" else "\n")
+ f" mirror_visibility: {cls['visibility']}\n"
)
reg_text = reg_text.rstrip("\n") + "\n" + entry
reg_path.write_text(reg_text, encoding="utf-8")
print(f" 已登记到 data/registry.yaml")
print(f"\n下一步:\n 1. 补完档案(重点 risk.claim_vs_reality)\n"
f" 2. python3 scripts/fetch_stats.py --id {pid}\n"
f" 3. bash scripts/mirror_sync.sh --id {pid}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""把 articles/ 下的 Markdown 源文发布(或更新)到 emlog 可巴蒂栏目。
用法
----
python3 scripts/publish_emlog.py --article ipollowork
python3 scripts/publish_emlog.py --article 00-overview
python3 scripts/publish_emlog.py --article ipollowork --draft y # 存草稿
两个关键设计
------------
1. **update 前必须先 detail 取回全文**。
emlog 的 article_update 接口在 content 为空时会把正文清空 —— 这是接口行为,
不是 bug。总纲文每次回填都要更新,所以这条保护是刚需。
2. **文章 ID 回写**。
项目文回写到 data/projects/<id>.yaml 的 article 块;
总纲文回写到 articles/00-overview.meta.yaml。
有了 ID 才能做"更新"而不是每次都发一篇新的。
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, "/Users/figmarlite/.workbuddy/skills/emlog-blog-publisher/scripts")
import yaml # noqa: E402
from _common import ARTICLES, PROJECTS, today, update_block # noqa: E402
import publish_article as emlog # noqa: E402
DEFAULT_SORT_ID = 17 # 可巴蒂
OVERVIEW_ID = "00-overview"
META_PATH = ARTICLES / f"{OVERVIEW_ID}.meta.yaml"
# ---------------------------------------------------------------- 转换
def md_to_html(md: str) -> str:
"""Markdown → HTML。emlog 只吃 HTML。"""
try:
import markdown
return markdown.markdown(
md, extensions=["tables", "fenced_code", "toc", "attr_list", "sane_lists"]
)
except ImportError:
sys.exit("缺少 markdown 库:pip install markdown")
def split_title(md: str) -> tuple[str, str]:
"""取第一个一级标题作为文章标题,其余作为正文。"""
lines = md.lstrip().split("\n")
if lines and lines[0].startswith("# "):
return lines[0][2:].strip(), "\n".join(lines[1:]).lstrip("\n")
return "未命名", md
def to_excerpt(md: str, limit: int = 110) -> str:
"""摘一段纯文本作为摘要:跳过标题与表格,取第一段实质内容。"""
for line in md.split("\n"):
s = line.strip()
if not s or s.startswith(("#", "|", ">", "-", "*", "!")) or s.startswith("<"):
continue
s = re.sub(r"[*`_]", "", s)
return s[:limit]
return ""
# ---------------------------------------------------------------- ID 读写
def read_emlog_id(aid: str) -> int | None:
if aid == OVERVIEW_ID:
if not META_PATH.exists():
return None
return (yaml.safe_load(META_PATH.read_text(encoding="utf-8")) or {}).get("emlog_id")
p = PROJECTS / f"{aid}.yaml"
if not p.exists():
return None
d = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
v = (d.get("article") or {}).get("emlog_id")
return int(v) if v else None
def write_emlog_id(aid: str, emlog_id: int) -> None:
url = f"{emlog.BLOG_URL}/?post={emlog_id}"
if aid == OVERVIEW_ID:
META_PATH.write_text(
f"# 总纲文发布元数据(由 scripts/publish_emlog.py 维护)\n"
f"emlog_id: {emlog_id}\nurl: \"{url}\"\nlast_published: \"{today()}\"\n",
encoding="utf-8",
)
print(f" 已写入 {META_PATH.name}")
return
p = PROJECTS / f"{aid}.yaml"
if not p.exists():
print(f" [warn] 档案不存在,未回写 ID: {p}")
return
text = p.read_text(encoding="utf-8")
try:
text = update_block(text, "article", {
"emlog_id": emlog_id, "published_at": today(), "url": url,
})
p.write_text(text, encoding="utf-8")
print(f" 已回写 emlog_id 到档案")
except ValueError as e:
print(f" [warn] 回写失败:{e}")
# ---------------------------------------------------------------- 发布
def do_publish(aid: str, title: str, html: str, sort_id: int, tags: str, draft: str, excerpt: str) -> None:
res = emlog.publish_article(
title=title, content=html, sort_id=sort_id,
excerpt=excerpt, tags=tags, draft=draft,
)
if res.get("code") != 0:
print(f" 发布失败:{res.get('msg')}")
sys.exit(1)
new_id = int(res.get("data", {}).get("article_id") or 0)
print(f" 已发布 id={new_id} {emlog.BLOG_URL}/?post={new_id}")
write_emlog_id(aid, new_id)
def do_update(aid: str, emlog_id: int, title: str, html: str, sort_id: int, tags: str, draft: str, excerpt: str) -> None:
"""更新既有文章。
关于"emlog update 不传 content 会清空正文"这个坑:
保护点应该是**本地正文非空**,而不是"先 detail 取回线上全文"。
两个理由——
1. 本脚本的 content 永远来自本地 Markdown 源文,不存在"忘了传"的情况;
2. detail 接口查不到**草稿**(实测:草稿 id 返回 400 Article not found,
已发布文章正常)。若把保护建立在 detail 上,草稿就永远无法转正。
所以 detail 降级为可选的健全性检查:取得到就对比一下长度,取不到也不阻断。
"""
print(f" 发现已有文章 id={emlog_id},走更新流程")
if not html.strip():
print(" 本地正文为空,拒绝更新(那会清空线上内容)")
sys.exit(1)
det = emlog.get_article_detail(emlog_id)
if det.get("code") == 0:
# 注意返回结构是 data.article.content,不是 data.content
art = (det.get("data") or {}).get("article") or {}
existing = art.get("content") or ""
print(f" 线上现有正文 {len(existing)} 字符 → 将覆盖为 {len(html)} 字符")
else:
print(f" 未能取回线上正文({det.get('msg', '未知')});"
f"本地正文非空,继续更新")
res = emlog.update_article(
article_id=emlog_id, title=title, content=html,
sort_id=sort_id, excerpt=excerpt, tags=tags, draft=draft,
)
if res.get("code") != 0:
print(f" 更新失败:{res.get('msg')}")
sys.exit(1)
print(f" 已更新 {emlog.BLOG_URL}/?post={emlog_id}")
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--article", required=True, help="articles/ 下的文件名(不含 .md)")
ap.add_argument("--sort-id", type=int, default=DEFAULT_SORT_ID)
ap.add_argument("--tags", default="")
ap.add_argument("--draft", default="n", choices=["y", "n"])
args = ap.parse_args()
md_path = ARTICLES / f"{args.article}.md"
if not md_path.exists():
print(f"文章不存在:{md_path}")
return 1
md = md_path.read_text(encoding="utf-8")
title, body = split_title(md)
html = md_to_html(body)
print(f"==> {args.article}")
print(f" 标题:{title}")
print(f" HTML {len(html)} 字符")
# 摘要要从正文里摘,不是从标题 —— 传标题进去等于没写摘要
excerpt = to_excerpt(body)
emlog_id = read_emlog_id(args.article)
if emlog_id:
do_update(args.article, emlog_id, title, html, args.sort_id, args.tags, args.draft, excerpt)
else:
do_publish(args.article, title, html, args.sort_id, args.tags, args.draft, excerpt)
return 0
if __name__ == "__main__":
raise SystemExit(main())