#!/usr/bin/env bash # 把被追踪项目的源码镜像到 gitinbox 组织,供代码级追踪。 # # 用法: # bash scripts/mirror_sync.sh --id ipollowork # bash scripts/mirror_sync.sh --id ipollowork --depth 200 # 超大仓只取近期历史 # # 可见性由档案的 business.license.redistributable 决定(L1/L2 → public,L3 → private)。 # 这条规则不能手工绕过:把 source-available 项目公开镜像,等于替对方做了一次 # 未经授权的公开分发。 set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" GITEA_URL="${GITEA_URL:-https://gitea.apescale.com}" DISPLAY_URL="${GITEA_DISPLAY_URL:-https://gitea.apescale.com}" ORG="${GITEA_ORG:-gitinbox}" ID="" DEPTH="" # auto = 体积超限时自动降级为源码快照;no = 始终尝试完整镜像(超限会 413 失败)。 # 这两个变量必须在 set -u 之前就有值:下面体积闸门的条件里会引用它们, # 而这个分支只在特定体积下才走到——不初始化就成了「偶尔才炸」的雷。 SOURCE_ONLY="auto" SOURCE_ONLY_FORCE="no" while [[ $# -gt 0 ]]; do case "$1" in --id) ID="$2"; shift 2;; --org) ORG="$2"; shift 2;; --url) GITEA_URL="$2"; shift 2;; --depth) DEPTH="$2"; shift 2;; --no-source-only) SOURCE_ONLY="no"; shift;; *) echo "未知参数: $1" >&2; exit 1;; esac done [[ -z "${ID}" ]] && { echo "缺少 --id" >&2; exit 1; } YAML="${ROOT}/data/projects/${ID}.yaml" [[ -f "${YAML}" ]] || { echo "档案不存在: ${YAML}" >&2; exit 1; } # ---- 从档案读取镜像配置(bash 不适合解析 YAML,借用 python 输出 shell 赋值) ---- # 用"写临时文件再 source"而不是 eval "$(...)": # eval 会二次解析内容,YAML 里的括号/引号一旦漏转义就会变成难定位的 # syntax error,而且报错行号会偏移到完全无关的位置。 CONF_FILE="$(mktemp)" trap 'rm -f "${CONF_FILE}"' EXIT /Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python - "${YAML}" > "${CONF_FILE}" <<'PY' import sys, shlex from pathlib import Path sys.path.insert(0, str(Path(sys.argv[1]).resolve().parent.parent.parent / "scripts")) from _common import load_yaml d = load_yaml(Path(sys.argv[1])) m = d.get("mirror") or {} lic = (d.get("business") or {}).get("license") or {} print(f"UPSTREAM={shlex.quote(str(m.get('upstream') or ''))}") print(f"REPO_NAME={shlex.quote(str(m.get('gitea_repo') or '').split('/')[-1] or d.get('id',''))}") # 可见性以许可证判定为准,不信任档案里手写的 visibility vis = "public" if lic.get("redistributable") else "private" print(f"VISIBILITY={shlex.quote(vis)}") print(f"LIC_LEVEL={shlex.quote(str(lic.get('openness') or '?'))}") print(f"LIC_NAME={shlex.quote(str(lic.get('name') or ''))}") PY # shellcheck disable=SC1090 source "${CONF_FILE}" [[ -z "${UPSTREAM}" ]] && { echo "档案缺少 mirror.upstream" >&2; exit 1; } echo "==> 镜像 ${ID}" echo " 上游 : ${UPSTREAM}" echo " 目标 : ${ORG}/${REPO_NAME}" echo " 许可证 : ${LIC_LEVEL} (${LIC_NAME})" echo " 可见性 : ${VISIBILITY}" if [[ "${VISIBILITY}" == "private" ]]; then echo " ⚠ L3 级 source-available,已强制私有。公开镜像可能构成未经授权的分发。" fi # ---- 凭据:优先环境变量,回退到 macOS 钥匙串 ---- if [[ -z "${GITEA_TOKEN:-}" ]]; then CRED="$(printf "host=%s\nprotocol=https\n" "${DISPLAY_URL#*://}" | git credential-osxkeychain get 2>/dev/null || true)" GITEA_USER="${GITEA_USER:-$(printf '%s' "${CRED}" | sed -n 's/^username=//p')}" GITEA_TOKEN="$(printf '%s' "${CRED}" | sed -n 's/^password=//p')" fi [[ -z "${GITEA_TOKEN:-}" || -z "${GITEA_USER:-}" ]] && { echo "未拿到 Gitea 凭据(设 GITEA_TOKEN/GITEA_USER,或存入钥匙串)" >&2; exit 1; } # ---- 确保目标仓库存在(组织端点,不是 /user/repos) ---- API="${GITEA_URL}/api/v1" HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -u "${GITEA_USER}:${GITEA_TOKEN}" "${API}/repos/${ORG}/${REPO_NAME}" || echo 000)" if [[ "${HTTP}" == "200" ]]; then echo " 仓库已存在" elif [[ "${HTTP}" == "404" ]]; then echo " 创建仓库 ${ORG}/${REPO_NAME} (private=$([[ ${VISIBILITY} == private ]] && echo true || echo false)) ..." CREATE="$(curl -sS -o /tmp/gitea_mirror_create.json -w '%{http_code}' -u "${GITEA_USER}:${GITEA_TOKEN}" \ -H 'Content-Type: application/json' -X POST "${API}/org/${ORG}/repos" \ -d "{\"name\":\"${REPO_NAME}\",\"private\":$([[ ${VISIBILITY} == private ]] && echo true || echo false),\"auto_init\":false,\"description\":\"Mirror of ${UPSTREAM} (license: ${LIC_NAME})\"}")" [[ "${CREATE}" =~ ^(200|201)$ ]] || { echo "创建失败 (HTTP ${CREATE}):" >&2; cat /tmp/gitea_mirror_create.json >&2; exit 1; } echo " 已创建" else echo "查询仓库失败 (HTTP ${HTTP})" >&2; exit 1 fi # 已存在的仓库若应为私有,显式修正(创建时的 private 对已存在仓库不生效) if [[ "${VISIBILITY}" == "private" ]]; then curl -sS -o /dev/null -u "${GITEA_USER}:${GITEA_TOKEN}" -X PATCH \ -H 'Content-Type: application/json' -d '{"private":true}' "${API}/repos/${ORG}/${REPO_NAME}" || true fi # ---- 同步 ---- MIRROR_DIR="${ROOT}/.mirrors/${REPO_NAME}.git" mkdir -p "${ROOT}/.mirrors" DEEPEN_STEP="${DEEPEN_STEP:-100}" # 注意:不能用 `rev-parse --git-dir` 判断仓库是否有效。git clone 会先建好 # 目录骨架再下载 pack,被中断的半成品照样能通过 rev-parse,于是被当成 # "已有仓库"走 fetch 分支,报出 "CONNECT tunnel failed" 这类误导性错误。 # 真正的判据是"里面有没有提交"。 repo_ok() { [[ -d "${MIRROR_DIR}" ]] || return 1 git -C "${MIRROR_DIR}" rev-parse --git-dir >/dev/null 2>&1 || return 1 local n n="$(git -C "${MIRROR_DIR}" rev-list --count --all 2>/dev/null || echo 0)" [[ "${n}" -gt 0 ]] } is_shallow() { [[ "$(git -C "${MIRROR_DIR}" rev-parse --is-shallow-repository 2>/dev/null || echo false)" == "true" ]]; } # clone/fetch 失败时清掉半成品,避免下次误判(脚本有 set -e,失败会直接退出) cleanup_broken() { if [[ -d "${MIRROR_DIR}" ]] && ! repo_ok; then echo " 清理未完成的半成品裸仓 ..." rm -rf "${MIRROR_DIR}" fi } trap cleanup_broken EXIT if [[ -d "${MIRROR_DIR}" ]] && ! repo_ok; then echo " 清理上次中断的半成品裸仓 ..." rm -rf "${MIRROR_DIR}" fi if repo_ok; then if is_shallow && [[ -z "${DEPTH}" ]]; then # 慢链路续传:分批 --deepen。每批成功即落盘,中断后重跑本脚本可继续, # 不像 `git clone` 那样失败一次全部作废。 echo " 浅仓分批续传(每批 --deepen=${DEEPEN_STEP},中断可重跑)..." while is_shallow; do PRE="$(git -C "${MIRROR_DIR}" rev-list --count --all 2>/dev/null || echo 0)" echo " 已有 ${PRE} 条历史,再加深 ${DEEPEN_STEP} ..." if ! git -C "${MIRROR_DIR}" fetch --deepen="${DEEPEN_STEP}" origin; then echo " ⚠ 本批中断;已落盘的 ${PRE} 条历史保留,重跑本脚本继续。" break fi POST="$(git -C "${MIRROR_DIR}" rev-list --count --all 2>/dev/null || echo 0)" [[ "${POST}" == "${PRE}" ]] && { echo " 已补齐完整历史"; break; } done elif is_shallow && [[ -n "${DEPTH}" ]]; then echo " 浅仓加深到 depth=${DEPTH} ..." git -C "${MIRROR_DIR}" fetch --depth="${DEPTH}" origin || \ echo " ⚠ 加深中断,已保留现有深度,重跑可继续" else echo " 更新已有裸仓 ..." git -C "${MIRROR_DIR}" remote update --prune fi else # 走沙箱代理时偶发 `CONNECT tunnel failed, response 502`,且 git clone # 失败不保留已下载内容,所以首次克隆必须带重试。 CLONE_ARGS=(clone --mirror) [[ -n "${DEPTH}" ]] && CLONE_ARGS+=(--depth "${DEPTH}") if [[ -n "${DEPTH}" ]]; then echo " 首次镜像(--depth ${DEPTH},只取近期历史)..." else echo " 首次全量镜像(大仓请耐心)..." fi for attempt in 1 2 3; do if git "${CLONE_ARGS[@]}" "${UPSTREAM}" "${MIRROR_DIR}"; then break fi echo " ⚠ 第 ${attempt} 次克隆失败,8s 后重试 ..." rm -rf "${MIRROR_DIR}" sleep 8 [[ ${attempt} -eq 3 ]] && { echo "克隆三次均失败,请检查网络/代理" >&2; exit 1; } done fi # ---- 浅仓处理:服务端拒绝 shallow update,必须先补齐历史 ---- # Git 服务端默认 `receive.shallowUpdate = deny`,推送浅仓会被拒: # ! [remote rejected] main -> main (shallow update not allowed) # 而 --depth 是本脚本自己提供的选项,推不上去等于自相矛盾,所以在这里兜住: # 先 --unshallow 补齐(增量下载,通常只比浅仓多几十 MB),失败再降级为快照。 UNSHALLOWED="no" if is_shallow; then echo " 浅仓无法直接推送(服务端拒绝 shallow update),先补齐完整历史 ..." for attempt in 1 2 3; do if git -C "${MIRROR_DIR}" fetch --unshallow origin 2>/dev/null; then UNSHALLOWED="yes"; break fi echo " ⚠ 第 ${attempt} 次 unshallow 失败,10s 后重试 ..." sleep 10 done if [[ "${UNSHALLOWED}" == "yes" ]]; then echo " 已补齐完整历史" else # 网络或体积原因补不齐 —— 退回快照模式,丢历史但至少把代码留下来。 echo " ⚠ 补齐失败,将降级为【源码快照】(会丢失提交历史)" SOURCE_ONLY_FORCE="yes" fi fi PROTO="${GITEA_URL%%://*}" HOST="${GITEA_URL#*//}" U_ENC="$(/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python -c \ 'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1],safe=""))' "${GITEA_USER}")" P_ENC="$(/Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python -c \ 'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1],safe=""))' "${GITEA_TOKEN}")" # ---- 体积闸门:Gitea 前置 nginx 有 client_max_body_size 上限 ---- # 实测 gitea.apescale.com:48 MB 放行、64 MB 拒绝(推算为 50m,宝塔常见默认值)。 # 超限推送会在传完全部对象后才收到 HTTP 413,白白浪费几十分钟下载。 # 所以推送前先量体积,超限则自动降级为「源码快照」:剔除图片/音视频/字体等 # 非源码素材——这些内容通常占一半以上体积,却不是"跟踪源码"要看的东西。 MAX_PUSH_MB="${MAX_PUSH_MB:-45}" PACK_KB="$(git -C "${MIRROR_DIR}" count-objects -v | awk '/^size-pack:/{print $2}')" PACK_MB=$(( PACK_KB / 1024 )) MIRROR_MODE="full" echo " 本地裸仓打包体积: ${PACK_MB} MB(推送上限 ${MAX_PUSH_MB} MB)" if [[ ( "${PACK_MB}" -gt "${MAX_PUSH_MB}" || "${SOURCE_ONLY_FORCE}" == "yes" ) \ && "${SOURCE_ONLY}" != "no" ]]; then MIRROR_MODE="source-only" if [[ "${SOURCE_ONLY_FORCE}" == "yes" ]]; then echo " ⚠ 降级为【源码快照】:提交历史不完整,按快照提交(仍会剔除素材)" else echo " ⚠ 超出上限,降级为【源码快照】:剔除非源码素材后推送" fi SNAP="$(mktemp -d)" # shellcheck disable=SC2064 trap "cleanup_broken; rm -rf '${SNAP}'" EXIT git init -q "${SNAP}/repo" git --git-dir="${MIRROR_DIR}" --work-tree="${SNAP}/repo" checkout HEAD -- . 2>/dev/null find "${SNAP}/repo" -type f -not -path '*/.git/*' \( \ -iname '*.png' -o -iname '*.gif' -o -iname '*.jpg' -o -iname '*.jpeg' \ -o -iname '*.webp' -o -iname '*.ico' -o -iname '*.mp4' -o -iname '*.mov' \ -o -iname '*.webm' -o -iname '*.wav' -o -iname '*.mp3' -o -iname '*.glb' \ -o -iname '*.ttf' -o -iname '*.woff' -o -iname '*.woff2' -o -iname '*.eot' \ -o -iname '*.pdf' -o -iname '*.zip' \) -delete cat > "${SNAP}/repo/MIRROR-NOTE.md" </dev/null || true SNAP_MB=$(( $(git -C "${PUSH_DIR}" count-objects -v | awk '/^size-pack:/{print $2}') / 1024 )) echo " 源码快照体积: ${SNAP_MB} MB" else PUSH_DIR="${MIRROR_DIR}" fi echo " 推送到 ${ORG}/${REPO_NAME} ..." if [[ "${MIRROR_MODE}" == "full" ]]; then git -C "${PUSH_DIR}" push --mirror "${PROTO}://${U_ENC}:${P_ENC}@${HOST}/${ORG}/${REPO_NAME}.git" else git -C "${PUSH_DIR}" push "${PROTO}://${U_ENC}:${P_ENC}@${HOST}/${ORG}/${REPO_NAME}.git" HEAD:refs/heads/main fi # ---- 回写档案 ---- /Users/figmarlite/.workbuddy/binaries/python/envs/default/bin/python - "${YAML}" "${VISIBILITY}" "${REPO_NAME}" "${MIRROR_MODE}" <<'PY' import sys from pathlib import Path sys.path.insert(0, str(Path(sys.argv[1]).resolve().parent.parent.parent / "scripts")) from _common import today, update_block p = Path(sys.argv[1]) text = p.read_text(encoding="utf-8") text = update_block(text, "mirror", { "visibility": sys.argv[2], "gitea_repo": f"{'gitinbox'}/{sys.argv[3]}", "last_sync": today(), "mode": sys.argv[4], }) p.write_text(text, encoding="utf-8") PY SIZE="$(du -sh "${MIRROR_DIR}" 2>/dev/null | cut -f1 || echo '?')" echo "==> 完成。${DISPLAY_URL}/${ORG}/${REPO_NAME} (模式 ${MIRROR_MODE},本地裸仓 ${SIZE})"