Merge pull request 'fix(gitea): 备份失败后按小时补跑' (#3 ) from codex/gitea-reliability-1-retry into master

fix(gitea): 保留跨日备份并恢复中断服务
fix(gitea): 备份失败后按小时补跑
2026-09-26 07:58:33 +08:00 · 2026-09-26 07:53:38 +08:00 · 2026-09-26 07:50:00 +08:00 · 2026-09-25 23:12:46 +08:00 · 2026-09-25 23:02:42 +08:00 · 2026-09-25 22:59:35 +08:00
16 changed files with 1415 additions and 350 deletions
@@ -1,10 +1,5 @@
 #!/bin/bash
-# Gitea 快速备份脚本
-
+# Backups use generated timestamps; legacy name arguments are not archive paths.
 set -euo pipefail
-
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-BACKUP_NAME="${1:-manual-$(date +%Y%m%d-%H%M%S)}"
-
-echo "🔄 正在备份 Gitea 数据: ${BACKUP_NAME}"
-"${SCRIPT_DIR}/gitea-data-protection.sh" backup "${BACKUP_NAME}"
+exec "${SCRIPT_DIR}/gitea-data-protection.sh" backup
@@ -283,29 +283,10 @@ __GITEA_WATCHDOG_PLIST__

 # 初始化设置
 setup() {
-    log "🔧 开始 Gitea 数据保护初始化设置..."
-
-    if ! check_docker; then
-        log "❌ Docker 环境检查失败"
-        return 1
-    fi
-
-    if [[ ! -f "${ENV_FILE}" ]]; then
-        log "📝 创建环境配置文件..."
-        cp "${SCRIPT_DIR}/.env.example" "${ENV_FILE}"
-        log "✅ 已创建 ${ENV_FILE}，请根据需要修改配置"
-    fi
-
-    load_env
-    init_config
-
-    create_auto_backup_script
-    setup_cron_backup
-    setup_launchd_backup
-    setup_launchd_watchdog
-    verify_setup
-
-    log "✅ Gitea 数据保护初始化完成"
+    local installer="${SCRIPT_DIR}/reliability/install.py"
+    [[ -f "${installer}" ]] || installer="${HOME}/ops/gitea-reliability/install.py"
+    [[ -f "${installer}" ]] || { log "请从当前源码的 gitea/reliability/install.py 安装备份服务"; return 1; }
+    exec /opt/homebrew/bin/python3 "${installer}"
 }

 # 判断是否应跳过本次自动备份
@@ -374,92 +355,9 @@ mark_backup_success() {

 # 执行备份
 backup() {
-    local backup_name="${1:-gitea-$(date +%Y%m%d-%H%M%S)}"
-    local backup_path
-    local restart_needed=false
-    local backup_ok=true
-    local auto_triggered=false
-
-    load_env
-    init_config
-
-    backup_path="${GITEA_BACKUP_DIR}/${backup_name}.tar.gz"
-
-    if [[ "${backup_name}" == auto-* ]]; then
-        auto_triggered=true
-    fi
-
-    if ! acquire_backup_lock; then
-        return 0
-    fi
-
-    if [[ "${auto_triggered}" == "true" ]] && backup_recently_ran; then
-        release_backup_lock
-        return 0
-    fi
-
-    log "🔄 开始备份 Gitea 数据: ${backup_name}"
-
-    if is_true "${GITEA_BACKUP_STOP_SERVICE}" && check_gitea_status; then
-        log "⏸️ 临时停止 Gitea 服务以确保数据一致性..."
-        if docker-compose -f "${COMPOSE_FILE}" stop gitea; then
-            restart_needed=true
-        else
-            log "❌ 停止 Gitea 服务失败"
-            backup_ok=false
-        fi
-    else
-        log "ℹ️ 在线备份模式：备份期间不停止 Gitea 服务"
-    fi
-
-    if [[ "${backup_ok}" == "true" ]]; then
-        log "📦 创建备份到: ${backup_path}"
-        if ! docker run --rm \
-            -v gitea-data:/source:ro \
-            -v "${GITEA_BACKUP_DIR}:/backup" \
-            busybox \
-            tar -czf "/backup/$(basename "${backup_path}")" -C /source .; then
-            log "❌ 备份执行失败"
-            backup_ok=false
-        fi
-    fi
-
-    if [[ "${restart_needed}" == "true" ]]; then
-        log "🚀 重启 Gitea 服务..."
-        if ! docker-compose -f "${COMPOSE_FILE}" start gitea; then
-            log "❌ Gitea 服务重启失败"
-            backup_ok=false
-        fi
-    fi
-
-    if [[ "${backup_ok}" != "true" ]]; then
-        release_backup_lock
-        return 1
-    fi
-
-    if [[ -f "${backup_path}" ]]; then
-        local backup_size
-        backup_size="$(du -h "${backup_path}" | cut -f1)"
-        log "✅ 备份完成: ${backup_path} (大小: ${backup_size})"
-
-        cat > "${backup_path}.meta" << __GITEA_BACKUP_META__
-backup_name: ${backup_name}
-backup_time: $(date -Iseconds)
-backup_size: ${backup_size}
-gitea_domain: ${GITEA_DOMAIN}
-script_version: 1.1
-__GITEA_BACKUP_META__
-
-        mark_backup_success
-        cleanup_old_backups
-    else
-        log "❌ 备份文件未找到"
-        release_backup_lock
-        return 1
-    fi
-
-    release_backup_lock
-    return 0
+    local tool="${HOME}/ops/gitea-reliability/backup.py"
+    [[ -f "${tool}" ]] || { log "请先安装 gitea/reliability；拒绝未验证备份"; return 1; }
+    exec /opt/homebrew/bin/python3 "${tool}" backup
 }

 # 清理旧备份
@@ -471,213 +369,22 @@ cleanup_old_backups() {

 # 恢复数据
 restore() {
-    local backup_file="$1"
-
-    load_env
-    init_config
-
-    if [[ -z "${backup_file}" ]]; then
-        log "📋 可用的备份文件:"
-        ls -lt "${GITEA_BACKUP_DIR}"/*.tar.gz 2>/dev/null | head -10 || log "❌ 未找到备份文件"
-        echo ""
-        echo "使用方法: $0 restore <backup_file>"
-        return 1
-    fi
-
-    if [[ ! -f "${backup_file}" ]]; then
-        backup_file="${GITEA_BACKUP_DIR}/${backup_file}"
-        if [[ ! -f "${backup_file}" ]]; then
-            log "❌ 备份文件不存在: ${backup_file}"
-            return 1
-        fi
-    fi
-
-    log "🔄 开始恢复 Gitea 数据从: $(basename "${backup_file}")"
-
-    log "⏸️ 停止 Gitea 服务..."
-    docker-compose -f "${COMPOSE_FILE}" down
-
-    if docker volume inspect gitea-data >/dev/null 2>&1; then
-        local safety_backup="${GITEA_BACKUP_DIR}/safety-backup-$(date +%Y%m%d-%H%M%S).tar.gz"
-        log "💾 创建安全备份: $(basename "${safety_backup}")"
-        docker run --rm \
-            -v gitea-data:/source:ro \
-            -v "${GITEA_BACKUP_DIR}:/backup" \
-            busybox \
-            tar -czf "/backup/$(basename "${safety_backup}")" -C /source .
-    fi
-
-    docker volume rm gitea-data 2>/dev/null || true
-    docker volume create gitea-data
-
-    log "📦 恢复数据..."
-    if docker run --rm \
-        -v gitea-data:/target \
-        -v "$(dirname "${backup_file}"):/backup:ro" \
-        busybox \
-        tar -xzf "/backup/$(basename "${backup_file}")" -C /target; then
-
-        log "🚀 启动 Gitea 服务..."
-        docker-compose -f "${COMPOSE_FILE}" up -d
-
-        sleep 10
-
-        if check_gitea_status && check_http_health; then
-            log "✅ 数据恢复完成，服务正常运行"
-        else
-            log "⚠️ 服务启动可能有问题，请检查日志"
-        fi
-    else
-        log "❌ 数据恢复失败"
-        return 1
+    local tool="${HOME}/ops/gitea-reliability/backup.py"
+    [[ -f "${tool}" ]] || { log "请先安装 gitea/reliability"; return 1; }
+    if [[ -n "${1:-}" ]]; then
+        exec /opt/homebrew/bin/python3 "${tool}" restore-check --record "$1" --external
    fi
+    exec /opt/homebrew/bin/python3 "${tool}" restore-check --external
 }

 # 自动自愈检查
 watchdog_check() {
-    load_env true
-    init_config
-
-    if ! check_docker; then
-        log "❌ Watchdog: Docker 环境不可用"
-        return 1
-    fi
-
-    local fail_count=0
-    local previous_state="unknown"
-    local healthy=true
-
-    if [[ -f "${WATCHDOG_FAIL_COUNT_FILE}" ]]; then
-        fail_count="$(tr -d '[:space:]' < "${WATCHDOG_FAIL_COUNT_FILE}" 2>/dev/null || true)"
-        if [[ ! "${fail_count}" =~ ^[0-9]+$ ]]; then
-            fail_count=0
-        fi
-    fi
-
-    if [[ -f "${WATCHDOG_STATE_FILE}" ]]; then
-        previous_state="$(tr -d '[:space:]' < "${WATCHDOG_STATE_FILE}" 2>/dev/null || true)"
-    fi
-
-    if ! check_gitea_status; then
-        healthy=false
-        log "⚠️ Watchdog: gitea 容器未运行"
-    elif ! check_http_health; then
-        healthy=false
-        log "⚠️ Watchdog: HTTP 探活失败 (127.0.0.1:33000)"
-    fi
-
-    if [[ "${healthy}" == "true" ]]; then
-        if (( fail_count > 0 )) || [[ "${previous_state}" != "healthy" ]]; then
-            log "✅ Watchdog: 服务已恢复正常"
-        fi
-        rm -f "${WATCHDOG_FAIL_COUNT_FILE}"
-        echo "healthy" > "${WATCHDOG_STATE_FILE}"
-        return 0
-    fi
-
-    fail_count=$((fail_count + 1))
-    echo "${fail_count}" > "${WATCHDOG_FAIL_COUNT_FILE}"
-    echo "unhealthy" > "${WATCHDOG_STATE_FILE}"
-    log "⚠️ Watchdog: 连续失败 ${fail_count}/${GITEA_WATCHDOG_CONSECUTIVE_FAILURES}"
-
-    if (( fail_count < GITEA_WATCHDOG_CONSECUTIVE_FAILURES )); then
-        return 1
-    fi
-
-    log "🔄 Watchdog: 开始自动重启 gitea 服务..."
-    if ! docker-compose -f "${COMPOSE_FILE}" restart gitea; then
-        log "❌ Watchdog: 自动重启命令执行失败"
-        return 1
-    fi
-
-    sleep 8
-
-    if check_http_health; then
-        log "✅ Watchdog: 自动重启成功"
-        rm -f "${WATCHDOG_FAIL_COUNT_FILE}"
-        echo "healthy" > "${WATCHDOG_STATE_FILE}"
-        return 0
-    fi
-
-    log "❌ Watchdog: 重启后健康检查仍失败"
-    return 1
+    exec /opt/homebrew/bin/python3 "${HOME}/ops/gitea-reliability/backup.py" guard
 }

 # 健康检查
 health_check() {
-    load_env
-    init_config
-
-    log "🏥 开始健康检查..."
-
-    if ! check_docker; then
-        log "❌ Docker 环境检查失败"
-        return 1
-    fi
-
-    if check_gitea_status; then
-        log "✅ Gitea 服务运行正常"
-    else
-        log "❌ Gitea 服务未运行"
-        return 1
-    fi
-
-    if check_http_health; then
-        log "✅ HTTP 健康检查通过 (127.0.0.1:33000)"
-    else
-        log "❌ HTTP 健康检查失败 (127.0.0.1:33000)"
-        return 1
-    fi
-
-    if docker volume inspect gitea-data >/dev/null 2>&1; then
-        log "✅ 数据卷 gitea-data 存在"
-
-        local repo_count
-        repo_count="$(docker run --rm \
-            -v gitea-data:/data:ro \
-            busybox \
-            find /data -name "*.git" -type d 2>/dev/null | wc -l)"
-        log "📊 发现 ${repo_count} 个 Git 仓库"
-    else
-        log "❌ 数据卷不存在"
-        return 1
-    fi
-
-    local recent_backups
-    recent_backups="$(find "${GITEA_BACKUP_DIR}" -name "*.tar.gz" ! -name "safety-backup-*.tar.gz" -mtime -1 2>/dev/null | wc -l)"
-    log "📈 最近24小时内的备份: ${recent_backups} 个"
-
-    if [[ ${recent_backups} -eq 0 ]]; then
-        log "⚠️ 警告: 最近24小时内无备份"
-    fi
-
-    if command -v launchctl >/dev/null 2>&1; then
-        if is_true "${GITEA_ENABLE_LAUNCHD_BACKUP}"; then
-            if launchctl list | grep -q "com.ljwx.gitea.backup"; then
-                log "✅ launchd 备份任务已设置"
-            else
-                log "⚠️ 未发现 launchd 备份任务"
-            fi
-        fi
-
-        if is_true "${GITEA_ENABLE_WATCHDOG}"; then
-            if launchctl list | grep -q "com.ljwx.gitea.watchdog"; then
-                log "✅ launchd watchdog 任务已设置"
-            else
-                log "⚠️ 未发现 launchd watchdog 任务"
-            fi
-        fi
-    fi
-
-    if is_true "${GITEA_ENABLE_CRON_BACKUP}"; then
-        if crontab -l 2>/dev/null | grep -Fq "${SCRIPT_DIR}/gitea-auto-backup.sh"; then
-            log "✅ cron 备份任务已设置"
-        else
-            log "⚠️ 未发现 cron 备份任务"
-        fi
-    fi
-
-    log "✅ 健康检查完成"
+    exec /opt/homebrew/bin/python3 "${HOME}/ops/gitea-reliability/backup.py" status
 }

 # 验证设置
@@ -715,23 +422,14 @@ verify_setup() {

 # 启动服务
 start() {
-    log "🚀 启动 Gitea 服务..."
-
-    load_env
-    init_config
-
-    docker-compose -f "${COMPOSE_FILE}" up -d
-
-    sleep 10
-
-    health_check
+    docker start gitea
+    exec /opt/homebrew/bin/python3 "${HOME}/ops/gitea-reliability/backup.py" guard
 }

 # 停止服务
 stop() {
-    log "⏸️ 停止 Gitea 服务..."
-    docker-compose -f "${COMPOSE_FILE}" down
-    log "✅ 服务已停止"
+    # Stop only the inspected service; never tear down the historical stack.
+    docker stop --time 30 gitea
 }

 # 主函数
@@ -765,20 +463,20 @@ main() {
            echo "  $0 setup               # 初始化设置"
            echo "  $0 start               # 启动 Gitea 服务"
            echo "  $0 stop                # 停止 Gitea 服务"
-            echo "  $0 backup [name]       # 创建备份"
-            echo "  $0 restore <backup>    # 恢复数据"
+            echo "  $0 backup              # 一致性加密备份并校验 T5"
+            echo "  $0 restore [record.json] # 从 T5 恢复到新测试卷"
            echo "  $0 health              # 健康检查"
            echo "  $0 watchdog            # 自愈探活（用于 launchd）"
            echo "  $0 help                # 显示帮助"
            echo ""
            echo "功能特性:"
-            echo "  ✅ 自动定时备份 (cron + launchd 可配置)"
-            echo "  ✅ 备份防重入与最小间隔保护"
-            echo "  ✅ 在线备份（默认不停服）"
+            echo "  ✅ 每日 03:30 launchd 备份"
+            echo "  ✅ 独占锁与异常中断恢复"
+            echo "  ✅ 冷副本确保一致性（短暂停机）"
            echo "  ✅ 自动自愈 watchdog"
-            echo "  ✅ 一键备份和恢复"
+            echo "  ✅ 新卷恢复演练，不覆盖生产"
            echo "  ✅ 健康监控"
-            echo "  ✅ 自动清理旧备份"
+            echo "  ✅ 完整校验及外置盘身份检查"
            ;;
    esac
 }
@@ -1,19 +1,5 @@
 #!/bin/bash
-# Gitea 快速恢复脚本
-
+# Restore is intentionally non-destructive: T5 -> a NEW Docker volume.
 set -euo pipefail
-
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [[ $# -eq 0 ]]; then
-    echo "📋 可用的备份文件:"
-    ls -lt ~/ops/backups/gitea/gitea-*.tar.gz 2>/dev/null | head -10 || echo "❌ 未找到备份文件"
-    echo ""
-    echo "使用方法: $0 <backup_file>"
-    echo "示例: $0 gitea-20251019-120000.tar.gz"
-    exit 1
-fi
-
-BACKUP_FILE="$1"
-echo "🔄 正在从备份恢复 Gitea 数据: $(basename "${BACKUP_FILE}")"
-"${SCRIPT_DIR}/gitea-data-protection.sh" restore "${BACKUP_FILE}"
+exec "${SCRIPT_DIR}/gitea-data-protection.sh" restore "${1:-}"
@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc
@@ -0,0 +1,45 @@
+# 2026-09-25 Gitea 接管准备回执
+
+问题：旧备份脚本固定读空卷 `gitea-data`，实际服务卷为 `gitea_gitea-data`；87 字节空归档会被报成功。
+旧 launchd 未加载，原 act_runner 为不适配当前架构的二进制。
+
+已实测完成：
+
+- 首次一致性数据备份 30,052,975,104 字节，restic 加密复制到 T5，并全量 read-data 校验通过。
+- 从 T5 恢复镜像与全新卷，207 仓库 / 41 issue / 17 PR / 6 用户一致，SQLite、全部 refs、所有库 git fsck 通过。
+- 隔离恢复实例 1.25.4 API/Git 读取通过，隔离升级到 1.27.3 API/Git/SQLite 通过。
+- 正式升级 1.27.3 固定 digest `sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef`；
+  新卷 `gitea-upgrade-20260925t145859z`，保留旧卷 `gitea_gitea-data`，停机 140.76 秒。
+  正式升级时 208 仓库 / 42 issue / 18 PR / 6 用户（增加了 CI 试点）。
+- `_gitea_ci` 专用账号、沙箱、私有仓库专属 runner、capacity 1、独立工具链已运行。
+- ai-shitiku 试点运行 #2：Python 182+239+869 项，Flutter 199 项，全部通过；70 变异体捕获率 100%，误冲突 0。
+- 试点失败运行 #3 被服务器 405 拒绝合并；修复运行 #4 通过。旧 HEAD 请求 409 拒绝；PR #1 正常合并。
+- 运维源码 27 项检查通过，包括真实 Docker 缺 blob 回归。独立 agent 交叉评审通过，评审不替代生产回执。
+
+当前入口 http://192.168.1.83:33000 ，SSH 32222。运行 Compose `~/ops/gitea/docker-compose.yml`。
+正式注册已关闭；既有用户、仓库可见性保留。未改变 GitHub 正式仓库、生产发布流程或订阅账单。
+
+运行回执目录 `~/ops/gitea-reliability/`：last-upgrade.json、last-success.json、last-restore.json、gate-*-receipt.json。
+旧实例配置 `compose-before-20260925T145859Z.yml` 固定旧镜像+旧卷；公开新写入后不能直接退旧卷。
+
+尚待验收（如后续回执已更新，以新增验收记录为准）：
+
+1. macOS 后台 Python 的可移除宗卷权限：首次 launchd audit 停在系统授权检查，已请求用户处理。
+2. backup.key 的独立保管：用户尚未确认密码管理器/另一载体；整机丢失恢复尚不能判定完成。
+3. 新版 CI 回归与升级后新快照正在执行，后续记录具体运行和 snapshot ID。
+
+正式项目逐个接管时仍须迁移 secrets、workflow、部署凭据与 status contexts，并验收一次真实发布和回退。
+当前只验证私有试点与平台运维，不能将其描述为所有 GitHub 部署已经迁移。
+
+## 2026-09-26 备份补跑
+
+03:30 的 Gitea 冷拷贝与本机 restic 加密校验成功，外置设备检查超时，所以定时任务退出码 1；
+07:44 使用同一份本机快照补复制到 T5，外置全量 read-data 校验通过，无需再次停止服务。
+新快照本机 ID `cc2940aca52a5c759c40742e0af3c138609512eb1d968887f6d7a4f19b2c04f3`，
+T5 ID `52f09d35e3fd31ce2aa3d24925ef101533dcff1f0810a203a983b307686ffdb6`。
+后台 launchd 状态审计于 07:45 再次通过，之前的外盘查询超时具有间歇性。
+
+本次修复把 03:30 单次定时改为 03:30 起按小时补跑：CI 占用则下一小时重试；
+外盘短暂不可用则优先复制已有本机加密快照；同一备份窗口通过后不重复停服务。
+跨日未完成的旧外置复制不阻止下一日生成新的本机快照；当日已复制但尚未发布 `last-success` 的回执会幂等补齐。
+这只能缩短临时故障造成的备份缺口。连续失败仍没有主动告警；密钥独立保管仍待用户确认。
@@ -0,0 +1,106 @@
+# Gitea 可靠性运维
+
+此目录是 Mac Studio 的运行源码。运行状态和凭据在 `~/ops/gitea-reliability/`，不跟踪。
+适合 1–2 个并行私有项目、单台 Studio；依赖 Mac 登录、OrbStack 和外置盘在线，不提供主机高可用。
+
+## 安装与日常检查
+
+依赖：Python 3.13+、PyYAML、Docker CLI、restic 0.19.1（Apple Silicon）。
+复制 `config.example.json` 为运行目录 `config.json`，核对所有绝对路径和 `diskutil info -plist` 返回的外置 VolumeUUID。
+运行目录 0700，配置及密钥 0600。禁止将 operator.token、backup.key、.runner、app.ini 或真实 .env 提交。
+
+```sh
+python3 gitea/reliability/install.py
+python3 ~/ops/gitea-reliability/backup.py status
+python3 ~/ops/gitea-reliability/backup.py backup
+python3 ~/ops/gitea-reliability/backup.py restore-check --external
+```
+
+安装器仅维护三个当前用户 LaunchAgent：每天 03:30 开始备份窗口，其后每小时检查一次，直到当天外盘快照校验成功；另有 60 秒异常停机恢复守卫、每小时备份健康检查。
+若 CI 占用，下一小时重试冷拷贝；若当天本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。跨日仍未补成的旧快照不会挡住今天的新本机冷拷贝。当天已验证后，后续小时只检查状态，不重做冷拷贝。
+不加载历史 Compose 中的其它 runner。后台 Python 首次访问 T5，需要 macOS 允许“可移除宗卷”。
+必须实际从 launchd 运行验证，终端成功不能代替后台权限验收。
+日志在运行目录 `logs/`；检查 `launchctl print gui/$(id -u)/com.ljwx.gitea.backup` 的退出码。
+停止用户登录会停止这些 LaunchAgent。CI runner 是独立 LaunchDaemon，但仍依赖 Gitea/OrbStack。
+
+`status` 检查 API 健康、外置设备 UUID/真实挂载点、最近 36 小时成功回执、数据及镜像快照实际存在。
+失败日志不会输出完整环境、命令 stderr 或凭据。它不发送邮件或聊天通知；应将非零退出接到已有监控。
+自动重试仍依赖用户登录、OrbStack 和 T5 在线；连续失败只记录在本机日志，没有主动通知。
+
+## 备份与恢复承诺
+
+- 从运行容器 inspect 找出唯一 `/data` 命名卷；绝不硬编码回退到空 `gitea-data`。
+- 拒绝运行中 CI；只停止 Gitea、复制冷数据卷，随后立即恢复服务。实测初次停机约 92 秒。
+- 冷拷贝超时 600 秒；SIGTERM/SIGINT 走 finally，SIGKILL 由独立 guard 根据维护标记恢复。
+- SQLite integrity、仓库/issue/PR/用户数和全部 refs 摘要进入回执。
+- Docker tar 流直接进入 restic 加密库；宿主和 T5 不落明文 tar。保留大小写、权限、SQLite、仓库、LFS、附件、Actions、配置及恢复镜像。
+- 本机成功后复制 T5，执行 `restic check --read-data` 全量校验；外置失败不记成功，可补复制而无需再次停机：
+
+```sh
+python3 ~/ops/gitea-reliability/backup.py replicate --record ~/ops/gitea-reliability/backup-YYYYMMDDTHHMMSSZ.json
+```
+
+本机库 `~/ops/backups/gitea-restic`；当前外置库 `/Volumes/t5/gitea-restic`。
+T5 为 HFS+，使用加密 tar 数据流避免文件名大小写冲突。拔盘或 UUID 不符明确失败，不自动创建系统盘替代目录。
+本机空闲不足 64 GiB 时拒绝创建新冷副本。失败的 stage 卷保留供调查，不自动删除未知数据。
+完成过恢复演练后，data 快照保留 7 日 / 4 周 / 3 月，另保留最近一次实际恢复的快照。
+恢复镜像快照不自动删除；旧的历史 tar.gz 不纳入清理。restic 内容去重降低重复镜像占用。
+
+恢复命令只创建新的 `gitea-restore-check-*` 卷，强制从备份加载镜像，验证 SQLite/计数/refs 以及每库 `git fsck --full`。
+它不覆盖生产卷。恢复成功回执 `last-restore.json` 记录新卷；要接管端口必须另行验证与切换。
+
+**整机故障恢复必须有系统盘之外的恢复密钥。** `backup.key` 不与同一 T5 上的加密数据放在一起。
+用户应单独存入密码管理器或另一载体；未确认前，不能声称“原机完全丢失恢复已验收”。
+
+替换主机恢复：安装 Docker/Python/restic，取回独立密钥，复制 T5 上的回执和本目录工具，调整 config 的新主机路径，
+执行 `restore-check --external --record <T5回执>`。使用回执指定镜像启动该新卷；先在隔离网络验证，再恢复 LAN 端口。
+原 container inspect 位于加密 data.tar 的 `recovery-runtime.json`，用于恢复环境、端口和挂载。
+
+## 版本升级与回退边界
+
+`upgrade.py` 要求同一镜像 digest 已在独立恢复副本上完成 API/Git/SQLite 验证并保存 `upgrade-drill.json`。
+升级前暂停该 Gitea 专属 runner，确认没有运行中 Actions，然后：
+
+```sh
+python3 ~/ops/gitea-reliability/upgrade.py \
+  --config ~/ops/gitea-reliability/config.json \
+  --compose ~/ops/gitea/docker-compose.yml \
+  --image 'docker.io/gitea/gitea:VERSION@sha256:VERIFIED_DIGEST'
+```
+
+过程：获取同一运维锁 → 停 Gitea → 克隆最新卷 → 新版在内部网络且无发布端口运行（禁 Actions/cron/mail）→
+验证数据库/refs → 恢复原 app.ini → 写持久化发布意图 → 切换生产 Compose → 核对实际 image ID、卷和健康。
+旧卷不被迁移或删除，原镜像 ID 和原卷已固定在 rollback compose。
+
+维护事务区分 validating 与 publishing。前者中断恢复旧服务；后者可能已经接受新写入，guard 只向前恢复新卷。
+**新版开放后，不能直接退旧卷，否则会丢失新 push/issue/CI 数据。** 如需回退，先停止写入、保存新版卷并处理增量；
+不得拿旧二进制打开已迁移数据库。所有未完成事务保留 maintenance.json，守卫持续重试。
+
+## CI 试点与隔离
+
+私有试点 `gao/ai-shitiku-ci-pilot` 使用仓库专属 runner `studio-gitea-pilot`，标签 `gitea-macos-arm64`，capacity 1。
+独立系统账号 `_gitea_ci`（UID 490），home `/Users/Shared/gitea-ci`，独立 Flutter/Python/cache/work。
+根所有的 `/Library/Application Support/GiteaCI/runner.sb` 限制访问 `/Users/brunogao` 与 `/Volumes`。
+runner 程序、配置、沙箱不可由 CI 用户写；注册凭据在 CI home 下 0600。
+服务 `/Library/LaunchDaemons/com.ljwx.gitea.ci.pilot.plist`：RunAtLoad、KeepAlive、专属用户。
+模板见 `runner-config.yaml.example`、`runner.sb.example`。act_runner 0.6.0 darwin-arm64；不可复用旧 Intel runner。
+
+macOS `/bin/ps` 自带 setuid，沙箱内禁止启动。CI 专用 bin 安装其不带 setuid 的副本并 ad-hoc 重签名，
+在 runner PATH 前置；系统 `/bin/ps` 不改动，保持真实进程检查，未替换成伪造测试结果。
+Vision 在新账号首次初始化较慢，在脚本检查前用真实空白图预热；不改项目测试超时或断言。
+Flutter 3.35.2 使用独立副本，关闭该 CI 账号遥测。
+
+该 runner 仅供已信任开发者的私有仓库；不是不可信公网 PR 的安全执行环境，仍能访问 LAN。
+正式项目切换需逐库移植 workflow/secrets/status contexts 和发布步骤。GitHub production workflows、账单未在本修复中关闭。
+当前试点验证 `scripts/check`、失败阻断合并、旧 HEAD 拒绝合并；未声称已有生产发布管道已迁移。
+
+## 代码与故障测试
+
+```sh
+python3 -m unittest discover -s gitea/reliability -v
+GITEA_TEST_IMAGE=docker.io/gitea/gitea:1.27.3 python3 -m unittest discover -s gitea/reliability -v
+bash -n gitea/gitea-data-protection.sh gitea/gitea-backup.sh gitea/gitea-restore.sh
+```
+
+测试覆盖空/错误卷、损坏数据库、外置 UUID/路径逃逸、外置快照消失、SIGTERM/SIGKILL/卡死服务恢复、
+升级阶段恢复、错镜像拒绝成功、实际 Git blob 丢失（仅 disposable 测试卷）。
@@ -0,0 +1,513 @@
+#!/usr/bin/env python3
+"""Cold, encrypted Gitea volume backups with independently checked USB copies.
+
+Only the inspected /data volume is read. Restore always creates a NEW volume;
+this tool deliberately has no command which overwrites production data.
+"""
+import argparse
+import datetime as dt
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import plistlib
+import secrets
+import signal
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+import uuid
+
+
+def run(args, **kwargs):
+    return subprocess.run(args, check=True, capture_output=True, **kwargs)
+
+
+def json_write(path, data):
+    path = Path(path)
+    temp = path.with_suffix(path.suffix + ".tmp")
+    with temp.open("w") as stream:
+        stream.write(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
+        stream.flush()
+        os.fsync(stream.fileno())
+    temp.chmod(0o600)
+    temp.replace(path)
+    sync_directory(path.parent)
+
+
+def sync_directory(path):
+    descriptor = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def data_volume(info):
+    mounts = [m for m in info["Mounts"] if m["Destination"] == "/data"]
+    if len(mounts) != 1 or mounts[0]["Type"] != "volume":
+        raise RuntimeError("Expected exactly one named /data volume; refusing fallback")
+    return mounts[0]["Name"]
+
+
+def external_disk(info, expected_uuid):
+    if info.get("Mounted") is False or not info.get("MountPoint") or info.get("Internal", True):
+        raise RuntimeError("Backup destination is not a mounted external device")
+    if info.get("VolumeUUID") != expected_uuid:
+        raise RuntimeError("Backup device UUID mismatch; refusing system-disk fallback")
+    if not info.get("MountPoint", "").startswith("/Volumes/"):
+        raise RuntimeError("Unexpected external mount location")
+    return Path(info["MountPoint"])
+
+
+def external_repository(mount, name):
+    name = Path(name)
+    if name.is_absolute() or ".." in name.parts or not name.parts:
+        raise RuntimeError("Invalid external repository name")
+    mount = Path(mount).resolve()
+    repository = (mount / name).resolve()
+    if not repository.is_relative_to(mount) or repository == mount:
+        raise RuntimeError("External repository escapes the verified disk")
+    return repository
+
+
+def parse_counts(output):
+    rows = output.strip().splitlines()
+    if len(rows) != 2 or rows[0] != "ok":
+        raise RuntimeError("SQLite integrity check failed")
+    keys = ["repositories", "issues", "pull_requests", "users"]
+    values = [int(v) for v in rows[1].split("|")]
+    if len(values) != 4 or values[0] < 1:
+        raise RuntimeError("Empty or invalid Gitea database; refusing success")
+    return dict(zip(keys, values))
+
+
+def backup_window(timestamp):
+    """The local backup day starts at 03:30, independent of run duration."""
+    return (timestamp.astimezone() - dt.timedelta(hours=3, minutes=30)).date()
+
+
+def record_window(stamp):
+    created = dt.datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)
+    return backup_window(created)
+
+
+class Backup:
+    def __init__(self, config):
+        self.c = config
+        self.root = Path(config["state_dir"])
+        self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self.root.chmod(0o700)
+        self.docker = config.get("docker", "/usr/local/bin/docker")
+        self.container = config.get("container", "gitea")
+        self.restic = config["restic"]
+        self.key = Path(config["password_file"])
+        self.local = Path(config["local_repository"])
+        self.image = None
+
+    def log(self, message):
+        print(dt.datetime.now(dt.timezone.utc).isoformat(), message, flush=True)
+
+    def inspect(self):
+        return json.loads(run([self.docker, "inspect", self.container]).stdout)[0]
+
+    def helper(self, volume, command, readonly=True):
+        mount = f"type=volume,src={volume},dst=/data" + (",readonly" if readonly else "")
+        return [self.docker, "run", "--rm", "--network", "none", "--mount", mount,
+                "--entrypoint", command[0], self.image, *command[1:]]
+
+    def inventory(self, volume, full=False):
+        sql = ("PRAGMA integrity_check; SELECT "
+               "(SELECT count(*) FROM repository),"
+               "(SELECT count(*) FROM issue),"
+               "(SELECT count(*) FROM pull_request),"
+               "(SELECT count(*) FROM user);")
+        counts = parse_counts(run(self.helper(volume, ["sqlite3", "-readonly",
+                              "/data/gitea/gitea.db", sql]), text=True, timeout=60).stdout)
+        # Hash refs rather than printing private repository or branch names.
+        script = r'''set -eu
+test -s /data/gitea/conf/app.ini
+find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git' | sort > /tmp/repositories
+test -s /tmp/repositories
+while IFS= read -r repo; do
+  printf '%s\n' "$repo"
+  git -c safe.directory='*' --git-dir="$repo" for-each-ref --format='%(objectname) %(refname)'
+done < /tmp/repositories
+'''
+        refs = run(self.helper(volume, ["/bin/sh", "-c", script]), timeout=120).stdout
+        counts["refs_sha256"] = hashlib.sha256(refs).hexdigest()
+        if full:
+            check = r'''set -eu
+find /data/git/repositories -mindepth 2 -maxdepth 2 -type d -name '*.git' > /tmp/repositories
+while IFS= read -r repo; do
+  git -c safe.directory='*' --git-dir="$repo" fsck --full --no-dangling >/dev/null
+done < /tmp/repositories
+'''
+            run(self.helper(volume, ["/bin/sh", "-c", check]), timeout=3600)
+        return counts
+
+    def check_external(self):
+        mount = self.c["external_mount"]
+        if not Path(mount).is_dir():
+            raise RuntimeError("External backup disk is absent")
+        info = plistlib.loads(run(["/usr/sbin/diskutil", "info", "-plist", mount], timeout=15).stdout)
+        actual = external_disk(info, self.c["external_uuid"])
+        if not actual.is_mount():
+            raise RuntimeError("External destination is not a live mount point")
+        return external_repository(actual, self.c.get("external_repository_name", "gitea-restic"))
+
+    def restic_cmd(self, repository, *args):
+        return [self.restic, "--repo", str(repository), "--password-file", str(self.key),
+                "--cache-dir", str(self.root / "restic-cache"), *args]
+
+    def init_repository(self, repository):
+        if not self.key.exists():
+            if (Path(repository) / "config").exists() or (self.local / "config").exists():
+                raise RuntimeError("Existing repository but recovery key missing")
+            self.key.write_text(secrets.token_hex(32) + "\n")
+            self.key.chmod(0o600)
+        if not (Path(repository) / "config").exists():
+            Path(repository).mkdir(parents=True, exist_ok=True, mode=0o700)
+            run(self.restic_cmd(repository, "init"))
+
+    def health(self):
+        with urllib.request.urlopen(self.c["health_url"], timeout=5) as response:
+            data = json.load(response)
+        if data.get("status") != "pass":
+            raise RuntimeError("Gitea health did not pass")
+
+    def start_and_wait(self):
+        run([self.docker, "start", self.container], timeout=120)
+        for _ in range(30):
+            try:
+                self.health()
+                return
+            except (OSError, ValueError, RuntimeError):
+                time.sleep(1)
+        raise RuntimeError("Gitea failed to recover after snapshot; inspect service immediately")
+
+    def recover_interrupted_snapshot(self):
+        marker = self.root / "maintenance.json"
+        if marker.exists():
+            data = json.loads(marker.read_text())
+            if data.get("container") != self.container:
+                raise RuntimeError("Unexpected maintenance marker")
+            if data.get("kind") == "upgrade":
+                from upgrade import recover_transaction
+                recover_transaction(self, data)
+                return
+            self.log("Recovering Gitea after an interrupted snapshot")
+            helper = data.get("copy_container")
+            if helper:
+                if not helper.startswith("gitea-snapshot-copy-"):
+                    raise RuntimeError("Unexpected snapshot helper name")
+                subprocess.run([self.docker, "rm", "-f", helper], capture_output=True, timeout=60)
+            self.start_and_wait()
+            marker.unlink()
+
+    def image_snapshot(self, image):
+        output = run(self.restic_cmd(self.local, "backup", "--json", "--host", "gitea-studio",
+                     "--tag", "gitea-recovery-image", "--stdin-filename", "image.tar",
+                     "--stdin-from-command", "--", self.docker, "image", "save", image), text=True).stdout
+        return next(json.loads(line)["snapshot_id"] for line in output.splitlines()
+                    if line.startswith("{") and json.loads(line).get("message_type") == "summary")
+
+    def copied_snapshot(self, repository, snapshot):
+        snapshots = json.loads(run(self.restic_cmd(repository, "snapshots", "--json"), text=True).stdout)
+        candidates = [s for s in snapshots if s.get("original", "").startswith(snapshot)
+                      or s["id"].startswith(snapshot)]
+        if len(candidates) != 1:
+            raise RuntimeError("Cannot uniquely identify copied snapshot")
+        return candidates[0]["id"]
+
+    def stream(self, source, target):
+        reader = subprocess.Popen(source, stdout=subprocess.PIPE)
+        try:
+            writer = subprocess.Popen(target, stdin=reader.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except BaseException:
+            reader.stdout.close()
+            reader.terminate()
+            reader.wait()
+            raise
+        reader.stdout.close()
+        try:
+            writer.communicate()
+            code = reader.wait()
+        except BaseException:
+            reader.terminate()
+            writer.terminate()
+            reader.wait()
+            writer.wait()
+            raise
+        if code or writer.returncode:
+            raise RuntimeError(f"Restore stream failed: source={code}, target={writer.returncode}")
+
+    def backup(self):
+        self.recover_interrupted_snapshot()
+        if shutil.disk_usage(self.root).free < self.c.get("minimum_free_bytes", 64 * 1024**3):
+            raise RuntimeError("Less than reserved free disk space; refusing snapshot")
+        # Retain a valid local backup even when the USB disk is unplugged, but
+        # never report the whole operation successful without an external copy.
+        external = None
+        external_error = None
+        try:
+            external = self.check_external()
+        except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
+            external_error = type(exc).__name__ + ": external disk unavailable"
+        info = self.inspect()
+        self.image = info["Image"]
+        source = data_volume(info)
+        if not info["State"]["Running"]:
+            raise RuntimeError("Gitea is not running; refusing to conceal an outage")
+        active = run([self.docker, "exec", self.container, "sqlite3", "-readonly",
+                      "/data/gitea/gitea.db", "SELECT count(*) FROM action_task WHERE status=6;"],
+                     text=True).stdout.strip()
+        if active != "0":
+            raise RuntimeError("CI job is running; retry backup when idle")
+        stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        stage = f"gitea-backup-stage-{stamp.lower()}-{uuid.uuid4().hex[:6]}"
+        record = {"created_utc": stamp, "source_volume": source, "image": self.image,
+                  "image_reference": info["Config"]["Image"], "stage_volume": stage,
+                  "external_verified": False}
+        record_file = self.root / f"backup-{stamp}.json"
+        run([self.docker, "volume", "create", "--label", "gitea.reliability=backup-stage", stage])
+        copy_container = "gitea-snapshot-copy-" + uuid.uuid4().hex[:12]
+        json_write(self.root / "maintenance.json", {"container": self.container,
+                   "started": time.time(), "stage": stage, "copy_container": copy_container})
+        started = time.monotonic()
+        try:
+            self.log("Stopping only Gitea for a consistent snapshot")
+            run([self.docker, "stop", "--time", "30", self.container], timeout=120)
+            command = [self.docker, "run", "--rm", "--name", copy_container, "--network", "none",
+                       "--mount", f"type=volume,src={source},dst=/source,readonly",
+                       "--mount", f"type=volume,src={stage},dst=/snapshot",
+                       "--entrypoint", "/bin/sh", self.image, "-c",
+                       "set -eu; test -s /source/gitea/gitea.db; "
+                       "test -s /source/gitea/conf/app.ini; cp -a /source/. /snapshot/"]
+            run(command, timeout=self.c.get("snapshot_timeout_seconds", 600))
+        finally:
+            # Stop any detached copy before allowing production writes again.
+            try:
+                subprocess.run([self.docker, "rm", "-f", copy_container], capture_output=True, timeout=60)
+            finally:
+                self.start_and_wait()
+            (self.root / "maintenance.json").unlink(missing_ok=True)
+        record["service_pause_seconds"] = round(time.monotonic() - started, 2)
+        self.log(f"Gitea healthy again; snapshot pause {record['service_pause_seconds']} seconds")
+        record["inventory"] = self.inventory(stage)
+        # Raw container configuration may contain secrets. It goes only inside
+        # the encrypted archive, never into the public receipt or source repo.
+        private = self.root / "recovery-runtime.json"
+        json_write(private, info)
+        helper = f"gitea-backup-metadata-{uuid.uuid4().hex[:8]}"
+        run([self.docker, "create", "--name", helper, "--network", "none", "--mount",
+             f"type=volume,src={stage},dst=/data", "--entrypoint", "/bin/true", self.image])
+        try:
+            run([self.docker, "cp", str(private), f"{helper}:/data/recovery-runtime.json"])
+        finally:
+            run([self.docker, "rm", helper])
+            private.unlink(missing_ok=True)
+        self.init_repository(self.local)
+        self.log("Encrypting snapshot into local restic repository")
+        output = run(self.restic_cmd(self.local, "backup", "--json", "--host", "gitea-studio",
+                     "--tag", "gitea-volume", "--stdin-filename", "data.tar", "--stdin-from-command", "--",
+                     *self.helper(stage, ["/bin/tar", "cf", "-", "-C", "/data", "."])), text=True).stdout
+        summaries = [json.loads(s) for s in output.splitlines() if s.startswith("{")]
+        summary = next(s for s in summaries if s.get("message_type") == "summary")
+        record["snapshot_id"] = summary["snapshot_id"]
+        record["bytes_processed"] = summary.get("total_bytes_processed")
+        record["image_snapshot_id"] = self.image_snapshot(self.image)
+        run(self.restic_cmd(self.local, "check"))
+        json_write(record_file, record)
+        self.log("Local encrypted snapshot verified: " + record["snapshot_id"])
+        # Local data is durable now. A failed USB copy can be retried without
+        # another service interruption using the replicate command.
+        run([self.docker, "volume", "rm", stage])
+        if external is None:
+            record["external_error"] = external_error
+            json_write(record_file, record)
+            raise RuntimeError("Local backup succeeded; external copy FAILED")
+        return self.replicate(record_file)
+
+    def scheduled(self, now=None):
+        """Retry a verified local copy first; take at most one cold copy per backup day."""
+        self.recover_interrupted_snapshot()
+        window = backup_window(now or dt.datetime.now(dt.timezone.utc))
+        success_file = self.root / "last-success.json"
+        previous = json.loads(success_file.read_text()) if success_file.exists() else None
+        previous_stamp = previous["created_utc"] if previous else ""
+        for record_file in sorted(self.root.glob("backup-*.json"), reverse=True):
+            record = json.loads(record_file.read_text())
+            if record["created_utc"] <= previous_stamp:
+                break
+            # A failed copy from an older window must not prevent a fresh
+            # local snapshot today when the external disk remains unavailable.
+            if record_window(record["created_utc"]) != window:
+                break
+            if record.get("snapshot_id") and record.get("image_snapshot_id"):
+                # Also repair a crash after marking the record verified but
+                # before publishing last-success and the USB receipt.
+                self.replicate(record_file)
+                self.status()
+                self.log("PASS: recovered the current backup window without another service pause")
+                return
+        previous = json.loads(success_file.read_text()) if success_file.exists() else None
+        if previous and previous.get("external_verified") and record_window(previous["created_utc"]) == window:
+            self.status()
+            self.log("PASS: current backup window already verified")
+            return
+        self.backup()
+
+    def replicate(self, record_file):
+        record = json.loads(Path(record_file).read_text())
+        external = self.check_external()
+        self.init_repository(external)
+        if "image_snapshot_id" not in record:
+            record["image_snapshot_id"] = self.image_snapshot(record["image"])
+            json_write(record_file, record)
+        self.log("Copying encrypted snapshots to external device")
+        run(self.restic_cmd(external, "copy", "--from-repo", str(self.local),
+                           "--from-password-file", str(self.key), record["snapshot_id"], record["image_snapshot_id"]))
+        run(self.restic_cmd(external, "check", "--read-data"))
+        self.check_external()
+        record["external_repository"] = str(external)
+        record["external_snapshot_id"] = self.copied_snapshot(external, record["snapshot_id"])
+        record["external_image_snapshot_id"] = self.copied_snapshot(external, record["image_snapshot_id"])
+        record["external_verified"] = True
+        record.pop("external_error", None)
+        json_write(record_file, record)
+        # Receipt is non-secret. Keep it with the USB repository so a failed
+        # system disk cannot also destroy the only recovery instructions.
+        receipts = external.parent / (external.name + "-recovery")
+        if receipts.is_symlink():
+            raise RuntimeError("Recovery receipt directory must not be a symlink")
+        receipts.mkdir(mode=0o700, exist_ok=True)
+        json_write(receipts / Path(record_file).name, record)
+        success_file = self.root / "last-success.json"
+        previous = json.loads(success_file.read_text()) if success_file.exists() else None
+        latest = record if not previous or record["created_utc"] >= previous["created_utc"] else previous
+        json_write(success_file, latest)
+        self.log("PASS: local backup and full external data verification")
+        self.retention(external, latest["snapshot_id"])
+        return record
+
+    def retention(self, external, current_snapshot=None):
+        # Only data snapshots managed by this tool are expired. Recovery-image
+        # snapshots remain, so an older retained data archive never loses its
+        # required image. Keep the actual restored snapshot as a recovery anchor.
+        restored_file = self.root / "last-restore.json"
+        if not restored_file.exists():
+            return
+        anchor = json.loads(restored_file.read_text())["source_snapshot"]
+        for repository in (self.local, external):
+            anchor_id = self.copied_snapshot(repository, anchor)
+            protected = {anchor_id}
+            if current_snapshot:
+                protected.add(self.copied_snapshot(repository,current_snapshot))
+            snapshots = json.loads(run(self.restic_cmd(repository,"snapshots","--json"),text=True).stdout)
+            for snapshot in snapshots:
+                if snapshot["id"] == anchor_id and "gitea-volume" not in snapshot.get("tags",[]):
+                    raise RuntimeError("Recovery anchor is not a managed data snapshot")
+            # Tags would rewrite snapshot IDs. Preserve the restored anchor
+            # by removing it from the dry-run deletion selection instead.
+            result = json.loads(run(self.restic_cmd(repository,"forget","--dry-run","--json",
+                              "--host","gitea-studio","--tag","gitea-volume",
+                              "--keep-daily","7","--keep-weekly","4","--keep-monthly","3"),text=True).stdout)
+            remove = [snap["id"] for group in result for snap in (group.get("remove") or [])
+                      if snap["id"] not in protected]
+            if remove:
+                run(self.restic_cmd(repository,"forget",*remove))
+                run(self.restic_cmd(repository,"prune"))
+
+    def restore(self, record_file, external=False):
+        record = json.loads(Path(record_file).read_text())
+        self.image = record["image"]
+        repository = self.check_external() if external else self.local
+        snapshot = record["snapshot_id"]
+        if external:
+            snapshot = self.copied_snapshot(repository, snapshot)
+        image_snapshot = record["image_snapshot_id"]
+        if external:
+            image_snapshot = self.copied_snapshot(repository, image_snapshot)
+        # Always exercise image recovery, including on a host with a warm cache.
+        self.stream(self.restic_cmd(repository, "dump", image_snapshot, "/image.tar"),
+                    [self.docker, "image", "load"])
+        run([self.docker, "image", "inspect", self.image])
+        target = "gitea-restore-check-" + uuid.uuid4().hex[:12]
+        run([self.docker, "volume", "create", "--label", "gitea.reliability=restore-check", target])
+        args = self.helper(target, ["/bin/tar", "xf", "-", "-C", "/data"], readonly=False)
+        args.insert(3, "-i")
+        self.stream(self.restic_cmd(repository, "dump", snapshot, "/data.tar"), args)
+        actual = self.inventory(target, full=True)
+        if actual != record["inventory"]:
+            raise RuntimeError("Restored inventory or Git refs differ from snapshot")
+        receipt = {"source_snapshot": record["snapshot_id"], "external": external,
+                   "volume": target, "image": self.image, "inventory": actual,
+                   "verified_utc": dt.datetime.now(dt.timezone.utc).isoformat()}
+        json_write(self.root / "last-restore.json", receipt)
+        self.log("PASS: restored SQLite and every repository ref match: " + target)
+        return receipt
+
+    def status(self):
+        self.health()
+        external = self.check_external()
+        record = json.loads((self.root / "last-success.json").read_text())
+        if not record.get("external_verified"):
+            raise RuntimeError("Last backup has no verified external copy")
+        created = dt.datetime.strptime(record["created_utc"], "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)
+        if (dt.datetime.now(dt.timezone.utc) - created).total_seconds() > 36 * 3600:
+            raise RuntimeError("Last verified backup is older than 36 hours")
+        self.copied_snapshot(external, record["snapshot_id"])
+        self.copied_snapshot(external, record["image_snapshot_id"])
+        return record
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", default=str(Path.home() / "ops/gitea-reliability/config.json"))
+    parser.add_argument("command", choices=["backup", "scheduled", "replicate", "restore-check", "status", "guard"])
+    parser.add_argument("--record")
+    parser.add_argument("--external", action="store_true")
+    args = parser.parse_args()
+    obj = Backup(json.loads(Path(args.config).read_text()))
+    if args.command == "status":
+        print(json.dumps(obj.status(), ensure_ascii=False, indent=2))
+        return
+    with (obj.root / "backup.lock").open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            if args.command == "guard":
+                return
+            raise RuntimeError("Another backup or restore check is already running") from None
+        if args.command == "guard":
+            obj.recover_interrupted_snapshot()
+            obj.health()
+        elif args.command == "backup":
+            obj.backup()
+        elif args.command == "scheduled":
+            obj.scheduled()
+        elif args.command == "replicate":
+            if not args.record:
+                raise RuntimeError("replicate requires --record")
+            obj.replicate(args.record)
+        else:
+            obj.restore(args.record or obj.root / "last-success.json", args.external)
+
+
+if __name__ == "__main__":
+    os.umask(0o077)
+    def terminate(signum, frame):
+        raise SystemExit(128 + signum)
+    signal.signal(signal.SIGTERM, terminate)
+    signal.signal(signal.SIGINT, terminate)
+    try:
+        main()
+    except Exception as exc:
+        # Do not print command arguments or subprocess stderr: they can include
+        # private paths, credential-bearing config, or repository names.
+        print(f"FAIL: {type(exc).__name__}: {exc if not isinstance(exc, subprocess.SubprocessError) else 'subprocess failed; inspect private logs'}", file=sys.stderr)
+        sys.exit(1)
@@ -0,0 +1,12 @@
+{
+  "state_dir": "/Users/brunogao/ops/gitea-reliability",
+  "container": "gitea",
+  "docker": "/usr/local/bin/docker",
+  "restic": "/Users/brunogao/ops/gitea-reliability/bin/restic",
+  "password_file": "/Users/brunogao/ops/gitea-reliability/backup.key",
+  "local_repository": "/Users/brunogao/ops/backups/gitea-restic",
+  "external_mount": "/Volumes/t5",
+  "external_uuid": "REPLACE_WITH_VERIFIED_EXTERNAL_VOLUME_UUID",
+  "external_repository_name": "gitea-restic",
+  "health_url": "http://127.0.0.1:33000/api/healthz"
+}
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Install only Gitea backup/guard user LaunchAgents, using an existing config."""
+import argparse
+import json
+import os
+from pathlib import Path
+import plistlib
+import shutil
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", default=str(Path.home() / "ops/gitea-reliability/config.json"))
+    args = parser.parse_args()
+    config_path = Path(args.config).resolve()
+    config = json.loads(config_path.read_text())
+    root = Path(config["state_dir"])
+    if root != config_path.parent:
+        raise SystemExit("config must reside in state_dir")
+    root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    root.chmod(0o700)
+    source = Path(__file__).with_name("backup.py")
+    target = root / "backup.py"
+    if source.resolve() != target.resolve():
+        shutil.copy2(source, target)
+    target.chmod(0o700)
+    upgrade = Path(__file__).with_name("upgrade.py")
+    if upgrade.resolve() != (root / "upgrade.py").resolve():
+        shutil.copy2(upgrade, root / "upgrade.py")
+    (root / "upgrade.py").chmod(0o700)
+    config_path.chmod(0o600)
+    log = root / "logs"
+    log.mkdir(mode=0o700, exist_ok=True)
+    agents = Path.home() / "Library/LaunchAgents"
+    agents.mkdir(parents=True, exist_ok=True)
+    domain = f"gui/{os.getuid()}"
+    jobs = {
+        "com.ljwx.gitea.backup": ("scheduled", {"StartCalendarInterval":[{"Hour":hour,"Minute":30} for hour in range(24)]}),
+        "com.ljwx.gitea.watchdog": ("guard", {"StartInterval":60,"RunAtLoad":True}),
+        "com.ljwx.gitea.backup-audit": ("status", {"StartInterval":3600,"RunAtLoad":True}),
+    }
+    for label, (command, schedule) in jobs.items():
+        plist = agents / (label + ".plist")
+        if plist.exists():
+            saved = root / (label + ".previous.plist")
+            if not saved.exists():
+                shutil.copy2(plist, saved)
+        subprocess.run(["launchctl", "bootout", f"{domain}/{label}"], capture_output=True)
+        payload = {"Label":label,"ProgramArguments":[sys.executable,str(target),"--config",str(config_path),command],
+                   "WorkingDirectory":str(root), "EnvironmentVariables":{"HOME":str(Path.home()),"PATH":"/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"},
+                   "StandardOutPath":str(log / (command + ".log")),"StandardErrorPath":str(log / (command + ".err.log")),
+                   **schedule}
+        plist.write_bytes(plistlib.dumps(payload))
+        plist.chmod(0o644)
+        subprocess.run(["plutil", "-lint", str(plist)], check=True)
+        subprocess.run(["launchctl", "bootstrap", domain, str(plist)], check=True)
+    print("Installed 03:30 backup with hourly retries, 60s recovery guard, hourly backup audit (login session required)")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,27 @@
+log:
+  level: info
+runner:
+  file: /Users/Shared/gitea-ci/.runner
+  capacity: 1
+  timeout: 30m
+  shutdown_timeout: 30s
+  fetch_timeout: 5s
+  fetch_interval: 5s
+  envs:
+    HOME: /Users/Shared/gitea-ci
+    PATH: /Library/Application Support/GiteaCI/bin:/Users/Shared/gitea-ci/toolchains/flutter/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin
+    PUB_CACHE: /Users/Shared/gitea-ci/cache/pub
+    PIP_CACHE_DIR: /Users/Shared/gitea-ci/cache/pip
+  labels:
+    - gitea-macos-arm64:host
+cache:
+  enabled: true
+  dir: /Users/Shared/gitea-ci/cache/actions
+  host: 127.0.0.1
+  port: 18090
+container:
+  privileged: false
+  valid_volumes: []
+  docker_host: '-'
+host:
+  workdir_parent: /Users/Shared/gitea-ci/work
@@ -0,0 +1,4 @@
+(version 1)
+(allow default)
+(deny file-read* file-write* (subpath "/Users/brunogao"))
+(deny file-read* file-write* (subpath "/Volumes"))
@@ -0,0 +1,202 @@
+import unittest
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+import json
+import datetime as dt
+from backup import Backup, backup_window, data_volume, external_disk, external_repository, parse_counts
+
+
+class BackupSafetyTests(unittest.TestCase):
+    def test_scheduled_backup_uses_local_0330_window(self):
+        zone = dt.datetime.now().astimezone().tzinfo
+        start = dt.datetime(2026, 9, 26, 3, 30, tzinfo=zone)
+        self.assertNotEqual(backup_window(start - dt.timedelta(seconds=1)), backup_window(start))
+        self.assertEqual(backup_window(start), backup_window(start + dt.timedelta(hours=20)))
+
+    def test_scheduled_backup_does_not_repeat_a_verified_cold_copy(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            (root / "last-success.json").write_text(json.dumps({"created_utc": now.strftime("%Y%m%dT%H%M%SZ"), "external_verified": True}))
+            with patch.object(obj, "status") as status, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                status.assert_called_once()
+                backup.assert_not_called()
+
+    def test_scheduled_backup_retries_external_copy_without_service_pause(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            current = now.strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            pending = root / f"backup-{current}.json"
+            pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False}))
+            with patch.object(obj, "replicate") as replicate, patch.object(obj, "status") as status, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                replicate.assert_called_once_with(pending)
+                status.assert_called_once()
+                backup.assert_not_called()
+
+    def test_older_failed_external_copy_does_not_block_todays_local_backup(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=2)).strftime("%Y%m%dT%H%M%SZ")
+            stale = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            (root / f"backup-{stale}.json").write_text(json.dumps({"created_utc": stale, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False}))
+            with patch.object(obj, "replicate", side_effect=RuntimeError("USB unavailable")) as replicate, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                replicate.assert_not_called()
+                backup.assert_called_once()
+
+    def test_current_verified_record_without_published_receipt_avoids_new_outage(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            current = now.strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            pending = root / f"backup-{current}.json"
+            pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": True}))
+            with patch.object(obj, "replicate") as replicate, patch.object(obj, "status") as status, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                replicate.assert_called_once_with(pending)
+                status.assert_called_once()
+                backup.assert_not_called()
+
+    def test_scheduled_backup_retries_after_busy_ci(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            with patch.object(obj, "backup", side_effect=[RuntimeError("CI running"), None]) as backup:
+                with self.assertRaisesRegex(RuntimeError, "CI running"):
+                    obj.scheduled(now)
+                obj.scheduled(now + dt.timedelta(hours=1))
+                self.assertEqual(backup.call_count, 2)
+
+    def test_scheduled_retry_recovers_interrupted_service_before_external_copy(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            current = now.strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            pending = root / f"backup-{current}.json"
+            pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False}))
+            (root / "maintenance.json").write_text(json.dumps({"container": "gitea"}))
+            with patch.object(obj, "start_and_wait") as start, patch.object(obj, "replicate") as replicate, patch.object(obj, "status"):
+                obj.scheduled(now)
+                start.assert_called_once()
+                replicate.assert_called_once_with(pending)
+            self.assertFalse((root / "maintenance.json").exists())
+
+    def test_actual_compose_volume_is_used(self):
+        self.assertEqual(data_volume({"Mounts": [{"Destination": "/data", "Type": "volume", "Name": "gitea_gitea-data"}]}), "gitea_gitea-data")
+
+    def test_missing_volume_never_falls_back_to_empty_named_volume(self):
+        with self.assertRaises(RuntimeError):
+            data_volume({"Mounts": []})
+
+    def test_unknown_bind_mount_is_rejected(self):
+        with self.assertRaises(RuntimeError):
+            data_volume({"Mounts": [{"Destination": "/data", "Type": "bind"}]})
+
+    def test_empty_backup_cannot_pass(self):
+        with self.assertRaises(RuntimeError):
+            parse_counts("ok\n0|0|0|0\n")
+
+    def test_corrupt_database_cannot_pass(self):
+        with self.assertRaises(RuntimeError):
+            parse_counts("database disk image is malformed\n207|1|1|3\n")
+
+    def test_real_inventory(self):
+        self.assertEqual(parse_counts("ok\n207|15|10|5\n")["repositories"], 207)
+
+    def test_internal_disk_is_rejected_even_with_matching_uuid(self):
+        with self.assertRaises(RuntimeError):
+            external_disk({"Mounted": True, "Internal": True, "VolumeUUID": "expected"}, "expected")
+
+    def test_wrong_external_disk_is_rejected(self):
+        with self.assertRaises(RuntimeError):
+            external_disk({"Mounted": True, "Internal": False, "VolumeUUID": "wrong", "MountPoint": "/Volumes/t5"}, "expected")
+
+    def test_expected_external_disk(self):
+        self.assertEqual(str(external_disk({"Mounted": True, "Internal": False, "VolumeUUID": "expected", "MountPoint": "/Volumes/t5"}, "expected")), "/Volumes/t5")
+
+    def test_macos_diskutil_omits_mounted_field(self):
+        self.assertEqual(str(external_disk({"Internal": False, "VolumeUUID": "expected", "MountPoint": "/Volumes/t5"}, "expected")), "/Volumes/t5")
+
+    def test_unmounted_external_disk_is_rejected(self):
+        with self.assertRaises(RuntimeError):
+            external_disk({"Internal": False, "VolumeUUID": "expected"}, "expected")
+
+    def test_external_path_escape(self):
+        for name in ["/tmp/backup", "../backup"]:
+            with self.assertRaises(RuntimeError):
+                external_repository("/Volumes/t5", name)
+
+    def test_external_symlink_escape(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            mount = root / "usb"
+            mount.mkdir()
+            (mount / "backup").symlink_to(root)
+            with self.assertRaises(RuntimeError):
+                external_repository(mount, "backup")
+
+    def test_retention_handles_no_expired_snapshots(self):
+        import subprocess
+        with tempfile.TemporaryDirectory() as directory:
+            root=Path(directory)
+            obj=Backup({"state_dir":directory,"restic":"unused","password_file":str(root/"key"),"local_repository":str(root/"repo")})
+            (root/"last-restore.json").write_text('{"source_snapshot":"anchor"}')
+            snapshots=subprocess.CompletedProcess([],0,stdout='[{"id":"anchor","tags":["gitea-volume"]}]')
+            selection=subprocess.CompletedProcess([],0,stdout='[{"remove":null}]')
+            with patch.object(obj,"copied_snapshot",return_value="anchor"),patch("backup.run",side_effect=[snapshots,selection,snapshots,selection]) as run:
+                obj.retention(root/"external")
+                self.assertEqual(run.call_count,4)
+
+    def test_deleted_external_snapshot_cannot_report_healthy(self):
+        import datetime
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir":directory,"restic":"unused","password_file":str(root / "key"),"local_repository":str(root / "repo")})
+            (root / "last-success.json").write_text(json.dumps({"external_verified":True,"created_utc":datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ"),"snapshot_id":"data","image_snapshot_id":"image"}))
+            with patch.object(obj,"health"), patch.object(obj,"check_external",return_value=root / "deleted"), patch.object(obj,"copied_snapshot",side_effect=RuntimeError("snapshot missing")):
+                with self.assertRaises(RuntimeError):
+                    obj.status()
+
+    def test_interrupted_snapshot_recovers_before_new_backup(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "restic", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            marker = root / "maintenance.json"
+            marker.write_text(json.dumps({"container": "gitea"}))
+            with patch.object(obj, "start_and_wait") as start:
+                obj.recover_interrupted_snapshot()
+                start.assert_called_once()
+            self.assertFalse(marker.exists())
+
+    def test_failed_recovery_keeps_marker_for_guard_retry(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "restic", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            marker = root / "maintenance.json"
+            marker.write_text(json.dumps({"container": "gitea"}))
+            with patch.object(obj, "start_and_wait", side_effect=RuntimeError("not ready")):
+                with self.assertRaises(RuntimeError):
+                    obj.recover_interrupted_snapshot()
+            self.assertTrue(marker.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,51 @@
+"""Optional destructive-object regression in a disposable Docker volume only."""
+import os
+import subprocess
+import tempfile
+import unittest
+import uuid
+from backup import Backup
+
+
+@unittest.skipUnless(os.environ.get("GITEA_TEST_IMAGE"), "set GITEA_TEST_IMAGE to run disposable Docker test")
+class GitObjectTests(unittest.TestCase):
+    def test_missing_blob_rejected_even_when_refs_match(self):
+        with tempfile.TemporaryDirectory() as root:
+            obj = Backup({"state_dir":root,"restic":"unused","password_file":root+"/key","local_repository":root+"/restic"})
+            obj.image = os.environ["GITEA_TEST_IMAGE"]
+            volume = "gitea-object-test-" + uuid.uuid4().hex[:10]
+            subprocess.run([obj.docker,"volume","create",volume],check=True,capture_output=True)
+            try:
+                seed = r'''set -eu
+mkdir -p /data/gitea/conf /data/git/repositories/test
+printf 'test\n' > /data/gitea/conf/app.ini
+sqlite3 /data/gitea/gitea.db 'CREATE TABLE repository(id); INSERT INTO repository VALUES(1); CREATE TABLE issue(id); CREATE TABLE pull_request(id); CREATE TABLE user(id);'
+git init -q /tmp/seed
+cd /tmp/seed
+printf 'critical blob\n' > file
+git add file
+git -c [email protected] -c user.name=test commit -qm initial
+git clone -q --bare --no-local /tmp/seed /data/git/repositories/test/repo.git
+git rev-parse HEAD:file > /data/blob-id
+'''
+                subprocess.run(obj.helper(volume,["/bin/sh","-c",seed],readonly=False),check=True,capture_output=True)
+                healthy = obj.inventory(volume,full=True)
+                remove = r'''set -eu
+blob=$(cat /data/blob-id)
+# A bare clone can pack objects; unpack then remove precisely the fixture blob.
+repo=/data/git/repositories/test/repo.git
+for pack in "$repo"/objects/pack/*.pack; do
+  test -f "$pack" || continue
+  mv "$pack" /tmp/input.pack
+  git -c safe.directory='*' --git-dir="$repo" unpack-objects < /tmp/input.pack
+done
+prefix=$(printf '%s' "$blob" | cut -c1-2)
+suffix=$(printf '%s' "$blob" | cut -c3-)
+rm "$repo/objects/$prefix/$suffix"
+'''
+                subprocess.run(obj.helper(volume,["/bin/sh","-c",remove],readonly=False),check=True,capture_output=True)
+                self.assertEqual(obj.inventory(volume),healthy)
+                with self.assertRaises(subprocess.CalledProcessError):
+                    obj.inventory(volume,full=True)
+            finally:
+                subprocess.run([obj.docker,"volume","rm",volume],check=True,capture_output=True)
@@ -0,0 +1,83 @@
+"""Real-process fault injection; fake Docker never touches production."""
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+TOOL = Path(__file__).with_name("backup.py")
+
+
+class ProcessRecoveryTests(unittest.TestCase):
+    def exercise_signal(self, sig):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            docker = root / "docker"
+            docker.write_text("#!" + sys.executable + "\n" + r'''
+import json, pathlib, sys, time
+root = pathlib.Path(__file__).parent
+args = sys.argv[1:]
+if args[0] == "inspect":
+    print(json.dumps([{"Image":"image", "Config":{"Image":"image"}, "State":{"Running":True}, "Mounts":[{"Destination":"/data","Type":"volume","Name":"test-only"}]}]))
+elif args[0] == "exec":
+    print("0")
+elif args[0] == "stop":
+    (root / "stopped").write_text("yes")
+elif args[0] == "start":
+    (root / "started").write_text("yes")
+elif args[0] == "run":
+    (root / "copying").write_text("yes")
+    time.sleep(30)
+''')
+            docker.chmod(0o700)
+            health = root / "health.json"
+            health.write_text('{"status":"pass"}')
+            config = root / "config.json"
+            config.write_text(json.dumps({
+                "state_dir":str(root), "docker":str(docker), "restic":"unused",
+                "password_file":str(root / "key"), "local_repository":str(root / "repo"),
+                "external_mount":str(root / "absent"), "health_url":health.as_uri(),
+                "snapshot_timeout_seconds": 1,
+            }))
+            command = [sys.executable, str(TOOL), "--config", str(config)]
+            process = subprocess.Popen(command + ["backup"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            try:
+                deadline = time.monotonic() + 10
+                while not (root / "copying").exists() and time.monotonic() < deadline:
+                    time.sleep(0.02)
+                self.assertTrue((root / "stopped").exists())
+                self.assertTrue((root / "copying").exists())
+                if sig is not None:
+                    process.send_signal(sig)
+                process.wait(timeout=10)
+                self.assertNotEqual(process.returncode, 0)
+                # SIGTERM must recover in finally; SIGKILL uses independent guard.
+                if sig != signal.SIGKILL:
+                    self.assertTrue((root / "started").exists())
+                else:
+                    self.assertTrue((root / "maintenance.json").exists())
+                subprocess.run(command + ["guard"], check=True, capture_output=True)
+                self.assertTrue((root / "started").exists())
+                self.assertFalse((root / "maintenance.json").exists())
+                self.assertFalse((root / "last-success.json").exists())
+            finally:
+                if process.poll() is None:
+                    process.kill()
+                    process.wait()
+
+    def test_hung_copy_times_out_and_recovers_service(self):
+        self.exercise_signal(None)
+
+    def test_sigterm_recovers_service(self):
+        self.exercise_signal(signal.SIGTERM)
+
+    def test_sigkill_guard_recovers_service(self):
+        self.exercise_signal(signal.SIGKILL)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+import tempfile
+import time
+import subprocess
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+from backup import Backup, json_write
+from upgrade import compose_data_key, recover_transaction, perform_upgrade
+
+
+class UpgradeRecoveryTests(unittest.TestCase):
+    def test_actual_mount_key_is_used(self):
+        config={"services":{"gitea":{"volumes":["actual:/data","/etc/localtime:/etc/localtime:ro"]}},"volumes":{"actual":{}}}
+        self.assertEqual(compose_data_key(config),"actual")
+
+    def test_unknown_mount_is_rejected(self):
+        with self.assertRaises(RuntimeError):
+            compose_data_key({"services":{"gitea":{"volumes":["/tmp/data:/data"]}},"volumes":{}})
+
+    def exercise_phase(self,phase,wrong_image=False):
+        with tempfile.TemporaryDirectory() as directory:
+            root=Path(directory)
+            obj=Backup({"state_dir":directory,"restic":"unused","password_file":str(root/"key"),"local_repository":str(root/"restic")})
+            old=root/"old.yml";old.write_text("old pinned image and old volume")
+            new=root/"new.yml";new.write_text("new pinned image and new volume")
+            compose=root/"compose.yml";compose.write_text("interrupted publication")
+            record=root/"record.json";json_write(record,{"isolated_validation":True})
+            marker={"kind":"upgrade","phase":phase,"container":"gitea","compose":str(compose),"rollback_compose":str(old),"candidate_compose":str(new),"record_file":str(record),"old_volume":"old","new_volume":"new","old_image_id":"old-image","new_image_id":"new-image","started":time.time()}
+            json_write(root/"maintenance.json",marker)
+            expected="new" if phase=="publishing" else "old"
+            actual={"Image":"wrong" if wrong_image else expected+"-image","Mounts":[{"Destination":"/data","Type":"volume","Name":expected}]}
+            with patch("upgrade.run"),patch.object(obj,"start_and_wait"),patch.object(obj,"inspect",return_value=actual):
+                if wrong_image:
+                    with self.assertRaises(RuntimeError):recover_transaction(obj,marker)
+                    self.assertTrue((root/"maintenance.json").exists())
+                    self.assertFalse((root/"last-upgrade.json").exists())
+                else:
+                    recover_transaction(obj,marker)
+                    self.assertEqual(compose.read_text(),(new if phase=="publishing" else old).read_text())
+                    self.assertFalse((root/"maintenance.json").exists())
+                    self.assertEqual(json.loads(record.read_text())["verified"],phase=="publishing")
+
+    def test_interruption_before_publication_restores_old_service(self):
+        self.exercise_phase("validating")
+
+    def test_interruption_during_publication_recovers_forward(self):
+        self.exercise_phase("publishing")
+
+    def test_running_wrong_image_cannot_mark_upgrade_verified(self):
+        self.exercise_phase("publishing",wrong_image=True)
+
+    def test_failed_publication_marker_does_not_open_new_service(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root=Path(directory)
+            obj=Backup({"state_dir":directory,"restic":"unused","password_file":str(root/"key"),"local_repository":str(root/"restic")})
+            image="registry/image@sha256:test"
+            (root/"upgrade-drill.json").write_text(json.dumps({"image":image,"isolated":True,"database":"ok\n1"}))
+            compose=root/"compose.yml"
+            compose.write_text("services:\n  gitea:\n    image: old-tag\n    environment: []\n    volumes: ['actual:/data']\nvolumes:\n  actual: {}\n")
+            info={"Image":"old-image","Mounts":[{"Destination":"/data","Type":"volume","Name":"old-volume"}]}
+            def fake_run(args,**kwargs):
+                out=""
+                if args[1:3]==["image","inspect"]:out='[{"Id":"new-image"}]'
+                elif args[1:3]==["network","inspect"]:out='[{"Internal":true}]'
+                elif "SELECT count(*) FROM action_task WHERE status=6;" in args:out="0"
+                elif "curl" in args:out='{"status":"pass"}'
+                return subprocess.CompletedProcess(args,0,stdout=out)
+            def fail_marker(path,data):
+                if Path(path).name=="maintenance.json" and data.get("phase")=="publishing":
+                    raise OSError("simulated ENOSPC")
+                json_write(path,data)
+            with patch.object(obj,"status"),patch.object(obj,"inspect",return_value=info),patch.object(obj,"inventory",return_value={"repositories":1}),patch("upgrade.run",side_effect=fake_run),patch("upgrade.json_write",side_effect=fail_marker),patch("upgrade.recover_transaction") as recover:
+                with self.assertRaises(OSError):
+                    perform_upgrade(obj,SimpleNamespace(compose=str(compose),image=image))
+                self.assertEqual(recover.call_args.args[1]["phase"],"validating")
+                self.assertEqual(json.loads((root/"maintenance.json").read_text())["phase"],"validating")
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Cold clone -> isolated validation -> durable publication; preserve old volume.
+
+Before publication, interruption restores the old service. Once publication can
+have accepted writes, recovery goes forward on the new volume, never silently
+backwards to stale data. The guard resumes this transaction after SIGKILL.
+"""
+import argparse
+import datetime as dt
+import fcntl
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import time
+import uuid
+import yaml
+from backup import Backup, data_volume, json_write, run, sync_directory
+
+
+def private_path(obj, value):
+    path = Path(value).resolve()
+    if path.parent != obj.root.resolve():
+        raise RuntimeError("Transaction file escapes private state directory")
+    return path
+
+
+def replace_file(path, content):
+    temp = path.with_suffix(path.suffix + ".upgrade-tmp")
+    with temp.open("wb") as stream:
+        stream.write(content)
+        stream.flush()
+        os.fsync(stream.fileno())
+    temp.chmod(0o600)
+    temp.replace(path)
+    sync_directory(path.parent)
+
+
+def compose_data_key(config):
+    sources = []
+    for mount in config["services"]["gitea"].get("volumes",[]):
+        if isinstance(mount,str):
+            parts = mount.split(":")
+            if len(parts) >= 2 and parts[1] == "/data":
+                sources.append(parts[0])
+        elif mount.get("target") == "/data" and mount.get("type","volume") == "volume":
+            sources.append(mount.get("source"))
+    if len(sources) != 1 or sources[0] not in config.get("volumes",{}):
+        raise RuntimeError("Expected a declared named volume at /data")
+    return sources[0]
+
+
+def recover_transaction(obj, marker):
+    marker_file = obj.root / "maintenance.json"
+    persisted = json.loads(marker_file.read_text())
+    if persisted != marker:
+        raise RuntimeError("Recovery direction differs from persisted transaction")
+    # Retry durability before any publication after an interrupted write.
+    with marker_file.open("rb") as stream:
+        os.fsync(stream.fileno())
+    sync_directory(obj.root)
+    compose = Path(marker["compose"])
+    if marker.get("kind") != "upgrade" or marker["container"] != obj.container:
+        raise RuntimeError("Invalid upgrade transaction")
+    for name in [marker.get("copy_container"), marker.get("validation_container")]:
+        if name:
+            if not name.startswith(("gitea-snapshot-copy-", "gitea-upgrade-check-")):
+                raise RuntimeError("Unexpected transaction helper")
+            subprocess.run([obj.docker,"rm","-f",name],capture_output=True,timeout=60)
+    publishing = marker["phase"] == "publishing"
+    selected = marker["candidate_compose"] if publishing else marker["rollback_compose"]
+    replace_file(compose, private_path(obj,selected).read_bytes())
+    run([obj.docker,"compose","-f",str(compose),"up","-d","--no-deps","gitea"],timeout=120)
+    obj.start_and_wait()
+    expected = marker["new_volume"] if publishing else marker["old_volume"]
+    actual = obj.inspect()
+    expected_image = marker["new_image_id"] if publishing else marker["old_image_id"]
+    if data_volume(actual) != expected or actual["Image"] != expected_image:
+        raise RuntimeError("Recovered service is using the wrong image or data volume")
+    record_file = private_path(obj,marker["record_file"])
+    record = json.loads(record_file.read_text())
+    record["recovery_direction"] = "forward" if publishing else "old-service"
+    record["verified"] = publishing
+    record["pause_seconds"] = round(time.time()-marker["started"],2)
+    json_write(record_file,record)
+    if publishing:
+        json_write(obj.root / "last-upgrade.json",record)
+    (obj.root / "maintenance.json").unlink(missing_ok=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", required=True)
+    parser.add_argument("--compose", required=True)
+    parser.add_argument("--image", required=True)
+    args = parser.parse_args()
+    if "@sha256:" not in args.image:
+        raise RuntimeError("An immutable image digest is required")
+    obj = Backup(json.loads(Path(args.config).read_text()))
+    with (obj.root / "backup.lock").open("a") as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX | fcntl.LOCK_NB)
+        obj.recover_interrupted_snapshot()
+        perform_upgrade(obj,args)
+
+
+def perform_upgrade(obj,args):
+    drill = json.loads((obj.root / "upgrade-drill.json").read_text())
+    if drill.get("image") != args.image or not drill.get("isolated") or not drill.get("database", "").startswith("ok\n"):
+        raise RuntimeError("Matching isolated upgrade rehearsal is required")
+    obj.status()
+    compose = Path(args.compose).resolve()
+    original = yaml.safe_load(compose.read_bytes())
+    if list(original["services"]) != ["gitea"]:
+        raise RuntimeError("Only a single Gitea service is supported")
+    info = obj.inspect()
+    obj.image = info["Image"]
+    old_volume = data_volume(info)
+    volume_key = compose_data_key(original)
+    target_image = json.loads(run([obj.docker,"image","inspect",args.image],timeout=30).stdout)[0]["Id"]
+    stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    volume = "gitea-upgrade-" + stamp.lower()
+    helper = "gitea-snapshot-copy-" + uuid.uuid4().hex[:12]
+    validation = "gitea-upgrade-check-" + uuid.uuid4().hex[:12]
+    record_file = obj.root / ("upgrade-"+stamp+".json")
+    rollback = obj.root / ("compose-before-"+stamp+".yml")
+    candidate = obj.root / ("compose-candidate-"+stamp+".yml")
+    # Pin rollback to the already-running image ID, not a mutable registry tag.
+    original["services"]["gitea"]["image"] = obj.image
+    original["volumes"][volume_key] = {"external":True,"name":old_volume}
+    replace_file(rollback,yaml.safe_dump(original,sort_keys=False).encode())
+    new = yaml.safe_load(yaml.safe_dump(original))
+    new["services"]["gitea"]["image"] = args.image
+    new["volumes"][volume_key] = {"external":True,"name":volume}
+    env = new["services"]["gitea"]["environment"]
+    if isinstance(env,list):
+        env = [e for e in env if not e.startswith("GITEA__service__DISABLE_REGISTRATION=")]
+        env.append("GITEA__service__DISABLE_REGISTRATION=true")
+    else:
+        env["GITEA__service__DISABLE_REGISTRATION"] = "true"
+    new["services"]["gitea"]["environment"] = env
+    replace_file(candidate,yaml.safe_dump(new,sort_keys=False).encode())
+    record = {"created_utc":stamp,"old_volume":old_volume,"old_image":obj.image,"new_volume":volume,"new_image":args.image,"compose":str(compose),"rollback_compose":str(rollback),"verified":False}
+    json_write(record_file,record)
+    marker = {"kind":"upgrade","phase":"validating","container":obj.container,"started":time.time(),"copy_container":helper,"validation_container":validation,"compose":str(compose),"rollback_compose":str(rollback),"candidate_compose":str(candidate),"record_file":str(record_file),"old_volume":old_volume,"new_volume":volume,"old_image_id":obj.image,"new_image_id":target_image}
+    active = run([obj.docker,"exec",obj.container,"sqlite3","-readonly","/data/gitea/gitea.db","SELECT count(*) FROM action_task WHERE status=6;"],text=True).stdout.strip()
+    if active != "0":
+        raise RuntimeError("Running CI must finish before upgrade")
+    run([obj.docker,"volume","create","--label","gitea.reliability=upgrade",volume])
+    network = "gitea-recovery-isolated"
+    network_info = json.loads(run([obj.docker,"network","inspect",network]).stdout)[0]
+    if not network_info["Internal"]:
+        raise RuntimeError("Validation network is not isolated")
+    json_write(obj.root / "maintenance.json",marker)
+    try:
+        obj.log("Stopping Gitea; cloning current data for isolated upgrade validation")
+        run([obj.docker,"stop","--time","30",obj.container],timeout=120)
+        run([obj.docker,"run","--rm","--name",helper,"--network","none","--mount",f"type=volume,src={old_volume},dst=/source,readonly","--mount",f"type=volume,src={volume},dst=/snapshot","--entrypoint","/bin/sh",obj.image,"-c","set -eu; test -s /source/gitea/gitea.db; cp -a /source/. /snapshot/"],timeout=600)
+        record["inventory"] = obj.inventory(volume)
+        run(obj.helper(volume,["/bin/cp","-p","/data/gitea/conf/app.ini","/data/gitea/conf/app.ini.pre-upgrade"],readonly=False),timeout=30)
+        flags = []
+        for key,value in {"actions__ENABLED":"false","cron__ENABLED":"false","cron__RUN_AT_START":"false","mailer__ENABLED":"false"}.items():
+            flags += ["-e","GITEA__"+key+"="+value]
+        run([obj.docker,"run","-d","--name",validation,"--network",network,"--mount",f"type=volume,src={volume},dst=/data",*flags,args.image],timeout=60)
+        for attempt in range(60):
+            try:
+                health = json.loads(run([obj.docker,"exec",validation,"curl","-fsS","--max-time","3","http://127.0.0.1:3000/api/healthz"],timeout=5).stdout)
+                if health.get("status") == "pass":
+                    break
+            except (subprocess.SubprocessError,ValueError):
+                pass
+            time.sleep(1)
+        else:
+            raise RuntimeError("Isolated upgraded instance did not become healthy")
+        run([obj.docker,"stop","--time","30",validation],timeout=60)
+        run([obj.docker,"rm",validation],timeout=30)
+        if obj.inventory(volume) != record["inventory"]:
+            raise RuntimeError("Isolated upgrade changed repository inventory or refs")
+        run(obj.helper(volume,["/bin/mv","/data/gitea/conf/app.ini.pre-upgrade","/data/gitea/conf/app.ini"],readonly=False),timeout=30)
+        record["isolated_validation"] = True
+        json_write(record_file,record)
+        publishing = dict(marker, phase="publishing")
+        # Never advance in-memory recovery direction before the marker commit.
+        json_write(obj.root / "maintenance.json",publishing)
+        recover_transaction(obj,publishing)
+        obj.log("PASS: verified upgrade published; old image and volume retained")
+    except BaseException:
+        # After publishing intent, recover forward. Never discard new writes.
+        marker_file = obj.root / "maintenance.json"
+        if marker_file.exists():
+            recover_transaction(obj,json.loads(marker_file.read_text()))
+        raise
+
+
+if __name__ == "__main__":
+    def terminate(signum, frame):
+        raise SystemExit(128 + signum)
+    signal.signal(signal.SIGTERM,terminate)
+    signal.signal(signal.SIGINT,terminate)
+    main()
Author	SHA1	Message	Date
gao	012d06cf49	Merge pull request 'fix(gitea): 备份失败后按小时补跑' (#3 ) from codex/gitea-reliability-1-retry into master	2026-09-26 07:58:33 +08:00
bruno	d8bb18a091	fix(gitea): 保留跨日备份并恢复中断服务 gitea-ci / reliability-source Run 3142 checked the exact source HEAD and passed operations tests Details	2026-09-26 07:53:38 +08:00
bruno	80480b7ce2	fix(gitea): 备份失败后按小时补跑	2026-09-26 07:50:00 +08:00
gao	917501939b	Merge pull request 'fix(gitea): 可靠备份恢复、隔离 CI 与受控升级' (#2 ) from codex/gitea-reliability-1 into master	2026-09-25 23:12:46 +08:00
bruno	b6e8204ea0	docs(gitea): 记录恢复演练与升级回执 gitea-ci / reliability-source Gitea pilot run 3137 verified this exact source HEAD (26 pass, Docker-only test skipped) Details Agent: codex	2026-09-25 23:02:42 +08:00
bruno	9b15030f2b	fix(gitea): 修复外置备份恢复与受控升级 Agent: codex Refs: gao/macos-tools#1	2026-09-25 22:59:35 +08:00