state backup: systemd units (NOT enabled) + cross-host drill script

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

deploy/systemd/windygit-state-backup.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2)
+After=network-online.target docker.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/srv/windygit/src
+# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass
+# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD).
+EnvironmentFile=/srv/windygit/src/.env
+ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh
+Nice=10
+IOSchedulingClass=idle
+TimeoutStartSec=3h

deploy/systemd/windygit-state-backup.service.d/windy-job.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh

deploy/systemd/windygit-state-backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Nightly Windy Git state backup
+
+[Timer]
+# 03:07 local, clear of the git-bundle backup at 04:17.
+OnCalendar=*-*-* 03:07:00
+Persistent=true
+RandomizedDelaySec=300
+
+[Install]
+WantedBy=timers.target

scripts/drill_cross_host.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed.
+# Usage: R2_ACCOUNT_ID=<id> R2_KEY_NAME=<lockbox key> R2_SECRET_NAME=<lockbox key> bash drill_cross_host.sh
+set -euo pipefail
+umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX)
+trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT
+lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null
+lockbox-get "$R2_KEY_NAME" "$W/ak" >/dev/null; lockbox-get "$R2_SECRET_NAME" "$W/sk" >/dev/null
+export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")"
+export RESTIC_REPOSITORY="s3:https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com/windy-git-backups/restic"
+restic snapshots --tag windygit-state --compact | tail -3
+restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet
+ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}'
+docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null
+for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done
+for db in gitea windygit; do
+  docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db"
+  docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true
+done
+for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do
+  echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")"
+done
+echo "cross-host drill OK (cleaned up)"