diff --git a/deploy/systemd/windygit-state-backup.service b/deploy/systemd/windygit-state-backup.service new file mode 100644 index 0000000..c9e79db --- /dev/null +++ b/deploy/systemd/windygit-state-backup.service @@ -0,0 +1,14 @@ +[Unit] +Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2) +After=network-online.target docker.service + +[Service] +Type=oneshot +WorkingDirectory=/srv/windygit/src +# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass +# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD). +EnvironmentFile=/srv/windygit/src/.env +ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=3h diff --git a/deploy/systemd/windygit-state-backup.service.d/windy-job.conf b/deploy/systemd/windygit-state-backup.service.d/windy-job.conf new file mode 100644 index 0000000..62c5da8 --- /dev/null +++ b/deploy/systemd/windygit-state-backup.service.d/windy-job.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh diff --git a/deploy/systemd/windygit-state-backup.timer b/deploy/systemd/windygit-state-backup.timer new file mode 100644 index 0000000..0db71f5 --- /dev/null +++ b/deploy/systemd/windygit-state-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Nightly Windy Git state backup + +[Timer] +# 03:07 local, clear of the git-bundle backup at 04:17. +OnCalendar=*-*-* 03:07:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target diff --git a/scripts/drill_cross_host.sh b/scripts/drill_cross_host.sh new file mode 100755 index 0000000..b7f8ef7 --- /dev/null +++ b/scripts/drill_cross_host.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed. +# Usage: R2_ACCOUNT_ID= R2_KEY_NAME= R2_SECRET_NAME= bash drill_cross_host.sh +set -euo pipefail +umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX) +trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT +lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null +lockbox-get "$R2_KEY_NAME" "$W/ak" >/dev/null; lockbox-get "$R2_SECRET_NAME" "$W/sk" >/dev/null +export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")" +export RESTIC_REPOSITORY="s3:https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com/windy-git-backups/restic" +restic snapshots --tag windygit-state --compact | tail -3 +restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet +ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}' +docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null +for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done +for db in gitea windygit; do + docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db" + docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true +done +for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do + echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")" +done +echo "cross-host drill OK (cleaned up)"