dogfood
Jump to…
snowcorepublic2 bookmarks
snowinitial commitqoxwzsukwmkx1mo
mainroot / scripts / backup.shHide BlameRaw
Matt W1#!/usr/bin/env bash
Matt W2#
Matt W3# Dogfood — backup, restore, and the restore *rehearsal* (spec §10, M6).
Matt W4#
Matt W5# ./scripts/backup.sh backup [DIR] pg_dump + a snapshot of the repo volume
Matt W6# ./scripts/backup.sh restore DIR restore both into the live stack
Matt W7# ./scripts/backup.sh rehearse [DIR] restore into a throwaway database and
Matt W8# prove the result is usable
Matt W9#
Matt W10# Spec §10:
Matt W11#
Matt W12# > Backups: nightly `pg_dump` plus a filesystem-level snapshot of
Matt W13# > `/srv/repos`, both offsite. Test restores quarterly. Note that the database
Matt W14# > and the repo volume can drift out of sync during a restore;
Matt W15# > `dogfood-admin reindex --all` exists to reconcile them, and that recovery
Matt W16# > path should be exercised at least once before the first real user.
Matt W17#
Matt W18# `rehearse` is that exercise, and it is the reason this script exists rather
Matt W19# than a line in a runbook. It restores into a *separate* database, runs the
Matt W20# reconciling reindex against it, and checks the result — so the recovery path
Matt W21# is proven without touching production.
Matt W22#
Matt W23# The two artefacts are captured in this order, deliberately:
Matt W24#
Matt W25# 1. the repository volume, then
Matt W26# 2. the database.
Matt W27#
Matt W28# A repository object that exists on disk but not in the index is invisible and
Matt W29# fixable with `reindex`. A row that points at an object which was never captured
Matt W30# is a broken page with no recovery. Capturing storage first makes the drift fall
Matt W31# on the recoverable side.
Matt W32
Matt W33set -euo pipefail
Matt W34
Matt W35ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
Matt W36COMPOSE=(docker compose -f "$ROOT/docker/compose.yaml")
Matt W37PG_IMAGE="postgres:17-alpine"
Matt W38VOLUME="dogfood_repos"
Matt W39
Matt W40c_red() { printf '\033[31m%s\033[0m\n' "$*" >&2; }
Matt W41c_green() { printf '\033[32m%s\033[0m\n' "$*"; }
Matt W42c_dim() { printf '\033[2m%s\033[0m\n' "$*"; }
Matt W43
Matt W44env_get() { sed -n "s/^$1=//p" "$ROOT/.env" | head -1; }
Matt W45
Matt W46require_env() {
Matt W47 [[ -f "$ROOT/.env" ]] || { c_red "no .env at $ROOT/.env"; exit 1; }
Matt W48 DATABASE_URL="$(env_get DATABASE_URL)"
Matt W49 [[ -n "$DATABASE_URL" ]] || { c_red "DATABASE_URL is not set in .env"; exit 1; }
Matt W50}
Matt W51
Matt W52# ─── backup ──────────────────────────────────────────────────────────────────
Matt W53
Matt W54cmd_backup() {
Matt W55 require_env
Matt W56 local dir="${1:-$ROOT/backups/$(date -u +%Y%m%dT%H%M%SZ)}"
Matt W57 mkdir -p "$dir"
Matt W58
Matt W59 # 1. Repository storage first (see the note at the top).
Matt W60 c_dim "snapshotting the repository volume…"
Matt W61 docker run --rm \
Matt W62 -v "$VOLUME":/srv/repos:ro \
Matt W63 -v "$dir":/backup \
Matt W64 "$PG_IMAGE" \
Matt W65 tar -C /srv/repos -czf /backup/repos.tar.gz .
Matt W66
Matt W67 # 2. The database. `--no-owner` and `--no-acl` so a restore does not depend
Matt W68 # on the role names of the instance it came from.
Matt W69 c_dim "dumping the database…"
Matt W70 docker run --rm -v "$dir":/backup "$PG_IMAGE" \
Matt W71 pg_dump --no-owner --no-acl --format=custom \
Matt W72 --file=/backup/dogfood.dump "$DATABASE_URL"
Matt W73
Matt W74 # A manifest, so a restore can tell what it is holding without opening it.
Matt W75 cat > "$dir/manifest.txt" <<EOF
Matt W76dogfood backup
Matt W77taken_at: $(date -u +%Y-%m-%dT%H:%M:%SZ)
Matt W78host: $(hostname)
Matt W79repos_bytes: $(stat -c %s "$dir/repos.tar.gz")
Matt W80dump_bytes: $(stat -c %s "$dir/dogfood.dump")
Matt W81note: storage was captured before the database, so any drift is recoverable
Matt W82 with 'dogfood-admin reindex --all'.
Matt W83EOF
Matt W84
Matt W85 c_green "backup written to $dir"
Matt W86 cat "$dir/manifest.txt"
Matt W87}
Matt W88
Matt W89# ─── restore ─────────────────────────────────────────────────────────────────
Matt W90
Matt W91cmd_restore() {
Matt W92 require_env
Matt W93 local dir="${1:?usage: backup.sh restore DIR}"
Matt W94 [[ -f "$dir/dogfood.dump" && -f "$dir/repos.tar.gz" ]] \
Matt W95 || { c_red "$dir does not look like a Dogfood backup"; exit 1; }
Matt W96
Matt W97 c_red "This overwrites the live database and repository volume."
Matt W98 read -rp "Type RESTORE to continue: " confirm
Matt W99 [[ "$confirm" == "RESTORE" ]] || { echo "aborted"; exit 1; }
Matt W100
Matt W101 c_dim "stopping the stack…"
Matt W102 "${COMPOSE[@]}" down
Matt W103
Matt W104 c_dim "restoring the repository volume…"
Matt W105 docker run --rm -v "$VOLUME":/srv/repos -v "$dir":/backup:ro "$PG_IMAGE" \
Matt W106 sh -c 'rm -rf /srv/repos/* && tar -C /srv/repos -xzf /backup/repos.tar.gz'
Matt W107
Matt W108 c_dim "restoring the database…"
Matt W109 docker run --rm -v "$dir":/backup:ro "$PG_IMAGE" \
Matt W110 pg_restore --clean --if-exists --no-owner --no-acl \
Matt W111 --dbname "$DATABASE_URL" /backup/dogfood.dump
Matt W112
Matt W113 c_dim "starting the stack…"
Matt W114 "${COMPOSE[@]}" up -d
Matt W115
Matt W116 # The whole point of §10's note: after a restore the two halves may disagree,
Matt W117 # and this is what reconciles them.
Matt W118 c_dim "reconciling the index with storage…"
Matt W119 docker compose -f "$ROOT/docker/compose.yaml" exec -T worker \
Matt W120 dogfood-admin reindex --all
Matt W121
Matt W122 c_green "restore complete"
Matt W123}
Matt W124
Matt W125# ─── rehearsal ───────────────────────────────────────────────────────────────
Matt W126
Matt W127cmd_rehearse() {
Matt W128 require_env
Matt W129 local dir="${1:-}"
Matt W130
Matt W131 # No backup named: take one now, so the rehearsal exercises the real path
Matt W132 # end to end rather than an artefact somebody prepared by hand.
Matt W133 if [[ -z "$dir" ]]; then
Matt W134 dir="$(mktemp -d)/backup"
Matt W135 c_dim "no backup given; taking one into $dir"
Matt W136 cmd_backup "$dir" >/dev/null
Matt W137 fi
Matt W138
Matt W139 # Not `local`: the EXIT trap below runs after this function's frame is gone,
Matt W140 # and a `local` would be unbound by then — which under `set -u` turns a
Matt W141 # successful rehearsal into a non-zero exit during cleanup.
Matt W142 pg_name="dogfood-restore-rehearsal-$$"
Matt W143 repo_vol="dogfood-rehearsal-repos-$$"
Matt W144 worker_name="dogfood-rehearsal-worker-$$"
Matt W145
Matt W146 cleanup() {
Matt W147 c_dim "cleaning up the rehearsal environment…"
Matt W148 docker rm -f "$worker_name" >/dev/null 2>&1 || true
Matt W149 docker rm -f "$pg_name" >/dev/null 2>&1 || true
Matt W150 docker volume rm "$repo_vol" >/dev/null 2>&1 || true
Matt W151 }
Matt W152 trap cleanup EXIT
Matt W153
Matt W154 c_dim "starting a throwaway database…"
Matt W155 docker run -d --name "$pg_name" \
Matt W156 -e POSTGRES_PASSWORD=rehearse -e POSTGRES_DB=dogfood \
Matt W157 "$PG_IMAGE" >/dev/null
Matt W158
Matt W159 # Wait for it rather than sleeping a guess.
Matt W160 for _ in $(seq 1 60); do
Matt W161 docker exec "$pg_name" pg_isready -U postgres >/dev/null 2>&1 && break
Matt W162 sleep 1
Matt W163 done
Matt W164 docker exec "$pg_name" pg_isready -U postgres >/dev/null \
Matt W165 || { c_red "the rehearsal database never became ready"; exit 1; }
Matt W166
Matt W167 local url="postgres://postgres:rehearse@127.0.0.1:5432/dogfood"
Matt W168
Matt W169 c_dim "restoring the dump…"
Matt W170 docker cp "$dir/dogfood.dump" "$pg_name":/tmp/dogfood.dump
Matt W171 docker exec "$pg_name" pg_restore --clean --if-exists --no-owner --no-acl \
Matt W172 --dbname "$url" /tmp/dogfood.dump
Matt W173
Matt W174 c_dim "restoring the repository volume…"
Matt W175 docker volume create "$repo_vol" >/dev/null
Matt W176 docker run --rm -v "$repo_vol":/srv/repos -v "$dir":/backup:ro "$PG_IMAGE" \
Matt W177 tar -C /srv/repos -xzf /backup/repos.tar.gz
Matt W178
Matt W179 # ── the checks that make this a rehearsal rather than a copy ──────────────
Matt W180 c_dim "checking the restored database…"
Matt W181
Matt W182 local checks_failed=0
Matt W183 check() {
Matt W184 local label="$1" sql="$2"
Matt W185 local out
Matt W186 out="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood -c "$sql" 2>&1 | tr -d '[:space:]')"
Matt W187 if [[ "$out" == "t" || "$out" == "0" ]]; then
Matt W188 printf ' \033[32m✓\033[0m %s\n' "$label"
Matt W189 else
Matt W190 printf ' \033[31m✗\033[0m %s (got: %s)\n' "$label" "$out"
Matt W191 checks_failed=$((checks_failed + 1))
Matt W192 fi
Matt W193 }
Matt W194
Matt W195 check "schema is present" \
Matt W196 "SELECT to_regclass('public.repos') IS NOT NULL"
Matt W197 check "migrations are recorded" \
Matt W198 "SELECT count(*) = 0 FROM _sqlx_migrations WHERE NOT success"
Matt W199 check "no change points at a missing repository" \
Matt W200 "SELECT count(*) FROM changes c LEFT JOIN repos r ON r.id = c.repo_id WHERE r.id IS NULL"
Matt W201 check "no revision points at a missing change" \
Matt W202 "SELECT count(*) FROM revisions v LEFT JOIN changes c ON c.id = v.change_id_fk WHERE c.id IS NULL"
Matt W203 check "no comment is anchored to a missing revision" \
Matt W204 "SELECT count(*) FROM comments m
Matt W205 WHERE m.anchor_revision IS NOT NULL
Matt W206 AND NOT EXISTS (SELECT 1 FROM revisions v WHERE v.id = m.anchor_revision)"
Matt W207
Matt W208 # Every repository row must have storage behind it. This is the drift §10
Matt W209 # warns about, and the one the reindex exists to reconcile.
Matt W210 c_dim "checking that every repository row has storage…"
Matt W211 local missing=0 total=0
Matt W212 while read -r id; do
Matt W213 [[ -n "$id" ]] || continue
Matt W214 total=$((total + 1))
Matt W215 local hex="${id//-/}"
Matt W216 if ! docker run --rm -v "$repo_vol":/srv/repos:ro "$PG_IMAGE" \
Matt W217 test -d "/srv/repos/${hex:0:2}/${hex}.git"; then
Matt W218 missing=$((missing + 1))
Matt W219 c_dim " missing storage for repo $id"
Matt W220 fi
Matt W221 done < <(docker exec "$pg_name" psql -tAX -U postgres -d dogfood -c "SELECT id FROM repos")
Matt W222
Matt W223 printf ' %s %d of %d repositories have storage\n' \
Matt W224 "$([[ $missing -eq 0 ]] && printf '\033[32m✓\033[0m' || printf '\033[33m!\033[0m')" \
Matt W225 "$((total - missing))" "$total"
Matt W226
Matt W227 if [[ $missing -gt 0 ]]; then
Matt W228 c_dim " (recoverable: 'dogfood-admin reindex --all' reconciles the index"
Matt W229 c_dim " with storage, but it cannot invent objects that were not captured)"
Matt W230 fi
Matt W231
Matt W232 # The reconciling reindex itself, against the rehearsal environment. This is
Matt W233 # the §10 recovery path, actually run — queue the work, then run a worker
Matt W234 # long enough to drain it, then check that it drained cleanly.
Matt W235 #
Matt W236 # `--entrypoint` is required: the worker image's entrypoint is the worker
Matt W237 # daemon, so passing `dogfood-admin …` as the command would start the
Matt W238 # daemon with those words as arguments and never return.
Matt W239 c_dim "queueing 'dogfood-admin reindex --all' against the restore…"
Matt W240 if docker run --rm --network "container:$pg_name" \
Matt W241 --entrypoint dogfood-admin \
Matt W242 -v "$repo_vol":/srv/repos \
Matt W243 -e DATABASE_URL="$url" -e REPO_ROOT=/srv/repos \
Matt W244 dogfood-worker reindex --all; then
Matt W245 printf ' \033[32m✓\033[0m reindex queued\n'
Matt W246 else
Matt W247 printf ' \033[31m✗\033[0m reindex could not be queued\n'
Matt W248 checks_failed=$((checks_failed + 1))
Matt W249 fi
Matt W250
Matt W251 c_dim "running a worker against the restore until the queue drains…"
Matt W252 docker run -d --name "$worker_name" --network "container:$pg_name" \
Matt W253 -v "$repo_vol":/srv/repos \
Matt W254 -e DATABASE_URL="$url" -e REPO_ROOT=/srv/repos \
Matt W255 dogfood-worker >/dev/null
Matt W256
Matt W257 local drained=0
Matt W258 for _ in $(seq 1 60); do
Matt W259 local queued
Matt W260 queued="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood \
Matt W261 -c "SELECT count(*) FROM jobs WHERE locked_at IS NULL" 2>/dev/null | tr -d '[:space:]')"
Matt W262 if [[ "$queued" == "0" ]]; then drained=1; break; fi
Matt W263 sleep 1
Matt W264 done
Matt W265
Matt W266 local failed
Matt W267 failed="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood \
Matt W268 -c "SELECT count(*) FROM jobs WHERE attempts >= max_attempts" 2>/dev/null | tr -d '[:space:]')"
Matt W269
Matt W270 docker logs "$worker_name" 2>&1 | grep -E '"level":"(ERROR|WARN)"' | head -5 || true
Matt W271 docker rm -f "$worker_name" >/dev/null 2>&1 || true
Matt W272
Matt W273 if [[ "$drained" == "1" && "$failed" == "0" ]]; then
Matt W274 printf ' \033[32m✓\033[0m the queue drained with no failed jobs\n'
Matt W275 else
Matt W276 printf ' \033[31m✗\033[0m reindex did not complete (drained=%s failed=%s)\n' \
Matt W277 "$drained" "$failed"
Matt W278 checks_failed=$((checks_failed + 1))
Matt W279 fi
Matt W280
Matt W281 echo
Matt W282 if [[ $checks_failed -eq 0 ]]; then
Matt W283 c_green "restore rehearsal passed — this backup is restorable"
Matt W284 else
Matt W285 c_red "restore rehearsal FAILED $checks_failed check(s)"
Matt W286 exit 1
Matt W287 fi
Matt W288}
Matt W289
Matt W290case "${1:-}" in
Matt W291 backup) shift; cmd_backup "$@" ;;
Matt W292 restore) shift; cmd_restore "$@" ;;
Matt W293 rehearse) shift; cmd_rehearse "$@" ;;
Matt W294 *)
Matt W295 sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
Matt W296 exit 1
Matt W297 ;;
Matt W298esac

298 lines · Shell