#!/usr/bin/env bash
#
# Dogfood — backup, restore, and the restore *rehearsal* (spec §10, M6).
#
#   ./scripts/backup.sh backup [DIR]     pg_dump + a snapshot of the repo volume
#   ./scripts/backup.sh restore DIR      restore both into the live stack
#   ./scripts/backup.sh rehearse [DIR]   restore into a throwaway database and
#                                        prove the result is usable
#
# Spec §10:
#
#   > Backups: nightly `pg_dump` plus a filesystem-level snapshot of
#   > `/srv/repos`, both offsite. Test restores quarterly. Note that the database
#   > and the repo volume can drift out of sync during a restore;
#   > `dogfood-admin reindex --all` exists to reconcile them, and that recovery
#   > path should be exercised at least once before the first real user.
#
# `rehearse` is that exercise, and it is the reason this script exists rather
# than a line in a runbook. It restores into a *separate* database, runs the
# reconciling reindex against it, and checks the result — so the recovery path
# is proven without touching production.
#
# The two artefacts are captured in this order, deliberately:
#
#   1. the repository volume, then
#   2. the database.
#
# A repository object that exists on disk but not in the index is invisible and
# fixable with `reindex`. A row that points at an object which was never captured
# is a broken page with no recovery. Capturing storage first makes the drift fall
# on the recoverable side.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
COMPOSE=(docker compose -f "$ROOT/docker/compose.yaml")
PG_IMAGE="postgres:17-alpine"
VOLUME="dogfood_repos"

c_red() { printf '\033[31m%s\033[0m\n' "$*" >&2; }
c_green() { printf '\033[32m%s\033[0m\n' "$*"; }
c_dim() { printf '\033[2m%s\033[0m\n' "$*"; }

env_get() { sed -n "s/^$1=//p" "$ROOT/.env" | head -1; }

require_env() {
    [[ -f "$ROOT/.env" ]] || { c_red "no .env at $ROOT/.env"; exit 1; }
    DATABASE_URL="$(env_get DATABASE_URL)"
    [[ -n "$DATABASE_URL" ]] || { c_red "DATABASE_URL is not set in .env"; exit 1; }
}

# ─── backup ──────────────────────────────────────────────────────────────────

cmd_backup() {
    require_env
    local dir="${1:-$ROOT/backups/$(date -u +%Y%m%dT%H%M%SZ)}"
    mkdir -p "$dir"

    # 1. Repository storage first (see the note at the top).
    c_dim "snapshotting the repository volume…"
    docker run --rm \
        -v "$VOLUME":/srv/repos:ro \
        -v "$dir":/backup \
        "$PG_IMAGE" \
        tar -C /srv/repos -czf /backup/repos.tar.gz .

    # 2. The database. `--no-owner` and `--no-acl` so a restore does not depend
    #    on the role names of the instance it came from.
    c_dim "dumping the database…"
    docker run --rm -v "$dir":/backup "$PG_IMAGE" \
        pg_dump --no-owner --no-acl --format=custom \
        --file=/backup/dogfood.dump "$DATABASE_URL"

    # A manifest, so a restore can tell what it is holding without opening it.
    cat > "$dir/manifest.txt" <<EOF
dogfood backup
taken_at: $(date -u +%Y-%m-%dT%H:%M:%SZ)
host: $(hostname)
repos_bytes: $(stat -c %s "$dir/repos.tar.gz")
dump_bytes: $(stat -c %s "$dir/dogfood.dump")
note: storage was captured before the database, so any drift is recoverable
      with 'dogfood-admin reindex --all'.
EOF

    c_green "backup written to $dir"
    cat "$dir/manifest.txt"
}

# ─── restore ─────────────────────────────────────────────────────────────────

cmd_restore() {
    require_env
    local dir="${1:?usage: backup.sh restore DIR}"
    [[ -f "$dir/dogfood.dump" && -f "$dir/repos.tar.gz" ]] \
        || { c_red "$dir does not look like a Dogfood backup"; exit 1; }

    c_red "This overwrites the live database and repository volume."
    read -rp "Type RESTORE to continue: " confirm
    [[ "$confirm" == "RESTORE" ]] || { echo "aborted"; exit 1; }

    c_dim "stopping the stack…"
    "${COMPOSE[@]}" down

    c_dim "restoring the repository volume…"
    docker run --rm -v "$VOLUME":/srv/repos -v "$dir":/backup:ro "$PG_IMAGE" \
        sh -c 'rm -rf /srv/repos/* && tar -C /srv/repos -xzf /backup/repos.tar.gz'

    c_dim "restoring the database…"
    docker run --rm -v "$dir":/backup:ro "$PG_IMAGE" \
        pg_restore --clean --if-exists --no-owner --no-acl \
        --dbname "$DATABASE_URL" /backup/dogfood.dump

    c_dim "starting the stack…"
    "${COMPOSE[@]}" up -d

    # The whole point of §10's note: after a restore the two halves may disagree,
    # and this is what reconciles them.
    c_dim "reconciling the index with storage…"
    docker compose -f "$ROOT/docker/compose.yaml" exec -T worker \
        dogfood-admin reindex --all

    c_green "restore complete"
}

# ─── rehearsal ───────────────────────────────────────────────────────────────

cmd_rehearse() {
    require_env
    local dir="${1:-}"

    # No backup named: take one now, so the rehearsal exercises the real path
    # end to end rather than an artefact somebody prepared by hand.
    if [[ -z "$dir" ]]; then
        dir="$(mktemp -d)/backup"
        c_dim "no backup given; taking one into $dir"
        cmd_backup "$dir" >/dev/null
    fi

    # Not `local`: the EXIT trap below runs after this function's frame is gone,
    # and a `local` would be unbound by then — which under `set -u` turns a
    # successful rehearsal into a non-zero exit during cleanup.
    pg_name="dogfood-restore-rehearsal-$$"
    repo_vol="dogfood-rehearsal-repos-$$"
    worker_name="dogfood-rehearsal-worker-$$"

    cleanup() {
        c_dim "cleaning up the rehearsal environment…"
        docker rm -f "$worker_name" >/dev/null 2>&1 || true
        docker rm -f "$pg_name" >/dev/null 2>&1 || true
        docker volume rm "$repo_vol" >/dev/null 2>&1 || true
    }
    trap cleanup EXIT

    c_dim "starting a throwaway database…"
    docker run -d --name "$pg_name" \
        -e POSTGRES_PASSWORD=rehearse -e POSTGRES_DB=dogfood \
        "$PG_IMAGE" >/dev/null

    # Wait for it rather than sleeping a guess.
    for _ in $(seq 1 60); do
        docker exec "$pg_name" pg_isready -U postgres >/dev/null 2>&1 && break
        sleep 1
    done
    docker exec "$pg_name" pg_isready -U postgres >/dev/null \
        || { c_red "the rehearsal database never became ready"; exit 1; }

    local url="postgres://postgres:rehearse@127.0.0.1:5432/dogfood"

    c_dim "restoring the dump…"
    docker cp "$dir/dogfood.dump" "$pg_name":/tmp/dogfood.dump
    docker exec "$pg_name" pg_restore --clean --if-exists --no-owner --no-acl \
        --dbname "$url" /tmp/dogfood.dump

    c_dim "restoring the repository volume…"
    docker volume create "$repo_vol" >/dev/null
    docker run --rm -v "$repo_vol":/srv/repos -v "$dir":/backup:ro "$PG_IMAGE" \
        tar -C /srv/repos -xzf /backup/repos.tar.gz

    # ── the checks that make this a rehearsal rather than a copy ──────────────
    c_dim "checking the restored database…"

    local checks_failed=0
    check() {
        local label="$1" sql="$2"
        local out
        out="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood -c "$sql" 2>&1 | tr -d '[:space:]')"
        if [[ "$out" == "t" || "$out" == "0" ]]; then
            printf '  \033[32m✓\033[0m %s\n' "$label"
        else
            printf '  \033[31m✗\033[0m %s (got: %s)\n' "$label" "$out"
            checks_failed=$((checks_failed + 1))
        fi
    }

    check "schema is present" \
        "SELECT to_regclass('public.repos') IS NOT NULL"
    check "migrations are recorded" \
        "SELECT count(*) = 0 FROM _sqlx_migrations WHERE NOT success"
    check "no change points at a missing repository" \
        "SELECT count(*) FROM changes c LEFT JOIN repos r ON r.id = c.repo_id WHERE r.id IS NULL"
    check "no revision points at a missing change" \
        "SELECT count(*) FROM revisions v LEFT JOIN changes c ON c.id = v.change_id_fk WHERE c.id IS NULL"
    check "no comment is anchored to a missing revision" \
        "SELECT count(*) FROM comments m
          WHERE m.anchor_revision IS NOT NULL
            AND NOT EXISTS (SELECT 1 FROM revisions v WHERE v.id = m.anchor_revision)"

    # Every repository row must have storage behind it. This is the drift §10
    # warns about, and the one the reindex exists to reconcile.
    c_dim "checking that every repository row has storage…"
    local missing=0 total=0
    while read -r id; do
        [[ -n "$id" ]] || continue
        total=$((total + 1))
        local hex="${id//-/}"
        if ! docker run --rm -v "$repo_vol":/srv/repos:ro "$PG_IMAGE" \
                test -d "/srv/repos/${hex:0:2}/${hex}.git"; then
            missing=$((missing + 1))
            c_dim "  missing storage for repo $id"
        fi
    done < <(docker exec "$pg_name" psql -tAX -U postgres -d dogfood -c "SELECT id FROM repos")

    printf '  %s %d of %d repositories have storage\n' \
        "$([[ $missing -eq 0 ]] && printf '\033[32m✓\033[0m' || printf '\033[33m!\033[0m')" \
        "$((total - missing))" "$total"

    if [[ $missing -gt 0 ]]; then
        c_dim "  (recoverable: 'dogfood-admin reindex --all' reconciles the index"
        c_dim "   with storage, but it cannot invent objects that were not captured)"
    fi

    # The reconciling reindex itself, against the rehearsal environment. This is
    # the §10 recovery path, actually run — queue the work, then run a worker
    # long enough to drain it, then check that it drained cleanly.
    #
    # `--entrypoint` is required: the worker image's entrypoint is the worker
    # daemon, so passing `dogfood-admin …` as the command would start the
    # daemon with those words as arguments and never return.
    c_dim "queueing 'dogfood-admin reindex --all' against the restore…"
    if docker run --rm --network "container:$pg_name" \
            --entrypoint dogfood-admin \
            -v "$repo_vol":/srv/repos \
            -e DATABASE_URL="$url" -e REPO_ROOT=/srv/repos \
            dogfood-worker reindex --all; then
        printf '  \033[32m✓\033[0m reindex queued\n'
    else
        printf '  \033[31m✗\033[0m reindex could not be queued\n'
        checks_failed=$((checks_failed + 1))
    fi

    c_dim "running a worker against the restore until the queue drains…"
    docker run -d --name "$worker_name" --network "container:$pg_name" \
        -v "$repo_vol":/srv/repos \
        -e DATABASE_URL="$url" -e REPO_ROOT=/srv/repos \
        dogfood-worker >/dev/null

    local drained=0
    for _ in $(seq 1 60); do
        local queued
        queued="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood \
            -c "SELECT count(*) FROM jobs WHERE locked_at IS NULL" 2>/dev/null | tr -d '[:space:]')"
        if [[ "$queued" == "0" ]]; then drained=1; break; fi
        sleep 1
    done

    local failed
    failed="$(docker exec "$pg_name" psql -tAX -U postgres -d dogfood \
        -c "SELECT count(*) FROM jobs WHERE attempts >= max_attempts" 2>/dev/null | tr -d '[:space:]')"

    docker logs "$worker_name" 2>&1 | grep -E '"level":"(ERROR|WARN)"' | head -5 || true
    docker rm -f "$worker_name" >/dev/null 2>&1 || true

    if [[ "$drained" == "1" && "$failed" == "0" ]]; then
        printf '  \033[32m✓\033[0m the queue drained with no failed jobs\n'
    else
        printf '  \033[31m✗\033[0m reindex did not complete (drained=%s failed=%s)\n' \
            "$drained" "$failed"
        checks_failed=$((checks_failed + 1))
    fi

    echo
    if [[ $checks_failed -eq 0 ]]; then
        c_green "restore rehearsal passed — this backup is restorable"
    else
        c_red "restore rehearsal FAILED $checks_failed check(s)"
        exit 1
    fi
}

case "${1:-}" in
    backup)   shift; cmd_backup "$@" ;;
    restore)  shift; cmd_restore "$@" ;;
    rehearse) shift; cmd_rehearse "$@" ;;
    *)
        sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
        exit 1
        ;;
esac