From ccb3da51adda99621c0dd8b2e96c79cbbc3ac6df Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 25 Aug 2026 09:59:43 -0400 Subject: [PATCH] Make the backup script wait for readiness and verify the archive restores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running it against the live service exposed two gaps. It announced "App restarted" the moment the container existed, so the very next request got a 502 from an app that wasn't listening yet; it now polls /healthz until the app is actually serving. And it only checked that the archive contained the right files — it now opens the archived database and runs SQLite's integrity_check, because an archive that exists but won't restore is the worst kind of backup. End to end on the server, with a throwaway part and photo inserted to exercise the verification path: 2.6 seconds of downtime, archive verified, service answering immediately on return, throwaway data removed afterwards. Co-Authored-By: Claude Opus 5 (1M context) --- tools/backup.sh | 74 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/tools/backup.sh b/tools/backup.sh index 2dedda2..2594ee5 100755 --- a/tools/backup.sh +++ b/tools/backup.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -# Consistent backup of the parts inventory. +# Consistent, verified backup of the parts inventory. # # The database and the photos are two resources that reference each other, so # capturing them at different moments is not a backup: a photo deleted between -# the two steps leaves the saved database pointing at a file that isn't in the -# archive, and one added leaves the reverse. Neither is repairable afterwards. +# the two steps leaves the saved database pointing at a file the archive does +# not contain, and one added leaves the reverse. Neither is repairable +# afterwards, and pruning only fixes the harmless direction. # -# So the app is stopped for the few seconds it takes to copy both. With no -# process attached, parts.db and its -wal/-shm sidecars are a consistent set and +# So the app is stopped for the couple of seconds the copy takes. With no +# process attached, parts.db and any -wal/-shm sidecars are a consistent set and # the images directory cannot move underneath us. # # ./tools/backup.sh [output-directory] @@ -15,27 +16,48 @@ set -euo pipefail OUT_DIR="$(cd "${1:-$PWD}" && pwd)" cd "$(dirname "$0")/.." -STAMP="$(date +%F-%H%M%S)" -NAME="parts-backup-$STAMP.tar.gz" -VOLUME="$(docker compose config --format json 2>/dev/null \ - | python3 -c 'import json,sys; print(list(json.load(sys.stdin)["volumes"])[0])' 2>/dev/null || echo parts_data)" -VOLUME="$(docker volume ls --format '{{.Name}}' | grep -E "parts.*data" | head -1)" +NAME="parts-backup-$(date +%F-%H%M%S).tar.gz" +ARCHIVE="$OUT_DIR/$NAME" + +VOLUME="$(docker volume ls --format '{{.Name}}' | grep -E '^parts.*data$' | head -1)" +if [ -z "$VOLUME" ]; then + echo "Could not find the parts data volume." >&2 + exit 1 +fi echo "Stopping the app so both halves are captured at one moment..." docker compose stop parts >/dev/null -restart() { docker compose start parts >/dev/null && echo "App restarted."; } +restarted=0 +restart() { + [ "$restarted" = 1 ] && return + restarted=1 + docker compose start parts >/dev/null + # Report readiness only once it is actually serving. Announcing "restarted" + # the instant the container exists is how you get a 502 from the very next + # request. + for _ in $(seq 1 60); do + if docker exec parts python -c \ + "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8100/healthz',timeout=2)" \ + >/dev/null 2>&1; then + echo "App restarted and serving." + return + fi + sleep 0.5 + done + echo "WARNING: the app was restarted but is not answering /healthz." >&2 +} trap restart EXIT -# What the database expects to exist, read while nothing can be writing. +# What the database expects to exist, read while nothing can be writing to it. EXPECTED="$(docker run --rm -v "$VOLUME":/data -e PARTS_DB=/data/parts.db \ --entrypoint python parts -m app.admin list-image-files | sort)" docker run --rm -v "$VOLUME":/data -v "$OUT_DIR":/out alpine \ tar -czf "/out/$NAME" -C /data . -# Verify every referenced photo actually made it into the archive. -IN_TAR="$(tar -tzf "$OUT_DIR/$NAME" | sed -n 's#^\./images/##p' | sort)" +# 1. Every referenced photo is actually in the archive. +IN_TAR="$(tar -tzf "$ARCHIVE" | sed -n 's#^\./images/##p' | sort)" MISSING="$(comm -23 <(printf '%s\n' "$EXPECTED" | grep -v '^$' || true) \ <(printf '%s\n' "$IN_TAR" | grep -v '^$' || true) || true)" if [ -n "$MISSING" ]; then @@ -44,5 +66,25 @@ if [ -n "$MISSING" ]; then exit 1 fi -COUNT="$(printf '%s\n' "$EXPECTED" | grep -c . || true)" -echo "Wrote $OUT_DIR/$NAME ($(du -h "$OUT_DIR/$NAME" | cut -f1)), $COUNT photo(s) verified present." +# 2. The archived database actually opens and passes SQLite's own check. An +# archive that exists but won't restore is the worst kind of backup. +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"; restart' EXIT +tar -xzf "$ARCHIVE" -C "$WORK" ./parts.db +VERDICT="$(docker run --rm -v "$WORK":/v --entrypoint python parts -c " +import sqlite3 +c = sqlite3.connect('/v/parts.db') +ok = c.execute('PRAGMA integrity_check').fetchone()[0] +parts = c.execute('SELECT COUNT(*) FROM parts').fetchone()[0] +photos = c.execute('SELECT COUNT(*) FROM part_images').fetchone()[0] +print(f'{ok}|{parts}|{photos}') +")" +STATUS="${VERDICT%%|*}" +COUNTS="${VERDICT#*|}" +if [ "$STATUS" != "ok" ]; then + echo "BACKUP UNUSABLE — archived database failed integrity_check: $STATUS" >&2 + exit 1 +fi + +echo "Wrote $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))" +echo "Verified: integrity_check ok, ${COUNTS%|*} part(s), ${COUNTS#*|} photo(s), all photo files present."