Make the backup script wait for readiness and verify the archive restores

Running it against the live service exposed two gaps. It announced "App
restarted" the moment the container existed, so the very next request got a 502
from an app that wasn't listening yet; it now polls /healthz until the app is
actually serving. And it only checked that the archive contained the right
files — it now opens the archived database and runs SQLite's integrity_check,
because an archive that exists but won't restore is the worst kind of backup.

End to end on the server, with a throwaway part and photo inserted to exercise
the verification path: 2.6 seconds of downtime, archive verified, service
answering immediately on return, throwaway data removed afterwards.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tools/backup.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
-# Consistent backup of the parts inventory.
+# Consistent, verified backup of the parts inventory.
 #
 # The database and the photos are two resources that reference each other, so
 # capturing them at different moments is not a backup: a photo deleted between
-# the two steps leaves the saved database pointing at a file that isn't in the
+# the two steps leaves the saved database pointing at a file the archive does
-# archive, and one added leaves the reverse. Neither is repairable afterwards.
+# not contain, and one added leaves the reverse. Neither is repairable
+# afterwards, and pruning only fixes the harmless direction.
 #
-# So the app is stopped for the few seconds it takes to copy both. With no
+# So the app is stopped for the couple of seconds the copy takes. With no
-# process attached, parts.db and its -wal/-shm sidecars are a consistent set and
+# process attached, parts.db and any -wal/-shm sidecars are a consistent set and
 # the images directory cannot move underneath us.
 #
 #   ./tools/backup.sh [output-directory]
@@ -15,27 +16,48 @@ set -euo pipefail
 
 OUT_DIR="$(cd "${1:-$PWD}" && pwd)"
 cd "$(dirname "$0")/.."
-STAMP="$(date +%F-%H%M%S)"
+NAME="parts-backup-$(date +%F-%H%M%S).tar.gz"
-NAME="parts-backup-$STAMP.tar.gz"
+ARCHIVE="$OUT_DIR/$NAME"
-VOLUME="$(docker compose config --format json 2>/dev/null \
+
-  | python3 -c 'import json,sys; print(list(json.load(sys.stdin)["volumes"])[0])' 2>/dev/null || echo parts_data)"
+VOLUME="$(docker volume ls --format '{{.Name}}' | grep -E '^parts.*data$' | head -1)"
-VOLUME="$(docker volume ls --format '{{.Name}}' | grep -E "parts.*data" | head -1)"
+if [ -z "$VOLUME" ]; then
+  echo "Could not find the parts data volume." >&2
+  exit 1
+fi
 
 echo "Stopping the app so both halves are captured at one moment..."
 docker compose stop parts >/dev/null
 
-restart() { docker compose start parts >/dev/null && echo "App restarted."; }
+restarted=0
+restart() {
+  [ "$restarted" = 1 ] && return
+  restarted=1
+  docker compose start parts >/dev/null
+  # Report readiness only once it is actually serving. Announcing "restarted"
+  # the instant the container exists is how you get a 502 from the very next
+  # request.
+  for _ in $(seq 1 60); do
+    if docker exec parts python -c \
+      "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8100/healthz',timeout=2)" \
+      >/dev/null 2>&1; then
+      echo "App restarted and serving."
+      return
+    fi
+    sleep 0.5
+  done
+  echo "WARNING: the app was restarted but is not answering /healthz." >&2
+}
 trap restart EXIT
 
-# What the database expects to exist, read while nothing can be writing.
+# What the database expects to exist, read while nothing can be writing to it.
 EXPECTED="$(docker run --rm -v "$VOLUME":/data -e PARTS_DB=/data/parts.db \
   --entrypoint python parts -m app.admin list-image-files | sort)"
 
 docker run --rm -v "$VOLUME":/data -v "$OUT_DIR":/out alpine \
   tar -czf "/out/$NAME" -C /data .
 
-# Verify every referenced photo actually made it into the archive.
+# 1. Every referenced photo is actually in the archive.
-IN_TAR="$(tar -tzf "$OUT_DIR/$NAME" | sed -n 's#^\./images/##p' | sort)"
+IN_TAR="$(tar -tzf "$ARCHIVE" | sed -n 's#^\./images/##p' | sort)"
 MISSING="$(comm -23 <(printf '%s\n' "$EXPECTED" | grep -v '^$' || true) \
                     <(printf '%s\n' "$IN_TAR" | grep -v '^$' || true) || true)"
 if [ -n "$MISSING" ]; then
@@ -44,5 +66,25 @@ if [ -n "$MISSING" ]; then
   exit 1
 fi
 
-COUNT="$(printf '%s\n' "$EXPECTED" | grep -c . || true)"
+# 2. The archived database actually opens and passes SQLite's own check. An
-echo "Wrote $OUT_DIR/$NAME ($(du -h "$OUT_DIR/$NAME" | cut -f1)), $COUNT photo(s) verified present."
+#    archive that exists but won't restore is the worst kind of backup.
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"; restart' EXIT
+tar -xzf "$ARCHIVE" -C "$WORK" ./parts.db
+VERDICT="$(docker run --rm -v "$WORK":/v --entrypoint python parts -c "
+import sqlite3
+c = sqlite3.connect('/v/parts.db')
+ok = c.execute('PRAGMA integrity_check').fetchone()[0]
+parts = c.execute('SELECT COUNT(*) FROM parts').fetchone()[0]
+photos = c.execute('SELECT COUNT(*) FROM part_images').fetchone()[0]
+print(f'{ok}|{parts}|{photos}')
+")"
+STATUS="${VERDICT%%|*}"
+COUNTS="${VERDICT#*|}"
+if [ "$STATUS" != "ok" ]; then
+  echo "BACKUP UNUSABLE — archived database failed integrity_check: $STATUS" >&2
+  exit 1
+fi
+
+echo "Wrote $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))"
+echo "Verified: integrity_check ok, ${COUNTS%|*} part(s), ${COUNTS#*|} photo(s), all photo files present."