Resolve the backup volume from the container, and verify photo sizes
Three audit items on the backup path. The script matched volume names by pattern and took the first hit, so a stale or restored volume could be backed up instead of the live one — and every verification step would then faithfully confirm the wrong database. It now asks the container what is mounted at /data and refuses ambiguity. Tested against a decoy volume that the old pattern would have matched first. check-images treated a photo as healthy if a file with the right name existed, so a truncated or partially restored file passed. It compares each file against the byte count its row records now; a one-byte stand-in for a 123KB photo is reported as WRONG SIZE and exits non-zero. The backup runs the same check against the stopped volume and exits 2 when the source was already damaged — still writing the archive, because a faithful copy of imperfect data is worth having, but saying so. The restart trap was installed after the app had already been stopped, so an interrupt in between could leave the service down with nothing to bring it back. The trap goes in first now, covers INT and TERM as well as EXIT, and records whether the container was running beforehand so a backup of an already-stopped app leaves it stopped. Verified on the live host: healthy source exits 0, damaged source exits 2 with the archive still written and verified, decoy volume correctly ignored, service answering immediately afterwards, and the test rows removed. Checks go from 290 to 294. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+74
-34
@@ -14,49 +14,87 @@
|
||||
# ./tools/backup.sh [output-directory]
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER=parts
|
||||
OUT_DIR="$(cd "${1:-$PWD}" && pwd)"
|
||||
cd "$(dirname "$0")/.."
|
||||
NAME="parts-backup-$(date +%F-%H%M%S).tar.gz"
|
||||
ARCHIVE="$OUT_DIR/$NAME"
|
||||
|
||||
VOLUME="$(docker volume ls --format '{{.Name}}' | grep -E '^parts.*data$' | head -1)"
|
||||
if [ -z "$VOLUME" ]; then
|
||||
echo "Could not find the parts data volume." >&2
|
||||
# --- resolve the data volume from the container, never by guessing ------------
|
||||
# Matching volume names by pattern can silently pick a stale or restored volume,
|
||||
# and every subsequent check would then faithfully verify the wrong database.
|
||||
# Ask the container what is actually mounted at /data, and refuse ambiguity.
|
||||
MOUNTS="$(docker inspect "$CONTAINER" --format \
|
||||
'{{range .Mounts}}{{if eq .Destination "/data"}}{{.Type}}|{{.Name}}|{{.Source}}{{"\n"}}{{end}}{{end}}' \
|
||||
| grep . || true)"
|
||||
MOUNT_COUNT="$(printf '%s\n' "$MOUNTS" | grep -c . || true)"
|
||||
if [ "$MOUNT_COUNT" -ne 1 ]; then
|
||||
echo "Expected exactly one mount at /data in container '$CONTAINER', found $MOUNT_COUNT." >&2
|
||||
printf '%s\n' "$MOUNTS" >&2
|
||||
exit 1
|
||||
fi
|
||||
MOUNT_TYPE="${MOUNTS%%|*}"
|
||||
MOUNT_REST="${MOUNTS#*|}"
|
||||
MOUNT_NAME="${MOUNT_REST%%|*}"
|
||||
MOUNT_SOURCE="${MOUNT_REST#*|}"
|
||||
case "$MOUNT_TYPE" in
|
||||
volume) DATA_REF="$MOUNT_NAME" ;;
|
||||
bind) DATA_REF="$MOUNT_SOURCE" ;;
|
||||
*) echo "Unsupported mount type at /data: $MOUNT_TYPE" >&2; exit 1 ;;
|
||||
esac
|
||||
echo "Backing up $MOUNT_TYPE '$DATA_REF' (mounted at /data in $CONTAINER)."
|
||||
|
||||
echo "Stopping the app so both halves are captured at one moment..."
|
||||
docker compose stop parts >/dev/null
|
||||
|
||||
restarted=0
|
||||
restart() {
|
||||
[ "$restarted" = 1 ] && return
|
||||
restarted=1
|
||||
docker compose start parts >/dev/null
|
||||
# Report readiness only once it is actually serving. Announcing "restarted"
|
||||
# the instant the container exists is how you get a 502 from the very next
|
||||
# request.
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec parts python -c \
|
||||
"import urllib.request;urllib.request.urlopen('http://127.0.0.1:8100/healthz',timeout=2)" \
|
||||
>/dev/null 2>&1; then
|
||||
echo "App restarted and serving."
|
||||
return
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
echo "WARNING: the app was restarted but is not answering /healthz." >&2
|
||||
# --- restore the original state whatever happens ------------------------------
|
||||
# Installed *before* stopping anything: an interrupt during the stop would
|
||||
# otherwise leave the service down with no trap to bring it back.
|
||||
WAS_RUNNING="$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo false)"
|
||||
WORK=""
|
||||
finished=0
|
||||
cleanup() {
|
||||
[ "$finished" = 1 ] && return
|
||||
finished=1
|
||||
[ -n "$WORK" ] && rm -rf "$WORK"
|
||||
if [ "$WAS_RUNNING" = "true" ]; then
|
||||
docker compose start "$CONTAINER" >/dev/null 2>&1 || true
|
||||
# Report readiness only once it is actually serving. Announcing "restarted"
|
||||
# the instant the container exists is how you get a 502 from the very next
|
||||
# request.
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$CONTAINER" python -c \
|
||||
"import urllib.request;urllib.request.urlopen('http://127.0.0.1:8100/healthz',timeout=2)" \
|
||||
>/dev/null 2>&1; then
|
||||
echo "App restarted and serving."
|
||||
return
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
echo "WARNING: the app was restarted but is not answering /healthz." >&2
|
||||
else
|
||||
echo "The app was not running before this backup; leaving it stopped."
|
||||
fi
|
||||
}
|
||||
trap restart EXIT
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# What the database expects to exist, read while nothing can be writing to it.
|
||||
EXPECTED="$(docker run --rm -v "$VOLUME":/data -e PARTS_DB=/data/parts.db \
|
||||
--entrypoint python parts -m app.admin list-image-files | sort)"
|
||||
if [ "$WAS_RUNNING" = "true" ]; then
|
||||
echo "Stopping the app so both halves are captured at one moment..."
|
||||
docker compose stop "$CONTAINER" >/dev/null
|
||||
fi
|
||||
|
||||
docker run --rm -v "$VOLUME":/data -v "$OUT_DIR":/out alpine \
|
||||
tar -czf "/out/$NAME" -C /data .
|
||||
run_in_data() { docker run --rm -v "$DATA_REF":/data "$@"; }
|
||||
|
||||
# 1. Every referenced photo is actually in the archive.
|
||||
# --- is the source data itself healthy? ---------------------------------------
|
||||
DATA_STATUS=ok
|
||||
if ! run_in_data -e PARTS_DB=/data/parts.db --entrypoint python parts \
|
||||
-m app.admin check-images; then
|
||||
DATA_STATUS=damaged
|
||||
fi
|
||||
|
||||
EXPECTED="$(run_in_data -e PARTS_DB=/data/parts.db --entrypoint python parts \
|
||||
-m app.admin list-image-files | sort)"
|
||||
|
||||
run_in_data -v "$OUT_DIR":/out alpine tar -czf "/out/$NAME" -C /data .
|
||||
|
||||
# --- is the archive complete and restorable? ----------------------------------
|
||||
IN_TAR="$(tar -tzf "$ARCHIVE" | sed -n 's#^\./images/##p' | sort)"
|
||||
MISSING="$(comm -23 <(printf '%s\n' "$EXPECTED" | grep -v '^$' || true) \
|
||||
<(printf '%s\n' "$IN_TAR" | grep -v '^$' || true) || true)"
|
||||
@@ -66,10 +104,7 @@ if [ -n "$MISSING" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2. The archived database actually opens and passes SQLite's own check. An
|
||||
# archive that exists but won't restore is the worst kind of backup.
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"; restart' EXIT
|
||||
tar -xzf "$ARCHIVE" -C "$WORK" ./parts.db
|
||||
VERDICT="$(docker run --rm -v "$WORK":/v --entrypoint python parts -c "
|
||||
import sqlite3
|
||||
@@ -88,3 +123,8 @@ fi
|
||||
|
||||
echo "Wrote $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))"
|
||||
echo "Verified: integrity_check ok, ${COUNTS%|*} part(s), ${COUNTS#*|} photo(s), all photo files present."
|
||||
if [ "$DATA_STATUS" != ok ]; then
|
||||
echo "WARNING: the source data was already inconsistent (see check-images above)." >&2
|
||||
echo "The archive is a faithful copy of it, but the damage predates this backup." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user