Fix upload concurrency, bound the request body, make backups consistent
Four audit findings on the photo feature. The upload route was async, so its blocking SQLite work ran on the event loop: under contention it stalled every other request for SQLite's busy timeout, not just its own. It also read the photo count without the write lock, so overlapping uploads all observed the same total and stored past the ceiling together. It is a synchronous endpoint now, running in the threadpool, taking BEGIN IMMEDIATE before re-checking the part, the ceiling and the position, and committing before it returns. Reverting either half makes the new test die with the same TimeoutError the audit reported. The 8MB cap protected nothing: Starlette parses and spools an entire multipart body before a route's dependencies run — before the login check — so the bytes were already on disk by the time anything rejected them, and an anonymous caller could make us write them. A plain ASGI middleware outside routing now refuses an over-large body first, and Caddy enforces the same ceiling at the edge. The documented backup captured the database and the photos at two different moments while the app stayed writable, so a photo deleted in between left the saved database pointing at a file the archive did not contain. tools/backup.sh stops the app for the few seconds the copy takes and verifies afterwards that every referenced photo is in the archive. Cleanup could destroy data rather than merely litter: prune-images could delete a file between an upload writing it and inserting its row, and deletions unlinked before their transaction committed. Pruning now ignores anything under an hour old unless forced, and deletes commit before unlinking — an orphaned file is recoverable, a row without its photo is not. check-images reports drift in both directions and fails only on the direction that loses data. Checks go from 271 to 291. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,9 @@ genuinely overlapping inside SQLite, which means a real server and real threads.
|
||||
|
||||
import http.cookiejar
|
||||
import json
|
||||
import secrets
|
||||
import struct
|
||||
import zlib
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
@@ -42,6 +45,44 @@ def call(opener, base, path, method="GET", body=None):
|
||||
return json.load(opener(r, timeout=60))
|
||||
|
||||
|
||||
def make_png(w=40, h=40, rgb=(120, 60, 200)):
|
||||
raw = b"".join(b"\x00" + bytes(rgb) * w for _ in range(h))
|
||||
|
||||
def chunk(tag, data):
|
||||
body = tag + data
|
||||
return struct.pack(">I", len(data)) + body + struct.pack(">I", zlib.crc32(body))
|
||||
|
||||
return (b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(raw))
|
||||
+ chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def multipart(content, filename="photo.png", ctype="image/png", fields=None):
|
||||
"""Build a multipart body by hand — no third-party HTTP client in the tests."""
|
||||
boundary = "----parts" + secrets.token_hex(8)
|
||||
out = []
|
||||
for key, value in (fields or {}).items():
|
||||
out.append(f"--{boundary}\r\nContent-Disposition: form-data; "
|
||||
f'name="{key}"\r\n\r\n{value}\r\n'.encode())
|
||||
out.append(f"--{boundary}\r\nContent-Disposition: form-data; "
|
||||
f'name="file"; filename="{filename}"\r\n'
|
||||
f"Content-Type: {ctype}\r\n\r\n".encode() + content + b"\r\n")
|
||||
out.append(f"--{boundary}--\r\n".encode())
|
||||
return b"".join(out), f"multipart/form-data; boundary={boundary}"
|
||||
|
||||
|
||||
def post_image(base, part_id, content, timeout=30):
|
||||
body, ctype = multipart(content)
|
||||
r = urllib.request.Request(f"{base}/api/parts/{part_id}/images", data=body,
|
||||
method="POST", headers={"Content-Type": ctype})
|
||||
try:
|
||||
with urllib.request.urlopen(r, timeout=timeout) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code
|
||||
|
||||
|
||||
@contextmanager
|
||||
def server(port, **env_extra):
|
||||
"""Run a real uvicorn against a throwaway database."""
|
||||
@@ -62,7 +103,7 @@ def server(port, **env_extra):
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise RuntimeError("server never started")
|
||||
yield base
|
||||
yield base, tmp
|
||||
finally:
|
||||
proc.send_signal(signal.SIGINT)
|
||||
try:
|
||||
@@ -95,7 +136,7 @@ def session_phase():
|
||||
password = "concurrency-test-password"
|
||||
with server(PORT + 1, PARTS_AUTH="on", PARTS_PASSWORD=password,
|
||||
PARTS_SECRET="concurrency-test-secret",
|
||||
PARTS_LOGIN_MAX_FAILURES="500") as base:
|
||||
PARTS_LOGIN_MAX_FAILURES="500") as (base, _tmp):
|
||||
# --- the plain case first ---
|
||||
keeper = session_for(base, password)
|
||||
others = [session_for(base, password) for _ in range(5)]
|
||||
@@ -142,7 +183,7 @@ def session_phase():
|
||||
|
||||
|
||||
def main():
|
||||
with server(PORT, PARTS_AUTH="off"):
|
||||
with server(PORT, PARTS_AUTH="off") as (_base, tmp):
|
||||
# --- concurrent decrements must not lose updates ---
|
||||
pid = req("/api/parts", "POST", {"name": "race target", "quantity": 100})["id"]
|
||||
with ThreadPoolExecutor(max_workers=50) as ex:
|
||||
@@ -244,6 +285,55 @@ def main():
|
||||
check("all concurrent creates are searchable",
|
||||
req("/api/parts?q=bulk&limit=100")["total"] == 25,
|
||||
str(req("/api/parts?q=bulk&limit=100")["total"]))
|
||||
# --- concurrent photo uploads ---
|
||||
# An async endpoint doing blocking SQLite work stalls the whole event
|
||||
# loop under contention; an unlocked count lets every overlapping upload
|
||||
# observe the same total and sail past the ceiling together. This checks
|
||||
# both: the ceiling holds exactly, and the server stays responsive while
|
||||
# it is being hammered.
|
||||
shot_part = req("/api/parts", "POST", {"name": "photo race", "quantity": 1})["id"]
|
||||
attempts = 20
|
||||
latency = []
|
||||
|
||||
def ping():
|
||||
start = time.time()
|
||||
try:
|
||||
call(urllib.request.urlopen, _base, "/healthz")
|
||||
except Exception:
|
||||
pass
|
||||
latency.append(time.time() - start)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=attempts + 4) as ex:
|
||||
futures = [ex.submit(post_image, _base, shot_part, make_png(60, 60, (i * 9, 40, 90)))
|
||||
for i in range(attempts)]
|
||||
for _ in range(4):
|
||||
ex.submit(ping)
|
||||
codes = [f.result() for f in futures]
|
||||
|
||||
created = codes.count(201)
|
||||
refused = codes.count(409)
|
||||
check("no upload returned a server error",
|
||||
not [c for c in codes if c >= 500], str(sorted(set(codes))))
|
||||
check("every upload was answered", len(codes) == attempts and None not in codes)
|
||||
check("exactly the ceiling was stored", created == 12, f"{created} created, {refused} refused")
|
||||
check("the rest were refused as over the limit", created + refused == attempts,
|
||||
str(sorted(set(codes))))
|
||||
stored = req(f"/api/parts/{shot_part}/images")["items"]
|
||||
check("the database agrees with the ceiling", len(stored) == 12, str(len(stored)))
|
||||
check("positions are unique", len({i["position"] for i in stored}) == 12,
|
||||
str(sorted(i["position"] for i in stored)))
|
||||
|
||||
image_dir = os.path.join(tmp, "images")
|
||||
on_disk = len(os.listdir(image_dir))
|
||||
check("refused uploads left no files behind", on_disk == 12, f"{on_disk} files")
|
||||
|
||||
worst = max(latency) if latency else 0
|
||||
# A blocked event loop parks unrelated requests for SQLite's busy
|
||||
# timeout, which is 15s. Anything in that neighbourhood means the
|
||||
# upload is running on the loop again.
|
||||
check("the server stayed responsive during the upload storm",
|
||||
worst < 5.0, f"slowest /healthz was {worst:.1f}s")
|
||||
|
||||
session_phase()
|
||||
|
||||
print()
|
||||
|
||||
Reference in New Issue
Block a user