images: fix two fetcher bugs + add source-level image-rights policy (Codex)

Fetcher (the two remaining bugs Codex found):
- Real redirects are now followed. _NoRedirect makes urllib RAISE HTTPError on 3xx, so
  the old status-branch was dead code (mocked tests masked it). Handle 301/302/303/307/308
  HTTPError as redirects (re-validate the destination); classify 4xx≠429 as PERMANENT
  (negative-cached), 429/5xx/network as transient. Real-opener redirect + 404/5xx tests.
- The megapixel ceiling is now enforced: explicit `w*h > _MAX_PIXELS` check BEFORE load()
  (Pillow only warns at MAX_IMAGE_PIXELS). Test with a lowered ceiling.

Image-rights policy (per Codex + owner decision — only cache what's cleared):
- sources.image_policy: 'cache' (re-host a downscaled copy — license/permission/PD only),
  'remote' (hotlink the publisher's image — the conservative DEFAULT), 'none' (no image).
- newsimg.display_url resolves the display URL per policy; applied in Article.from_row so
  feed/brief/history return the right URL, and in share.py (og/twitter still reference the
  publisher's own image, never re-hosted). warm() + /api/img both gated on 'cache'.
- Frontend uses the server-resolved image_url (reverted the hardcoded /api/img); the
  graceful retry covers remote hotlinks too. Admin: per-source image-policy selector +
  POST /api/admin/sources/{id}/image-policy. Default 'remote' → nothing re-hosted until
  a source is explicitly cleared.

445 backend + 36 frontend tests pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-30 14:01:11 -04:00
parent a55ba185a8
commit 8a7606e20d
10 changed files with 238 additions and 62 deletions
+26 -3
View File
@@ -351,7 +351,9 @@ class Article(BaseModel):
title=row["title"],
description=row.get("description"),
url=row["canonical_url"],
image_url=row.get("image_url"),
# Resolve per the source's image policy: our cached copy, the publisher's URL
# (hotlink), or none — so we never re-host an image we haven't cleared.
image_url=newsimg.display_url(row["id"], row.get("image_policy"), row.get("image_url")),
published_at=row.get("published_at"),
source=row["source_name"],
source_id=row.get("source_id"),
@@ -591,6 +593,10 @@ class SourcePaywallBody(BaseModel):
override: str | None = None # None = use domain rule · 'free' · 'paywalled'
class SourceImagePolicyBody(BaseModel):
policy: str = "remote" # 'cache' · 'remote' (default) · 'none'
class CandidateSuggestBody(BaseModel):
feed_url: str = ""
name: str | None = None
@@ -1111,7 +1117,7 @@ def create_app() -> FastAPI:
with get_conn() as conn:
row = conn.execute(
"SELECT a.id, a.title, a.description, a.image_url, a.canonical_url, "
"a.duplicate_of, a.source_id, src.name AS source_name, s.reason_text, s.accepted, "
"a.duplicate_of, a.source_id, src.name AS source_name, src.image_policy, s.reason_text, s.accepted, "
"(SELECT group_concat(t.tag) FROM article_tags t WHERE t.article_id = a.id) AS tags "
"FROM articles a JOIN sources src ON src.id = a.source_id "
"LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
@@ -1416,6 +1422,22 @@ def create_app() -> FastAPI:
conn.commit()
return {"ok": True, "override": ov}
@app.post("/api/admin/sources/{sid}/image-policy")
def admin_source_image_policy(sid: int, body: SourceImagePolicyBody, request: Request) -> dict:
# Image rights policy: 'cache' (re-host a downscaled copy — only for sources we've
# cleared: open license / permission / public-domain), 'remote' (hotlink the
# publisher's image), 'none' (no image). Default is the conservative 'remote'.
pol = body.policy
if pol not in ("cache", "remote", "none"):
raise HTTPException(status_code=422, detail="policy must be 'cache', 'remote', or 'none'")
with get_conn() as conn:
_require_admin(conn, request)
cur = conn.execute("UPDATE sources SET image_policy = ? WHERE id = ?", (pol, sid))
if cur.rowcount == 0:
raise HTTPException(status_code=404, detail="source not found")
conn.commit()
return {"ok": True, "policy": pol}
# --- Source candidates (supervised add-a-source pipeline) ----------------
def _candidate_dict(row) -> dict:
@@ -2346,7 +2368,8 @@ def create_app() -> FastAPI:
with get_conn() as conn:
row = conn.execute(
"SELECT a.image_url FROM articles a JOIN article_scores s ON s.article_id = a.id "
"WHERE a.id = ? AND s.accepted = 1 AND a.duplicate_of IS NULL",
"JOIN sources src ON src.id = a.source_id "
"WHERE a.id = ? AND s.accepted = 1 AND a.duplicate_of IS NULL AND src.image_policy = 'cache'",
(article_id,),
).fetchone()
url = row["image_url"] if row else None