images: fix two fetcher bugs + add source-level image-rights policy (Codex)
Fetcher (the two remaining bugs Codex found):
- Real redirects are now followed. _NoRedirect makes urllib RAISE HTTPError on 3xx, so
the old status-branch was dead code (mocked tests masked it). Handle 301/302/303/307/308
HTTPError as redirects (re-validate the destination); classify 4xx≠429 as PERMANENT
(negative-cached), 429/5xx/network as transient. Real-opener redirect + 404/5xx tests.
- The megapixel ceiling is now enforced: explicit `w*h > _MAX_PIXELS` check BEFORE load()
(Pillow only warns at MAX_IMAGE_PIXELS). Test with a lowered ceiling.
Image-rights policy (per Codex + owner decision — only cache what's cleared):
- sources.image_policy: 'cache' (re-host a downscaled copy — license/permission/PD only),
'remote' (hotlink the publisher's image — the conservative DEFAULT), 'none' (no image).
- newsimg.display_url resolves the display URL per policy; applied in Article.from_row so
feed/brief/history return the right URL, and in share.py (og/twitter still reference the
publisher's own image, never re-hosted). warm() + /api/img both gated on 'cache'.
- Frontend uses the server-resolved image_url (reverted the hardcoded /api/img); the
graceful retry covers remote hotlinks too. Admin: per-source image-policy selector +
POST /api/admin/sources/{id}/image-policy. Default 'remote' → nothing re-hosted until
a source is explicitly cleared.
445 backend + 36 frontend tests pass.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+26
-3
@@ -351,7 +351,9 @@ class Article(BaseModel):
|
||||
title=row["title"],
|
||||
description=row.get("description"),
|
||||
url=row["canonical_url"],
|
||||
image_url=row.get("image_url"),
|
||||
# Resolve per the source's image policy: our cached copy, the publisher's URL
|
||||
# (hotlink), or none — so we never re-host an image we haven't cleared.
|
||||
image_url=newsimg.display_url(row["id"], row.get("image_policy"), row.get("image_url")),
|
||||
published_at=row.get("published_at"),
|
||||
source=row["source_name"],
|
||||
source_id=row.get("source_id"),
|
||||
@@ -591,6 +593,10 @@ class SourcePaywallBody(BaseModel):
|
||||
override: str | None = None # None = use domain rule · 'free' · 'paywalled'
|
||||
|
||||
|
||||
class SourceImagePolicyBody(BaseModel):
|
||||
policy: str = "remote" # 'cache' · 'remote' (default) · 'none'
|
||||
|
||||
|
||||
class CandidateSuggestBody(BaseModel):
|
||||
feed_url: str = ""
|
||||
name: str | None = None
|
||||
@@ -1111,7 +1117,7 @@ def create_app() -> FastAPI:
|
||||
with get_conn() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.id, a.title, a.description, a.image_url, a.canonical_url, "
|
||||
"a.duplicate_of, a.source_id, src.name AS source_name, s.reason_text, s.accepted, "
|
||||
"a.duplicate_of, a.source_id, src.name AS source_name, src.image_policy, s.reason_text, s.accepted, "
|
||||
"(SELECT group_concat(t.tag) FROM article_tags t WHERE t.article_id = a.id) AS tags "
|
||||
"FROM articles a JOIN sources src ON src.id = a.source_id "
|
||||
"LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
|
||||
@@ -1416,6 +1422,22 @@ def create_app() -> FastAPI:
|
||||
conn.commit()
|
||||
return {"ok": True, "override": ov}
|
||||
|
||||
@app.post("/api/admin/sources/{sid}/image-policy")
|
||||
def admin_source_image_policy(sid: int, body: SourceImagePolicyBody, request: Request) -> dict:
|
||||
# Image rights policy: 'cache' (re-host a downscaled copy — only for sources we've
|
||||
# cleared: open license / permission / public-domain), 'remote' (hotlink the
|
||||
# publisher's image), 'none' (no image). Default is the conservative 'remote'.
|
||||
pol = body.policy
|
||||
if pol not in ("cache", "remote", "none"):
|
||||
raise HTTPException(status_code=422, detail="policy must be 'cache', 'remote', or 'none'")
|
||||
with get_conn() as conn:
|
||||
_require_admin(conn, request)
|
||||
cur = conn.execute("UPDATE sources SET image_policy = ? WHERE id = ?", (pol, sid))
|
||||
if cur.rowcount == 0:
|
||||
raise HTTPException(status_code=404, detail="source not found")
|
||||
conn.commit()
|
||||
return {"ok": True, "policy": pol}
|
||||
|
||||
# --- Source candidates (supervised add-a-source pipeline) ----------------
|
||||
|
||||
def _candidate_dict(row) -> dict:
|
||||
@@ -2346,7 +2368,8 @@ def create_app() -> FastAPI:
|
||||
with get_conn() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.image_url FROM articles a JOIN article_scores s ON s.article_id = a.id "
|
||||
"WHERE a.id = ? AND s.accepted = 1 AND a.duplicate_of IS NULL",
|
||||
"JOIN sources src ON src.id = a.source_id "
|
||||
"WHERE a.id = ? AND s.accepted = 1 AND a.duplicate_of IS NULL AND src.image_policy = 'cache'",
|
||||
(article_id,),
|
||||
).fetchone()
|
||||
url = row["image_url"] if row else None
|
||||
|
||||
Reference in New Issue
Block a user