Bounded hero-image enrichment (og:image for brief items only)

The grid stays typographic; the hero is the one intentional visual slot. At brief-build time we fetch a hero-quality image for the daily five that lack one: - enrich.py reads ONLY a page's <head> og:image/twitter:image and stores just the URL (never the body). - SSRF-guarded: http(s) only, 6s timeout, 300KB cap, <=3 manual redirects each re-validated, and hosts rejected if any resolved address is private, loopback, link-local, multicast, reserved, or unspecified. - image_checked_at column caches success AND failure, so an article is never retried forever. - Wired into build-brief and cycle (brief items only, only if image missing and unchecked). Everything else stays metadata-only. - Verified live: today's five all carry images (feed + enriched). Tests: og:image parser, head-only scope, IP guard across internal ranges, and enrich success + failure-caching (85 total). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-05-31 12:37:41 +00:00
parent 3858380ffe
commit 9e8eddf46d
5 changed files with 230 additions and 1 deletions
@@ -0,0 +1,55 @@
+import pytest
+
+from goodnews.db import connect, init_db
+from goodnews.enrich import og_image_from_html, _host_is_public, enrich_brief_images
+
+
+def test_og_image_parser_handles_attr_order_and_fallback():
+    assert og_image_from_html(b'<head><meta property="og:image" content="https://x.com/a.jpg"></head>') == "https://x.com/a.jpg"
+    assert og_image_from_html(b'<meta content="https://x.com/b.jpg" property="og:image">') == "https://x.com/b.jpg"
+    assert og_image_from_html(b'<meta name="twitter:image" content="https://x.com/c.jpg">') == "https://x.com/c.jpg"
+    assert og_image_from_html(b"<html><body>nope</body></html>") is None
+
+
+def test_og_image_only_reads_head():
+    # a meta after </head> must be ignored
+    assert og_image_from_html(b'<head></head><meta property="og:image" content="https://x.com/d.jpg">') is None
+
+
+def test_host_public_guard_blocks_internal_ranges():
+    assert _host_is_public("8.8.8.8")          # public
+    assert not _host_is_public("127.0.0.1")    # loopback
+    assert not _host_is_public("10.0.0.1")     # private
+    assert not _host_is_public("192.168.1.1")  # private
+    assert not _host_is_public("169.254.0.1")  # link-local
+    assert not _host_is_public("0.0.0.0")      # unspecified
+    assert not _host_is_public("")
+    assert not _host_is_public(None)
+
+
+@pytest.fixture
+def conn():
+    c = connect(":memory:"); init_db(c)
+    c.execute("INSERT INTO sources (id,name,feed_url,trust_score) VALUES (1,'S','http://s/f',5)")
+    c.execute("INSERT INTO articles (id,source_id,canonical_url,title,url_hash) VALUES (1,1,'https://phys.org/x','t1','h1')")
+    c.execute("INSERT INTO daily_briefs (id,brief_date,title) VALUES (1,'2026-05-31','B')")
+    c.execute("INSERT INTO daily_brief_items (brief_id,article_id,rank) VALUES (1,1,1)")
+    c.commit(); yield c; c.close()
+
+
+def test_enrich_sets_image_and_stamps(conn):
+    calls = []
+    found = enrich_brief_images(conn, "2026-05-31", fetch=lambda u: calls.append(u) or "https://img.example/p.jpg")
+    assert found == 1 and calls == ["https://phys.org/x"]
+    r = conn.execute("SELECT image_url, image_checked_at FROM articles WHERE id=1").fetchone()
+    assert r["image_url"] == "https://img.example/p.jpg" and r["image_checked_at"] is not None
+
+
+def test_enrich_caches_failure_and_does_not_retry(conn):
+    calls = []
+    fail = lambda u: calls.append(u) or None
+    assert enrich_brief_images(conn, "2026-05-31", fetch=fail) == 0
+    r = conn.execute("SELECT image_url, image_checked_at FROM articles WHERE id=1").fetchone()
+    assert r["image_url"] is None and r["image_checked_at"] is not None  # checked, cached
+    assert enrich_brief_images(conn, "2026-05-31", fetch=fail) == 0
+    assert len(calls) == 1  # not retried once checked