diff --git a/goodnews/cli.py b/goodnews/cli.py index 3d935cd..ed1be69 100644 --- a/goodnews/cli.py +++ b/goodnews/cli.py @@ -9,7 +9,7 @@ from .briefs import build_daily_brief, show_brief from .db import connect, init_db from .localtime import local_today from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, dedup as run_dedup -from .enrich import enrich_brief_images +from .enrich import enrich_brief_images, enrich_summarized_images from .summarize import generate_summary, get_summary from .feeds import ( fetch_feed, @@ -127,6 +127,11 @@ def main() -> None: cycle_parser.add_argument("--base-url", help="OpenAI-compatible base URL for classify") cycle_parser.add_argument("--model", help="Local model name for classify") + enrich_images_parser = subparsers.add_parser( + "enrich-images", help="Backfill og:images for already-summarized articles that lack one" + ) + enrich_images_parser.add_argument("--limit", type=int, default=50, help="Max articles to fetch this batch") + dedup_parser = subparsers.add_parser("dedup", help="Cluster near-duplicate stories via local embeddings") dedup_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help="Cosine similarity cutoff") dedup_parser.add_argument("--window-days", type=int, default=DEFAULT_WINDOW_DAYS) @@ -266,6 +271,9 @@ def main() -> None: print("Dry run only; database was not updated.") elif args.command == "cycle": run_cycle(conn, args) + elif args.command == "enrich-images": + found = enrich_summarized_images(conn, limit=args.limit) + print(f"enrich-images: {found} new image(s) for summarized articles") elif args.command == "dedup": init_db(conn) client = llm_client_from_args(args) diff --git a/goodnews/enrich.py b/goodnews/enrich.py index e41f4a0..3b329d7 100644 --- a/goodnews/enrich.py +++ b/goodnews/enrich.py @@ -50,6 +50,13 @@ _GENERIC_IMAGE_MARKERS = ( "/placeholder", "share-default", "social-default", + # tracking pixels / spacers / data-URIs — never a real share image + "data:image", + "/pixel", + "1x1", + "spacer", + "/blank.", + "transparent.", ) @@ -207,3 +214,64 @@ def enrich_brief_images( found += 1 conn.commit() return found + + +def enrich_article_image( + conn: sqlite3.Connection, article_id: int, fetch=fetch_og_image, retry_days: int = 7 +) -> bool: + """Attention-triggered: fetch an og:image for ONE article that lacks one. + + Called when an article earns a summary (i.e. it's actually being read), so we + only spend a fetch on articles a reader has reached. Leaves an existing image + alone; retries a still-imageless article only after `retry_days`. Returns True + if a new image was stored. Best-effort — never raises. + """ + row = conn.execute( + """ + SELECT id, canonical_url FROM articles + WHERE id = ? + AND (image_url IS NULL OR image_url = '') + AND (image_checked_at IS NULL OR image_checked_at < datetime('now', ?)) + """, + (article_id, f"-{retry_days} days"), + ).fetchone() + if not row: + return False # has an image already, or checked too recently + try: + image = fetch(row["canonical_url"]) + except Exception: + image = None + conn.execute( + "UPDATE articles SET image_url = COALESCE(?, image_url), image_checked_at = CURRENT_TIMESTAMP " + "WHERE id = ?", + (image, article_id), + ) + conn.commit() + return bool(image) + + +def enrich_summarized_images( + conn: sqlite3.Connection, fetch=fetch_og_image, limit: int = 50, retry_days: int = 7 +) -> int: + """Slow backfill: give already-summarized, accepted articles an image if they + lack one. Run in modest batches so we never blast publishers. Returns count + of newly-found images. + """ + rows = conn.execute( + """ + SELECT a.id FROM articles a + JOIN article_summaries m ON m.article_id = a.id + JOIN article_scores s ON s.article_id = a.id + WHERE s.accepted = 1 AND a.duplicate_of IS NULL + AND (a.image_url IS NULL OR a.image_url = '') + AND (a.image_checked_at IS NULL OR a.image_checked_at < datetime('now', ?)) + ORDER BY a.id DESC + LIMIT ? + """, + (f"-{retry_days} days", limit), + ).fetchall() + found = 0 + for row in rows: + if enrich_article_image(conn, row["id"], fetch=fetch, retry_days=retry_days): + found += 1 + return found diff --git a/goodnews/summarize.py b/goodnews/summarize.py index 2fef235..8a0e5d8 100644 --- a/goodnews/summarize.py +++ b/goodnews/summarize.py @@ -104,4 +104,12 @@ def generate_summary(conn: sqlite3.Connection, article_id: int, client: LocalMod (article_id, summary, client.model), ) conn.commit() + # Attention-triggered image enrichment: a summarized article is one a reader + # has reached, so it's worth a real image. Best-effort — an image fetch + # failure must never break summarization. + try: + from .enrich import enrich_article_image + enrich_article_image(conn, article_id) + except Exception: + pass return summary diff --git a/tests/test_enrich_images.py b/tests/test_enrich_images.py new file mode 100644 index 0000000..19de071 --- /dev/null +++ b/tests/test_enrich_images.py @@ -0,0 +1,58 @@ +from goodnews.db import connect, init_db +from goodnews import enrich + + +def _setup(tmp_path): + c = connect(str(tmp_path / "t.db")); init_db(c) + c.execute("INSERT INTO sources (id,name,feed_url) VALUES (1,'S','http://s/f')") + return c + + +def _add(c, aid, *, image=None, accepted=1, dup=None, summarized=True): + c.execute( + "INSERT INTO articles (id,source_id,canonical_url,title,url_hash,image_url,duplicate_of,published_at) " + "VALUES (?,1,?,?,?,?,?,datetime('now'))", + (aid, f"http://s/{aid}", f"T{aid}", f"h{aid}", image, dup), + ) + c.execute("INSERT INTO article_scores (article_id,accepted) VALUES (?,?)", (aid, accepted)) + if summarized: + c.execute("INSERT INTO article_summaries (article_id,summary) VALUES (?, 's')", (aid,)) + c.commit() + + +def test_enrich_article_image_fills_missing(tmp_path): + c = _setup(tmp_path); _add(c, 1, image=None) + got = enrich.enrich_article_image(c, 1, fetch=lambda url: "http://img/og.jpg") + assert got is True + assert c.execute("SELECT image_url FROM articles WHERE id=1").fetchone()[0] == "http://img/og.jpg" + # image_checked_at is stamped so we don't re-fetch endlessly + assert c.execute("SELECT image_checked_at FROM articles WHERE id=1").fetchone()[0] is not None + + +def test_enrich_skips_when_image_present(tmp_path): + c = _setup(tmp_path); _add(c, 1, image="http://existing/img.jpg") + calls = [] + got = enrich.enrich_article_image(c, 1, fetch=lambda url: calls.append(url) or "http://new.jpg") + assert got is False and calls == [] # never fetched + assert c.execute("SELECT image_url FROM articles WHERE id=1").fetchone()[0] == "http://existing/img.jpg" + + +def test_enrich_records_checked_even_when_none_found(tmp_path): + c = _setup(tmp_path); _add(c, 1, image=None) + got = enrich.enrich_article_image(c, 1, fetch=lambda url: None) + assert got is False + # checked_at stamped → won't retry until the retry window passes + assert c.execute("SELECT image_checked_at FROM articles WHERE id=1").fetchone()[0] is not None + again = enrich.enrich_article_image(c, 1, fetch=lambda url: "http://late.jpg", retry_days=7) + assert again is False # still within retry window + + +def test_backfill_only_targets_summarized_accepted_imageless(tmp_path): + c = _setup(tmp_path) + _add(c, 1, image=None, summarized=True, accepted=1) # eligible + _add(c, 2, image=None, summarized=False, accepted=1) # no summary → skip + _add(c, 3, image=None, summarized=True, accepted=0) # rejected → skip + _add(c, 4, image="http://has.jpg", summarized=True) # has image → skip + n = enrich.enrich_summarized_images(c, fetch=lambda url: "http://og.jpg", limit=50) + assert n == 1 + assert c.execute("SELECT image_url FROM articles WHERE id=2").fetchone()[0] is None