From 1d715759822ffad2d3715e57b766bbdf6680f3df Mon Sep 17 00:00:00 2001
From: jay <thejayman77@gmail.com>
Date: Wed, 3 Jun 2026 18:08:40 +0000
Subject: [PATCH] Share pages: lazy, cached, our-own-words article summaries
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The /a/<id> page now carries an original short summary so it stands on its own,
without republishing the publisher's article:
- summarize.py: transient SSRF-guarded fetch of the article text → local LLM
  writes a 2-4 sentence ORIGINAL summary (our words). Cached in article_summaries
  forever; we store only our summary, never the body. Generated lazily (only for
  shared/viewed articles), de-duped so concurrent hits don't double-generate.
- /a serves cached-or-pending; when pending it shows a calm "summary on its way,
  read at {source}" note and self-polls /api/summary/<id>, swapping the summary
  in the moment it's ready (never blocks the page on the batch-tier LLM).
- Share menu warms generation on open so recipients usually get the rich version.
- Container reaches the arbiter at arbiter:8080 over caddy_web (LLM env added to
  the API container). 124 tests pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../src/lib/components/ArticleCard.svelte     |   7 +-
 goodnews/api.py                               |  46 +++++++-
 goodnews/db.py                                |   9 ++
 goodnews/share.py                             |  43 ++++++-
 goodnews/summarize.py                         | 108 ++++++++++++++++++
 tests/test_summary.py                         |  56 +++++++++
 6 files changed, 256 insertions(+), 13 deletions(-)
 create mode 100644 goodnews/summarize.py
 create mode 100644 tests/test_summary.py

diff --git a/frontend/src/lib/components/ArticleCard.svelte b/frontend/src/lib/components/ArticleCard.svelte
index ead2410..34f87bf 100644
--- a/frontend/src/lib/components/ArticleCard.svelte
+++ b/frontend/src/lib/components/ArticleCard.svelte
@@ -57,6 +57,11 @@
     }
     shareOpen = false;
   }
+  // Kick off summary generation when the user signals intent to share, so it's
+  // cached by the time a recipient opens the link.
+  function warmSummary() {
+    fetch(`/api/summary/${article.id}`).catch(() => {});
+  }
   async function copy(text, label) {
     try {
       await navigator.clipboard.writeText(text);
@@ -127,7 +132,7 @@
         </button>
       {/if}
       <span class="sharewrap">
-        <button class="share" onclick={() => (shareOpen = !shareOpen)} aria-expanded={shareOpen}>Share</button>
+        <button class="share" onclick={() => { shareOpen = !shareOpen; if (shareOpen) warmSummary(); }} aria-expanded={shareOpen}>Share</button>
         {#if shareOpen}
           <button class="backdrop" aria-label="Close" onclick={() => (shareOpen = false)}></button>
           <div class="sharemenu" role="menu">
diff --git a/goodnews/api.py b/goodnews/api.py
index 92b7f5e..877d2c8 100644
--- a/goodnews/api.py
+++ b/goodnews/api.py
@@ -31,7 +31,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
 
-from . import auth, email_send, feeds, oauth_google, queries, share
+from . import auth, email_send, feeds, oauth_google, queries, share, summarize
 from .db import connect
 from .filters import filter_articles, prefs_from_json
 from .hero import safe_to_lead
@@ -113,6 +113,28 @@ def _user_out(user: sqlite3.Row) -> dict:
     }
 
 
+# Articles whose summary is being generated right now — so concurrent pollers /
+# scrapers don't each kick off a duplicate LLM call.
+_summarizing: set[int] = set()
+
+
+def _run_summary(article_id: int) -> None:
+    try:
+        with get_conn() as conn:
+            summarize.generate_summary(conn, article_id)
+    except Exception:
+        pass
+    finally:
+        _summarizing.discard(article_id)
+
+
+def _kick_summary(article_id: int, background_tasks: BackgroundTasks) -> None:
+    if article_id in _summarizing:
+        return
+    _summarizing.add(article_id)
+    background_tasks.add_task(_run_summary, article_id)
+
+
 def _send_link_safe(email: str, link: str) -> None:
     """Send the magic link, swallowing failures (runs off the request path)."""
     try:
@@ -622,7 +644,7 @@ def create_app() -> FastAPI:
     # --- Public share/landing page for an article -------------------------
 
     @app.get("/a/{article_id}", response_class=HTMLResponse)
-    def share_page(article_id: str) -> HTMLResponse:
+    def share_page(article_id: str, background_tasks: BackgroundTasks) -> HTMLResponse:
         not_found = HTMLResponse(share.render_not_found(PUBLIC_BASE_URL), status_code=404)
         try:
             aid = int(article_id)
@@ -636,10 +658,22 @@ def create_app() -> FastAPI:
                 "LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
                 (aid,),
             ).fetchone()
-        # Only render real, accepted, non-duplicate stories.
-        if not row or row["duplicate_of"] is not None or not row["accepted"]:
-            return not_found
-        return HTMLResponse(share.render_share_page(dict(row), PUBLIC_BASE_URL))
+            # Only render real, accepted, non-duplicate stories.
+            if not row or row["duplicate_of"] is not None or not row["accepted"]:
+                return not_found
+            summary = summarize.get_summary(conn, aid)
+        if not summary:
+            _kick_summary(aid, background_tasks)  # generate for next time; page polls
+        return HTMLResponse(share.render_share_page(dict(row), PUBLIC_BASE_URL, summary=summary))
+
+    @app.get("/api/summary/{article_id}")
+    def article_summary(article_id: int, background_tasks: BackgroundTasks) -> dict:
+        with get_conn() as conn:
+            summary = summarize.get_summary(conn, article_id)
+        if summary:
+            return {"status": "ready", "summary": summary}
+        _kick_summary(article_id, background_tasks)
+        return {"status": "pending", "summary": None}
 
     @app.post("/api/import")
     def import_local(body: ImportBody, request: Request) -> dict:
diff --git a/goodnews/db.py b/goodnews/db.py
index 41fd957..ee2bddc 100644
--- a/goodnews/db.py
+++ b/goodnews/db.py
@@ -196,6 +196,15 @@ CREATE TABLE IF NOT EXISTS user_prefs (
     prefs_json TEXT NOT NULL DEFAULT '{}',
     updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
+
+-- Our OWN short summary of an article (generated on demand, cached forever).
+-- We store only our derived summary text — never the publisher's article body.
+CREATE TABLE IF NOT EXISTS article_summaries (
+    article_id INTEGER PRIMARY KEY REFERENCES articles(id) ON DELETE CASCADE,
+    summary TEXT NOT NULL,
+    model TEXT,
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
 """
 
 
diff --git a/goodnews/share.py b/goodnews/share.py
index ad16db2..ec5e187 100644
--- a/goodnews/share.py
+++ b/goodnews/share.py
@@ -17,7 +17,7 @@ def _tag(name: str, content: str | None, attr: str = "property") -> str:
     return f'<meta {attr}="{escape(name)}" content="{escape(content)}">'
 
 
-def render_share_page(article: dict, base_url: str) -> str:
+def render_share_page(article: dict, base_url: str, summary: str | None = None) -> str:
     aid = article["id"]
     title = (article.get("title") or "Upbeat Bytes").strip()
     why = (article.get("reason_text") or article.get("description")
@@ -49,6 +49,31 @@ def render_share_page(article: dict, base_url: str) -> str:
         if image else ""
     )
 
+    source_short = source.split(" ")[0] if source else "the source"
+    if summary:
+        summary_block = f'<p class="summary" id="summary">{escape(summary)}</p>'
+        poll = ""
+    else:
+        pending = (
+            f"✦ We’re putting together a quick summary — in the meantime, "
+            f"read the full story at {escape(source)} below."
+        )
+        summary_block = f'<p class="summary pending" id="summary">{pending}</p>'
+        # Quietly poll; swap the real summary in the moment it's cached.
+        poll = f"""<script>
+(function(){{
+  var id={aid}, box=document.getElementById('summary'), n=0;
+  function go(){{
+    n++;
+    fetch('/api/summary/'+id).then(function(r){{return r.json();}}).then(function(d){{
+      if(d&&d.status==='ready'&&d.summary){{ box.textContent=d.summary; box.className='summary'; }}
+      else if(n<12){{ setTimeout(go,2500); }}
+    }}).catch(function(){{ if(n<12) setTimeout(go,3000); }});
+  }}
+  go();
+}})();
+</script>"""
+
     return f"""<!doctype html>
 <html lang="en">
 <head>
@@ -76,8 +101,12 @@ def render_share_page(article: dict, base_url: str) -> str:
   .body {{ padding:24px 26px 28px; }}
   .src {{ color:var(--muted); font-size:.82rem; text-transform:uppercase; letter-spacing:.08em; }}
   h1 {{ font-family:"Iowan Old Style",Palatino,Georgia,serif; font-weight:600;
-        font-size:1.7rem; line-height:1.2; margin:6px 0 12px; }}
-  .why {{ color:#3b4754; font-style:italic; border-left:2px solid #e0eef3; padding-left:12px; margin:0 0 22px; }}
+        font-size:1.7rem; line-height:1.2; margin:6px 0 14px; }}
+  .summary {{ font-size:1.05rem; color:var(--ink); margin:0 0 18px; }}
+  .summary.pending {{ color:var(--muted); font-style:italic; }}
+  .why {{ color:#3b4754; font-size:.92rem; margin:0 0 22px; }}
+  .why .lbl {{ display:block; text-transform:uppercase; letter-spacing:.08em; font-size:.7rem;
+               color:var(--muted); margin-bottom:3px; }}
   .actions {{ display:flex; flex-wrap:wrap; gap:12px; align-items:center; }}
   .primary {{ background:var(--accent); color:#fff; text-decoration:none; font-weight:600;
               padding:12px 20px; border-radius:999px; display:inline-block; }}
@@ -95,15 +124,17 @@ def render_share_page(article: dict, base_url: str) -> str:
       <div class="body">
         <div class="src">{escape(source)}</div>
         <h1>{escape(title)}</h1>
-        <p class="why">{escape(why)}</p>
+        {summary_block}
+        <p class="why"><span class="lbl">Why it's here</span>{escape(why)}</p>
         <div class="actions">
-          <a class="primary" href="{escape(src_url)}" rel="noopener" data-src-click>Read the full story at {escape(source)}</a>
+          <a class="primary" href="{escape(src_url)}" rel="noopener" data-src-click>Read the full story at {escape(source_short)}</a>
           <a class="secondary" href="/">Explore more on Upbeat Bytes →</a>
         </div>
-        <p class="note">Upbeat Bytes links to the original publisher — it doesn't host the article.</p>
+        <p class="note">Upbeat Bytes summarizes in its own words and links to the original publisher — it doesn't host the article.</p>
       </div>
     </article>
   </main>
+  {poll}
 </body>
 </html>"""
 
diff --git a/goodnews/summarize.py b/goodnews/summarize.py
new file mode 100644
index 0000000..01d6b43
--- /dev/null
+++ b/goodnews/summarize.py
@@ -0,0 +1,108 @@
+"""On-demand, cached article summaries.
+
+Lazy: generated only for articles that actually get shared/viewed, then cached in
+article_summaries forever. We fetch the article text *transiently* and ask the
+local LLM for a short, ORIGINAL summary in our own words. We store only that
+summary — never the publisher's article body — and the page always credits and
+links to the source.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import urllib.error
+import urllib.request
+from urllib.parse import urljoin, urlsplit
+
+from .enrich import USER_AGENT, _NoRedirect, _host_is_public
+from .llm import LocalModelClient
+
+_FETCH_BYTES = 600_000
+_FETCH_TIMEOUT = 8
+_MAX_REDIRECTS = 3
+_DROP = re.compile(rb"<(script|style|noscript|template|svg)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
+_TAGS = re.compile(rb"<[^>]+>")
+_WS = re.compile(r"\s+")
+
+_SYSTEM = (
+    "You write a short, original summary of a news story for a calm, constructive news "
+    "site. Use your own words — never copy sentences from the source. 2 to 4 plain "
+    "sentences: what happened and why it's encouraging. No preamble, no markdown, no "
+    "headline — just the summary."
+)
+
+
+def _fetch_text(url: str) -> str:
+    """SSRF-guarded fetch of an article page, reduced to plain text (capped)."""
+    opener = urllib.request.build_opener(_NoRedirect)
+    for _ in range(_MAX_REDIRECTS + 1):
+        if not url:
+            return ""
+        parts = urlsplit(url)
+        if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
+            return ""
+        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
+        try:
+            response = opener.open(request, timeout=_FETCH_TIMEOUT)
+        except (urllib.error.URLError, OSError, ValueError):
+            return ""
+        status = getattr(response, "status", 200) or 200
+        if status in (301, 302, 303, 307, 308):
+            location = response.headers.get("Location")
+            response.close()
+            if not location:
+                return ""
+            url = urljoin(url, location)
+            continue
+        if "html" not in response.headers.get("Content-Type", "").lower():
+            response.close()
+            return ""
+        try:
+            raw = response.read(_FETCH_BYTES)
+        finally:
+            response.close()
+        raw = _DROP.sub(b" ", raw)
+        text = _TAGS.sub(b" ", raw).decode("utf-8", "replace")
+        return _WS.sub(" ", text).strip()[:4000]
+    return ""
+
+
+def summarize_article(client: LocalModelClient, title: str, snippet: str, body_text: str) -> str:
+    material = (body_text or snippet or title or "")[:4000]
+    user = f"Title: {title}\n\nArticle text:\n{material}\n\nWrite the summary."
+    messages = [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": user}]
+    resp = client._chat(client._build_payload(messages, None))
+    content = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or ""
+    return content.strip()[:1200]
+
+
+def get_summary(conn: sqlite3.Connection, article_id: int) -> str | None:
+    row = conn.execute(
+        "SELECT summary FROM article_summaries WHERE article_id = ?", (article_id,)
+    ).fetchone()
+    return row["summary"] if row else None
+
+
+def generate_summary(conn: sqlite3.Connection, article_id: int, client: LocalModelClient | None = None) -> str | None:
+    """Generate + cache a summary for one article. Returns it, or None if skipped."""
+    if get_summary(conn, article_id):
+        return get_summary(conn, article_id)  # already cached
+    row = conn.execute(
+        "SELECT a.title, a.description, a.canonical_url, a.duplicate_of, s.accepted "
+        "FROM articles a LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
+        (article_id,),
+    ).fetchone()
+    if not row or row["duplicate_of"] is not None or not row["accepted"]:
+        return None
+    client = client or LocalModelClient.from_env()
+    body = _fetch_text(row["canonical_url"])
+    summary = summarize_article(client, row["title"], row["description"] or "", body)
+    if not summary:
+        return None
+    conn.execute(
+        "INSERT OR REPLACE INTO article_summaries (article_id, summary, model) VALUES (?, ?, ?)",
+        (article_id, summary, client.model),
+    )
+    conn.commit()
+    return summary
diff --git a/tests/test_summary.py b/tests/test_summary.py
new file mode 100644
index 0000000..cdf33f8
--- /dev/null
+++ b/tests/test_summary.py
@@ -0,0 +1,56 @@
+from types import SimpleNamespace
+
+import pytest
+from fastapi.testclient import TestClient
+
+from goodnews import summarize
+from goodnews.db import connect, init_db
+
+
+def _db(path):
+    c = connect(str(path)); init_db(c)
+    c.execute("INSERT INTO sources (id,name,feed_url,trust_score) VALUES (1,'NPR','http://s/f',5)")
+    c.execute("INSERT INTO articles (id,source_id,canonical_url,title,url_hash,description) "
+              "VALUES (1,1,'https://npr.org/x','Voles return','h1','A snippet.')")
+    c.execute("INSERT INTO article_scores (article_id,accepted,reason_text) VALUES (1,1,'Hopeful.')")
+    c.commit()
+    return c
+
+
+def test_generate_caches_summary(tmp_path, monkeypatch):
+    c = _db(tmp_path / "t.sqlite3")
+    monkeypatch.setattr(summarize, "_fetch_text", lambda url: "Full article body about voles.")
+    monkeypatch.setattr(summarize, "summarize_article", lambda client, t, s, b: "Our own short summary.")
+    # avoid constructing a real LLM client
+    monkeypatch.setattr(summarize.LocalModelClient, "from_env", classmethod(lambda cls: SimpleNamespace(model="m")))
+    out = summarize.generate_summary(c, 1)
+    assert out == "Our own short summary."
+    assert summarize.get_summary(c, 1) == "Our own short summary."
+    # second call is a cache hit (no regeneration)
+    monkeypatch.setattr(summarize, "summarize_article", lambda *a: "DIFFERENT")
+    assert summarize.generate_summary(c, 1) == "Our own short summary."
+
+
+def test_summary_endpoint_pending_then_ready(tmp_path, monkeypatch):
+    db = tmp_path / "t.sqlite3"
+    _db(db).close()
+    monkeypatch.setenv("GOODNEWS_DB", str(db))
+    monkeypatch.setenv("GOODNEWS_PUBLIC_BASE_URL", "https://upbeatbytes.com")
+    import importlib
+    import goodnews.api as api
+    importlib.reload(api)
+    # stub the actual generation the background task runs
+    monkeypatch.setattr(
+        api.summarize, "generate_summary",
+        lambda conn, aid, client=None: conn.execute(
+            "INSERT OR REPLACE INTO article_summaries (article_id, summary) VALUES (?, 'Cached summary.')", (aid,)
+        ) and conn.commit(),
+    )
+    tc = TestClient(api.create_app())
+    first = tc.get("/api/summary/1").json()
+    assert first["status"] == "pending"
+    # TestClient runs the background task; the next call sees the cache
+    assert tc.get("/api/summary/1").json() == {"status": "ready", "summary": "Cached summary."}
+    # and the share page now embeds it (no pending poll)
+    html = tc.get("/a/1").text
+    assert "Cached summary." in html