Read-time: full-article "Full story · ~N min" badge (Option B)

Replaces the gist-based read-time with the SOURCE article's full read time — the contrast that sells the gist ("calm 1-min version here; ~10 min for the deep dive"). - goodnews/readtime.py: word_count_from_html (strips script/style/nav/header/ footer/form/button/aside furniture before counting) + source_read_minutes (~225 wpm, 200-word floor, None when extraction looks failed/too thin). - articles.source_words + read_checked_at columns (count only, never the body; fits the privacy posture). Idempotent migration. - enrich.fetch_source_words + enrich_read_times: a bounded, retry-guarded cycle step (mirrors the image enrichers) that counts words for recent accepted articles. Only ever writes a real count; never overwrites good with zero. Wired into the cycle after recent-image enrichment. - queries: source_words flows through _ARTICLE_COLUMNS; api exposes source_read_minutes on Article (null when unknown). - home3: News card shows "Full story · ~N min", hidden entirely when null (no misleading "1 min"). - Tests: furniture stripping, threshold/rounding, enrich idempotency + no zero-overwrite, API null handling. 412 backend. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-23 08:09:00 -04:00
parent bdf3b1f47b
commit dc23277b38
8 changed files with 230 additions and 7 deletions
@@ -174,6 +174,47 @@ def fetch_og_image(url: str | None) -> str | None:
    return None  # too many redirects


+# Word counting reads more of the body than image metadata (which only needs <head>).
+_READ_MAX_BYTES = 900_000
+
+
+def fetch_source_words(url: str | None) -> int | None:
+    """Fetch a page and return its full-article word count (furniture stripped), or
+    None on any failure or a too-thin extraction (JS/video/paywall pages). Same SSRF
+    safety as fetch_og_image; we read the count only, never store the body."""
+    from .readtime import source_read_minutes, word_count_from_html
+    opener = urllib.request.build_opener(_NoRedirect)
+    for _ in range(MAX_REDIRECTS + 1):
+        if not url:
+            return None
+        parts = urlsplit(url)
+        if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
+            return None
+        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
+        try:
+            response = opener.open(request, timeout=TIMEOUT)
+        except (urllib.error.URLError, OSError, ValueError):
+            return None
+        status = getattr(response, "status", 200) or 200
+        if status in (301, 302, 303, 307, 308):
+            location = response.headers.get("Location")
+            response.close()
+            if not location:
+                return None
+            url = urljoin(url, location)
+            continue
+        if "html" not in response.headers.get("Content-Type", "").lower():
+            response.close()
+            return None
+        try:
+            body = response.read(_READ_MAX_BYTES)
+        finally:
+            response.close()
+        words = word_count_from_html(body)
+        return words if source_read_minutes(words) is not None else None
+    return None  # too many redirects
+
+
 def _image_dimensions(data: bytes) -> "tuple[int, int] | None":
    """Best-effort (width, height) from an image file's header bytes — PNG, GIF,
    JPEG, WebP. Returns None for formats we can't cheaply measure (e.g. SVG)."""
@@ -411,3 +452,42 @@ def enrich_summarized_images(
        if enrich_article_image(conn, row["id"], fetch=fetch, retry_days=retry_days):
            found += 1
    return found
+
+
+def enrich_read_times(
+    conn: sqlite3.Connection, fetch=fetch_source_words, limit: int = 40, retry_days: int = 14
+) -> int:
+    """Give recent accepted articles a full-article word count, so the front door can
+    show "Full story · ~N min" next to our one-minute gist. Bounded per run (mirrors
+    the image enrichers); fetches each article once, retrying a failed/too-thin
+    extraction only after `retry_days`. Returns how many real counts were stored."""
+    rows = conn.execute(
+        """
+        SELECT a.id, a.canonical_url FROM articles a
+        JOIN article_scores s ON s.article_id = a.id
+        WHERE s.accepted = 1 AND a.duplicate_of IS NULL
+          AND a.source_words IS NULL
+          AND (a.read_checked_at IS NULL OR a.read_checked_at < datetime('now', ?))
+        ORDER BY COALESCE(a.published_at, a.discovered_at) DESC
+        LIMIT ?
+        """,
+        (f"-{retry_days} days", limit),
+    ).fetchall()
+    found = 0
+    for row in rows:
+        try:
+            words = fetch(row["canonical_url"])
+        except Exception:
+            words = None
+        # Only ever write a REAL count; never overwrite a good value with null/zero.
+        # Always stamp the check time so failed/thin pages aren't re-fetched until retry.
+        if words:
+            conn.execute(
+                "UPDATE articles SET source_words = ?, read_checked_at = CURRENT_TIMESTAMP WHERE id = ?",
+                (words, row["id"]),
+            )
+            found += 1
+        else:
+            conn.execute("UPDATE articles SET read_checked_at = CURRENT_TIMESTAMP WHERE id = ?", (row["id"],))
+    conn.commit()
+    return found