From ba801d90f65d2c965a8837e6e3036e820ee28f77 Mon Sep 17 00:00:00 2001
From: jay <thejayman77@gmail.com>
Date: Sun, 31 May 2026 01:36:53 +0000
Subject: [PATCH] Make paywalls systemic + fix ArticleCard reactivity
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- ArticleCard: derive safeHref from article.url and reset image-failure state
  when the article changes, so in-place replacements re-evaluate correctly
  (clears the Svelte capture warning; build is warning-free again).
- Downweight paywalled stories below readable ones (stable sort) when composing
  the daily five and in feed results — the brief now leads readable and rarely
  hands over a locked door.
- review_sources gains a 'paywall-heavy' advisory flag (Nature, New Scientist
  flag at 100%); never auto-deactivates.
- New Scientist/Nature kept active but no longer reach the daily five; they
  remain browsable with the label + Replace.
- Tests: brief readability preference + paywall-heavy flag (79 total).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../src/lib/components/ArticleCard.svelte     | 18 ++++++----
 goodnews/api.py                               |  3 ++
 goodnews/briefs.py                            |  6 ++++
 goodnews/sources.py                           |  7 +++-
 ideas.md                                      |  2 +-
 tests/test_brief_paywall.py                   | 36 +++++++++++++++++++
 tests/test_review_paywall.py                  | 25 +++++++++++++
 7 files changed, 89 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_brief_paywall.py
 create mode 100644 tests/test_review_paywall.py

diff --git a/frontend/src/lib/components/ArticleCard.svelte b/frontend/src/lib/components/ArticleCard.svelte
index 976e81c..3803642 100644
--- a/frontend/src/lib/components/ArticleCard.svelte
+++ b/frontend/src/lib/components/ArticleCard.svelte
@@ -1,11 +1,17 @@
 <script>
   let { article, onaction, onreplace, hero = false } = $props();
 
-  let imgOk = $state(!!article.image_url);
-  let hasImg = $derived(!!(article.image_url && imgOk));
-
-  const safeHref =
-    typeof article.url === 'string' && /^https?:\/\//.test(article.url) ? article.url : '#';
+  // Cards can be replaced in place, so derive from the current article prop and
+  // reset the image-failure flag whenever the image URL changes.
+  let failed = $state(false);
+  $effect(() => {
+    void article.image_url;
+    failed = false;
+  });
+  let hasImg = $derived(!!article.image_url && !failed);
+  let safeHref = $derived(
+    typeof article.url === 'string' && /^https?:\/\//.test(article.url) ? article.url : '#'
+  );
 
   function act(kind, value) {
     if (value) onaction?.(kind, value);
@@ -24,7 +30,7 @@
   {#if hasImg}
     <a class="media" href={safeHref} target="_blank" rel="noopener">
       <img src={article.image_url} alt="" loading="lazy" referrerpolicy="no-referrer"
-           onerror={() => (imgOk = false)} />
+           onerror={() => (failed = true)} />
     </a>
   {/if}
 
diff --git a/goodnews/api.py b/goodnews/api.py
index d788a14..ca0cf67 100644
--- a/goodnews/api.py
+++ b/goodnews/api.py
@@ -281,6 +281,9 @@ def create_app() -> FastAPI:
                         conn, topic=topic, flavor=flavor, accepted_only=accepted_only,
                         limit=limit, offset=offset, **kw,
                     )
+        # Keep the top of a browse view readable: stable-sort paywalled items
+        # below readable ones (composite order preserved within each group).
+        rows = sorted(rows, key=lambda r: is_paywalled(r["canonical_url"]))
         return FeedResponse(
             topic=topic,
             flavor=flavor,
diff --git a/goodnews/briefs.py b/goodnews/briefs.py
index 9fc9d0f..76c0ea7 100644
--- a/goodnews/briefs.py
+++ b/goodnews/briefs.py
@@ -3,6 +3,8 @@ from __future__ import annotations
 import sqlite3
 from datetime import date
 
+from .paywall import is_paywalled
+
 
 def build_daily_brief(
     conn: sqlite3.Connection,
@@ -24,6 +26,10 @@ def build_daily_brief(
     ).lastrowid
 
     rows = _candidate_articles(conn, target_date, window_days)
+    # A calm daily brief shouldn't repeatedly hand the reader a locked door:
+    # push paywalled candidates below readable ones (stable, so composite order
+    # is preserved within each group) before selecting the five.
+    rows = sorted(rows, key=lambda r: is_paywalled(r["canonical_url"]))
     selected = _select_diverse(rows, limit)
     for index, row in enumerate(selected, start=1):
         conn.execute(
diff --git a/goodnews/sources.py b/goodnews/sources.py
index 1f80530..5ccb720 100644
--- a/goodnews/sources.py
+++ b/goodnews/sources.py
@@ -7,6 +7,8 @@ from datetime import datetime, timezone
 from pathlib import Path
 from urllib.parse import urlsplit
 
+from .paywall import is_paywalled
+
 
 def load_sources(path: Path | str) -> list[dict]:
     data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
@@ -183,7 +185,7 @@ def review_sources(
         recent = conn.execute(
             """
             SELECT sc.accepted, sc.cortisol_score, sc.ragebait_score, a.duplicate_of,
-                   COALESCE(a.published_at, a.discovered_at) AS dt
+                   a.canonical_url, COALESCE(a.published_at, a.discovered_at) AS dt
             FROM articles a
             JOIN article_scores sc ON sc.article_id = a.id
             WHERE a.source_id = ?
@@ -220,6 +222,9 @@ def review_sources(
                 avg_rage = sum(r["ragebait_score"] or 0 for r in recent) / n
                 if avg_rage > 3:
                     reasons.append(f"high ragebait (avg {avg_rage:.1f})")
+                paywalled = sum(1 for r in recent if is_paywalled(r["canonical_url"])) / n
+                if paywalled > 0.5:
+                    reasons.append(f"paywall-heavy ({paywalled * 100:.0f}%)")
 
         flag = 1 if reasons else 0
         reason = "; ".join(reasons) if reasons else None
diff --git a/ideas.md b/ideas.md
index b7bd2d7..5d10202 100644
--- a/ideas.md
+++ b/ideas.md
@@ -8,7 +8,7 @@
 | Favorite/save articles  [tabled: needs accounts/logins for a larger footprint]
 | Soothing background colors/gradients per each category as you scroll.  Maybe a user preference.  [tabled: revisit deliberately; if done, whisper-quiet translucent tints, not neon]
 - I really like the coloring for the metadata highlighting in each card (The grading bubbles)
-* Some articles are behind paywalls.. what can we do?  [done: domain-level paywall flag + readable hero; future: down-weight paywalled sources like New Scientist]
+* Some articles are behind paywalls.. what can we do?  [done: domain-level paywall flag, readable hero, paywalled downweighted out of the daily five, and a "paywall-heavy" advisory source-health flag (Nature/New Scientist). Replace handles any that remain in browse.]
 * After an article is read, can we add a refresh button to fetch a replacement for it in the list?  [done: "Find one I can read" / Replace swaps in the next readable article]
 * I want the top 5 to be tere, but I want the remaining categories to be hidden behing their selections.  So the main screen should show just the current highlights, and then the other articles should only be visible when in that category.  [done]
 * Title headings should be a little larger -- if you select Today, Today should look like a proper heading, bold and beautiful.  Switching to Wondow should show "Wonder" all nice and whatnot.  [done]
\ No newline at end of file
diff --git a/tests/test_brief_paywall.py b/tests/test_brief_paywall.py
new file mode 100644
index 0000000..74075e8
--- /dev/null
+++ b/tests/test_brief_paywall.py
@@ -0,0 +1,36 @@
+from datetime import date
+
+from goodnews.db import connect, init_db
+from goodnews.briefs import build_daily_brief, show_brief
+from goodnews.paywall import is_paywalled
+
+
+def test_brief_prefers_readable_over_higher_scored_paywalled():
+    c = connect(":memory:"); init_db(c)
+    today = date.today().isoformat()
+    for sid in range(1, 8):
+        c.execute("INSERT INTO sources (id,name,feed_url,trust_score) VALUES (?,?,?,5)",
+                  (sid, f"S{sid}", f"http://s{sid}/f"))
+
+    def add(aid, sid, url, score):
+        c.execute("INSERT INTO articles (id,source_id,canonical_url,title,published_at,url_hash) "
+                  "VALUES (?,?,?,?,?,?)", (aid, sid, url, f"t{aid}", today + "T12:00:00+00:00", f"h{aid}"))
+        c.execute("INSERT INTO article_scores (article_id,constructive_score,agency_score,human_benefit_score,"
+                  "cortisol_score,ragebait_score,pr_risk_score,accepted,topic,flavor) "
+                  "VALUES (?,?,?,?,0,0,2,1,'science','discovery')", (aid, score, score, score))
+
+    # Paywalled ones are scored HIGHER — readability must still win for the five.
+    add(1, 1, "https://www.newscientist.com/a", 9)
+    add(2, 2, "https://www.nature.com/b", 9)
+    add(3, 3, "https://phys.org/c", 4)
+    add(4, 4, "https://www.goodnewsnetwork.org/d", 4)
+    add(5, 5, "https://e360.yale.edu/e", 4)
+    add(6, 6, "https://news.mongabay.com/f", 4)
+    add(7, 7, "https://theconversation.com/g", 4)
+    c.commit()
+
+    build_daily_brief(c, brief_date=today, limit=5, replace=True)
+    urls = [r["canonical_url"] for r in show_brief(c, brief_date=today, limit=10)]
+    c.close()
+    assert len(urls) == 5
+    assert not any(is_paywalled(u) for u in urls)  # five readable chosen over paywalled
diff --git a/tests/test_review_paywall.py b/tests/test_review_paywall.py
new file mode 100644
index 0000000..9ceaccf
--- /dev/null
+++ b/tests/test_review_paywall.py
@@ -0,0 +1,25 @@
+import pytest
+from datetime import datetime, timezone
+
+from goodnews.db import connect, init_db
+from goodnews.sources import review_sources
+
+
+@pytest.fixture
+def conn():
+    c = connect(":memory:"); init_db(c)
+    c.execute("INSERT INTO sources (id,name,feed_url,trust_score) VALUES (1,'Pay','http://p/f',5)")
+    yield c; c.close()
+
+
+def test_paywall_heavy_flagged(conn):
+    now = datetime.now(timezone.utc).isoformat()
+    for i in range(20):
+        url = f"https://www.newscientist.com/{i}" if i < 15 else f"https://phys.org/{i}"
+        conn.execute("INSERT INTO articles (id,source_id,canonical_url,title,published_at,url_hash) "
+                     "VALUES (?,1,?,?,?,?)", (i, url, f"t{i}", now, f"h{i}"))
+        conn.execute("INSERT INTO article_scores (article_id,cortisol_score,ragebait_score,accepted) "
+                     "VALUES (?,1,0,1)", (i,))
+    conn.commit()
+    flagged = review_sources(conn)
+    assert flagged and "paywall-heavy" in flagged[0]["reason"]