Durability pass: tests, clearer diversity/classify behavior, Calm Filters foundation

- Add pytest suite (34 tests) covering scoring thresholds, dedup clustering +
  representative selection + time window, brief source/category diversity,
  avoid-term phrase matching, and text canonicalization/truncation.
- Rewrite _select_diverse with an explicit, tested contract (best-first, one
  per source, backfill, then inject a second category by evicting the
  lowest-ranked pick).
- classify_articles now returns attempted/succeeded/skipped (ClassifyReport) so
  silent model failures are visible in both the cycle and classify output.
- Fix clean_text truncation to stay within max_len (ellipsis no longer
  overshoots).
- New filters.py: canonical FilterPrefs shape (include/mute topics+flavors,
  avoid_terms, pauses) and pure word/phrase-boundary matching engine seeding
  Calm Filters. Not yet wired into the API.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_briefs.py

@@ -0,0 +1,50 @@
+from goodnews.briefs import _select_diverse
+
+
+def row(id, source, category):
+    # _select_diverse only reads these three keys; plain dicts support [] access.
+    return {"id": id, "source_name": source, "default_category": category}
+
+
+def test_prefers_distinct_sources_best_first():
+    rows = [
+        row(1, "A", "science"),
+        row(2, "A", "science"),   # same source as #1 — should be skipped while others remain
+        row(3, "B", "science"),
+        row(4, "C", "environment"),
+    ]
+    selected = _select_diverse(rows, limit=3)
+    ids = [r["id"] for r in selected]
+    assert ids == [1, 3, 4]  # one per source, ranked order preserved
+
+
+def test_backfills_when_sources_exhausted():
+    rows = [row(1, "A", "science"), row(2, "A", "science"), row(3, "A", "science")]
+    selected = _select_diverse(rows, limit=2)
+    assert len(selected) == 2  # repeats source A only because no others exist
+
+
+def test_injects_second_category_without_shrinking():
+    rows = [
+        row(1, "A", "science"),
+        row(2, "B", "science"),
+        row(3, "C", "science"),
+        row(4, "D", "environment"),  # the only other category, lowest ranked
+    ]
+    selected = _select_diverse(rows, limit=3)
+    cats = {r["default_category"] for r in selected}
+    assert len(selected) == 3
+    assert len(cats) >= 2  # environment swapped in for diversity
+    assert any(r["default_category"] == "environment" for r in selected)
+
+
+def test_keeps_single_category_when_no_alternative_exists():
+    rows = [row(1, "A", "science"), row(2, "B", "science"), row(3, "C", "science")]
+    selected = _select_diverse(rows, limit=3)
+    assert len(selected) == 3
+    assert {r["default_category"] for r in selected} == {"science"}
+
+
+def test_never_returns_more_than_limit():
+    rows = [row(i, f"S{i}", "science") for i in range(10)]
+    assert len(_select_diverse(rows, limit=5)) == 5

tests/test_dedup.py

@@ -0,0 +1,83 @@
+import math
+from array import array
+
+import pytest
+
+from goodnews.db import connect, init_db
+from goodnews.dedup import _day_ordinal, _unit, cluster_duplicates
+
+
+def test_unit_normalizes_to_length_one():
+    u = _unit([3.0, 4.0])
+    assert math.isclose(u[0], 0.6) and math.isclose(u[1], 0.8)
+
+
+def test_unit_handles_zero_vector():
+    assert _unit([0.0, 0.0]) == [0.0, 0.0]
+
+
+def test_day_ordinal_parsing():
+    from datetime import date
+
+    assert _day_ordinal("2026-05-30T12:00:00+00:00") == date(2026, 5, 30).toordinal()
+    assert _day_ordinal(None) == 0
+    assert _day_ordinal("not-a-date") == 0
+
+
+@pytest.fixture
+def conn():
+    c = connect(":memory:")
+    init_db(c)
+    c.execute(
+        "INSERT INTO sources (id, name, feed_url, trust_score) VALUES (1, 'S1', 'http://s1/feed', 5)"
+    )
+    yield c
+    c.close()
+
+
+def _add(conn, article_id, vector, constructive, when="2026-05-30T10:00:00+00:00"):
+    conn.execute(
+        "INSERT INTO articles (id, source_id, canonical_url, title, published_at, url_hash) "
+        "VALUES (?, 1, ?, ?, ?, ?)",
+        (article_id, f"http://s1/{article_id}", f"Title {article_id}", when, f"hash{article_id}"),
+    )
+    conn.execute(
+        "INSERT INTO article_scores (article_id, constructive_score, agency_score, "
+        "human_benefit_score, cortisol_score, ragebait_score, pr_risk_score, accepted) "
+        "VALUES (?, ?, 0, 0, 0, 0, 0, 1)",
+        (article_id, constructive),
+    )
+    conn.execute(
+        "INSERT INTO article_embeddings (article_id, vector, dim, model) VALUES (?, ?, ?, 'test')",
+        (article_id, array("f", vector).tobytes(), len(vector)),
+    )
+    conn.commit()
+
+
+def test_near_duplicates_collapse_to_highest_ranked(conn):
+    # A and B are near-identical; A has the higher constructive score so it wins.
+    _add(conn, 1, [1.0, 0.0, 0.0, 0.0], constructive=9)      # A (rep)
+    _add(conn, 2, [0.99, 0.02, 0.0, 0.0], constructive=3)    # B (dup of A)
+    _add(conn, 3, [0.0, 1.0, 0.0, 0.0], constructive=8)      # C (distinct)
+
+    stats = cluster_duplicates(conn, threshold=0.86, window_days=3)
+    assert stats["duplicates"] == 1
+
+    dup_of = {r["id"]: r["duplicate_of"] for r in conn.execute("SELECT id, duplicate_of FROM articles")}
+    assert dup_of[2] == 1     # B points at A
+    assert dup_of[1] is None  # A is representative
+    assert dup_of[3] is None  # C stands alone
+
+
+def test_distinct_articles_are_not_clustered(conn):
+    _add(conn, 1, [1.0, 0.0, 0.0, 0.0], constructive=5)
+    _add(conn, 2, [0.0, 1.0, 0.0, 0.0], constructive=5)
+    stats = cluster_duplicates(conn, threshold=0.86, window_days=3)
+    assert stats["duplicates"] == 0
+
+
+def test_outside_time_window_not_clustered(conn):
+    _add(conn, 1, [1.0, 0.0, 0.0, 0.0], constructive=9, when="2026-05-30T10:00:00+00:00")
+    _add(conn, 2, [1.0, 0.0, 0.0, 0.0], constructive=3, when="2026-05-10T10:00:00+00:00")
+    stats = cluster_duplicates(conn, threshold=0.86, window_days=3)
+    assert stats["duplicates"] == 0  # identical vectors, but 20 days apart

tests/test_filters.py

@@ -0,0 +1,88 @@
+from datetime import datetime, timezone
+
+from goodnews.filters import (
+    FilterPrefs,
+    Pause,
+    filter_articles,
+    text_matches_avoid_terms,
+)
+
+NOW = datetime(2026, 6, 1, tzinfo=timezone.utc)
+
+
+def art(topic="science", flavor="discovery", title="A calm discovery", description=""):
+    return {"topic": topic, "flavor": flavor, "title": title, "description": description}
+
+
+# --- avoid-term matching: the trust-critical pure function ---
+
+def test_single_word_matches_whole_word_only():
+    assert text_matches_avoid_terms("New cancer drug approved", ["cancer"])
+    assert not text_matches_avoid_terms("Cancerous growth studied", ["cancer"])
+
+
+def test_substring_does_not_match():
+    # "pan" must not match "pandemic"
+    assert not text_matches_avoid_terms("Pandemic preparedness improves", ["pan"])
+
+
+def test_phrase_matches_as_phrase():
+    assert text_matches_avoid_terms("The stock market crashed today", ["stock market"])
+    assert not text_matches_avoid_terms("Stocks and other markets", ["stock market"])
+
+
+def test_punctuation_and_case_normalized():
+    assert text_matches_avoid_terms("An Anti-Aging breakthrough", ["anti aging"])
+    assert text_matches_avoid_terms("ELECTION results", ["election"])
+
+
+def test_empty_inputs_are_safe():
+    assert not text_matches_avoid_terms("", ["cancer"])
+    assert not text_matches_avoid_terms("anything", [])
+    assert not text_matches_avoid_terms(None, ["cancer"])
+
+
+# --- filter_articles over the canonical prefs ---
+
+def test_empty_prefs_pass_everything_through():
+    items = [art(), art(topic="health")]
+    assert filter_articles(items, FilterPrefs(), NOW) == items
+
+
+def test_mute_topic_drops_matching_articles():
+    items = [art(topic="science"), art(topic="health")]
+    prefs = FilterPrefs.from_dict({"mute_topics": ["health"]})
+    out = filter_articles(items, prefs, NOW)
+    assert [a["topic"] for a in out] == ["science"]
+
+
+def test_include_topics_keeps_only_those():
+    items = [art(topic="science"), art(topic="animals"), art(topic="health")]
+    prefs = FilterPrefs.from_dict({"include_topics": ["science", "animals"]})
+    out = filter_articles(items, prefs, NOW)
+    assert {a["topic"] for a in out} == {"science", "animals"}
+
+
+def test_avoid_terms_match_title_and_description():
+    items = [art(title="Update on the election"), art(description="about an election too"), art()]
+    prefs = FilterPrefs.from_dict({"avoid_terms": ["election"]})
+    out = filter_articles(items, prefs, NOW)
+    assert len(out) == 1
+
+
+def test_active_pause_hides_topic_but_expired_does_not():
+    items = [art(topic="health")]
+    active = FilterPrefs.from_dict(
+        {"pauses": [{"kind": "topic", "value": "health", "until": "2026-06-02T00:00:00Z"}]}
+    )
+    expired = FilterPrefs.from_dict(
+        {"pauses": [{"kind": "topic", "value": "health", "until": "2026-05-01T00:00:00Z"}]}
+    )
+    assert filter_articles(items, active, NOW) == []
+    assert filter_articles(items, expired, NOW) == items
+
+
+def test_pause_active_helper():
+    assert Pause("topic", "health", "2026-06-02T00:00:00Z").active(NOW)
+    assert not Pause("topic", "health", "2026-05-01T00:00:00Z").active(NOW)
+    assert not Pause("topic", "health", "garbage").active(NOW)

tests/test_scoring.py

@@ -0,0 +1,48 @@
+from goodnews.scoring import score_article
+
+
+def test_constructive_story_is_accepted():
+    s = score_article("Community volunteers restore creek habitat", "A hopeful recovery effort", 3)
+    assert s["accepted"] == 1
+    assert s["constructive_score"] >= 5
+    assert s["reason_code"] == "heuristic_constructive_candidate"
+
+
+def test_neutral_story_needs_review():
+    s = score_article("The weather report for tomorrow", None, 3)
+    assert s["accepted"] == 0
+    assert s["reason_code"] == "heuristic_needs_review"
+
+
+def test_cortisol_heavy_is_rejected():
+    s = score_article("War and death as murder and attack escalate", None, 3)
+    assert s["accepted"] == 0
+    assert s["cortisol_score"] > 5
+    assert s["reason_code"] == "heuristic_reject_cortisol_heavy"
+
+
+def test_ragebait_is_rejected_before_cortisol():
+    s = score_article("Senator slams rival and sparks backlash", None, 3)
+    assert s["accepted"] == 0
+    assert s["ragebait_score"] > 3
+    assert s["reason_code"] == "heuristic_reject_ragebait_language"
+
+
+def test_pr_risk_from_source_and_terms_rejects():
+    s = score_article("Startup announces funding round and unveils brand", None, 6)
+    assert s["pr_risk_score"] > 7
+    assert s["accepted"] == 0
+
+
+def test_all_scores_within_bounds():
+    s = score_article("breakthrough cure restores hope " * 10, "progress " * 20, 3)
+    for key in (
+        "constructive_score",
+        "cortisol_score",
+        "ragebait_score",
+        "agency_score",
+        "human_benefit_score",
+        "novelty_score",
+        "pr_risk_score",
+    ):
+        assert 0 <= s[key] <= 10, key

tests/test_text.py

@@ -0,0 +1,36 @@
+from goodnews.text import canonicalize_url, clean_text, sha256_text
+
+
+def test_clean_text_strips_tags_and_entities():
+    assert clean_text("<p>Hello&amp; &nbsp;world</p>") == "Hello& world"
+
+
+def test_clean_text_truncates():
+    out = clean_text("x" * 50, max_len=10)
+    assert out.endswith("...") and len(out) <= 10
+
+
+def test_clean_text_empty_is_none():
+    assert clean_text("") is None
+    assert clean_text(None) is None
+
+
+def test_canonicalize_strips_tracking_params():
+    url = "https://Example.com/story?utm_source=x&id=7&fbclid=abc"
+    out = canonicalize_url(url)
+    assert "utm_source" not in out and "fbclid" not in out
+    assert "id=7" in out
+    assert out.startswith("https://example.com")  # scheme/host lowercased
+
+
+def test_canonicalize_sorts_query_for_stable_hash():
+    a = canonicalize_url("https://e.com/p?b=2&a=1")
+    b = canonicalize_url("https://e.com/p?a=1&b=2")
+    assert a == b
+    assert sha256_text(a) == sha256_text(b)
+
+
+def test_canonicalize_rejects_non_http():
+    assert canonicalize_url("ftp://e.com/x") is None
+    assert canonicalize_url("javascript:alert(1)") is None
+    assert canonicalize_url(None) is None