Durability pass: tests, clearer diversity/classify behavior, Calm Filters foundation

- Add pytest suite (34 tests) covering scoring thresholds, dedup clustering +
  representative selection + time window, brief source/category diversity,
  avoid-term phrase matching, and text canonicalization/truncation.
- Rewrite _select_diverse with an explicit, tested contract (best-first, one
  per source, backfill, then inject a second category by evicting the
  lowest-ranked pick).
- classify_articles now returns attempted/succeeded/skipped (ClassifyReport) so
  silent model failures are visible in both the cycle and classify output.
- Fix clean_text truncation to stay within max_len (ellipsis no longer
  overshoots).
- New filters.py: canonical FilterPrefs shape (include/mute topics+flavors,
  avoid_terms, pauses) and pure word/phrase-boundary matching engine seeding
  Calm Filters. Not yet wired into the API.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

goodnews/briefs.py

@@ -141,21 +141,30 @@ def _candidate_articles(
 
 
 def _select_diverse(rows: list[sqlite3.Row], limit: int) -> list[sqlite3.Row]:
-    selected = []
-    seen_sources = set()
-    seen_categories = set()
+    """Pick up to `limit` items from `rows` (already ranked best-first).
 
+    Contract:
+    1. Prefer higher-ranked items.
+    2. Source diversity: take at most one item per source while other sources
+       remain; only repeat a source once distinct sources are exhausted.
+    3. Category diversity: if the result ended up single-category and a different
+       category is available in the pool, swap in the highest-ranked off-category
+       candidate by evicting the lowest-ranked currently-selected item (so we
+       gain breadth without dropping a higher-ranked pick).
+    """
+    selected: list[sqlite3.Row] = []
+    seen_sources: set = set()
+
+    # Pass 1: best-first, one per source.
     for row in rows:
         if len(selected) >= limit:
             break
-        source = row["source_name"]
-        category = row["default_category"]
-        if source in seen_sources and len(rows) > limit:
+        if row["source_name"] in seen_sources:
             continue
         selected.append(row)
-        seen_sources.add(source)
-        seen_categories.add(category)
+        seen_sources.add(row["source_name"])
 
+    # Pass 2: if short on distinct sources, backfill best-first regardless.
     if len(selected) < limit:
         selected_ids = {row["id"] for row in selected}
         for row in rows:
@@ -166,13 +175,15 @@ def _select_diverse(rows: list[sqlite3.Row], limit: int) -> list[sqlite3.Row]:
             selected.append(row)
             selected_ids.add(row["id"])
 
-    if len(seen_categories) < 2 and len(rows) > limit:
+    # Pass 3: ensure >= 2 categories when the pool allows it.
+    categories = {row["default_category"] for row in selected}
+    if len(categories) < 2:
         selected_ids = {row["id"] for row in selected}
         for row in rows:
             if row["id"] in selected_ids:
                 continue
-            if row["default_category"] not in seen_categories:
-                selected[-1] = row
+            if row["default_category"] not in categories:
+                selected[-1] = row  # evict the lowest-ranked selected item
                 break
 
     return selected

goodnews/cli.py

@@ -144,20 +144,24 @@ def main() -> None:
     elif args.command == "classify":
         init_db(conn)
         client = llm_client_from_args(args)
-        results = classify_articles(
+        report = classify_articles(
             conn,
             client,
             limit=args.limit,
             include_rejected=args.include_rejected,
             dry_run=args.dry_run,
         )
-        for article_id, scores in results:
+        for article_id, scores in report.results:
             accepted = "yes" if scores["accepted"] else "no"
             print(
                 f"[{article_id}] accepted={accepted} {scores['topic']}/{scores['flavor']} "
                 f"reason={scores['reason_code']}"
             )
             print(f"  {scores['reason_text']}")
+        print(
+            f"classify: attempted={report.attempted} succeeded={report.succeeded} "
+            f"skipped={report.skipped}"
+        )
         if args.dry_run:
             print("Dry run only; database was not updated.")
     elif args.command == "cycle":
@@ -294,7 +298,7 @@ def _run_cycle_locked(conn: sqlite3.Connection, args: argparse.Namespace) -> Non
             print(f"  classify {done}/{total} (article {article_id})", flush=True)
 
         try:
-            results = classify_articles(
+            report = classify_articles(
                 conn,
                 client,
                 limit=args.classify_limit,
@@ -302,7 +306,11 @@ def _run_cycle_locked(conn: sqlite3.Connection, args: argparse.Namespace) -> Non
                 only_unclassified=True,
                 progress=_progress,
             )
-            print(f"classify: {len(results)} new article(s) scored by {client.model}", flush=True)
+            print(
+                f"classify: attempted={report.attempted} succeeded={report.succeeded} "
+                f"skipped={report.skipped} (model {client.model})",
+                flush=True,
+            )
         except Exception as exc:  # endpoint down, timeout, etc. — keep going
             print(f"classify: skipped ({exc})", flush=True)
 

goodnews/filters.py

@@ -0,0 +1,123 @@
+"""Calm Filters — the canonical preference model and pure matching engine.
+
+Everything (localStorage today, query params on the API, a user_preferences row
+later) speaks this one shape, so the surfaces never drift. The functions here are
+deliberately pure and side-effect-free so they are easy to test and reuse from
+both the API and the CLI.
+
+The humane surface ("Not today" / "Less like this" / "Always hide this") maps onto
+this machinery: a pause is a topic/flavor muted *until* a timestamp; a mute is a
+standing exclusion; avoid-terms drop anything mentioning a phrase the reader would
+rather not see.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+
+# Split on any run of non-alphanumerics so matching is punctuation- and
+# case-insensitive, and anchored to whole words/phrases (no substring surprises:
+# "pan" must not match "pandemic", and "stock market" matches as a phrase).
+_NONWORD = re.compile(r"[^a-z0-9]+")
+
+
+def _normalize(text: str) -> str:
+    """Lowercase, collapse non-alphanumerics to single spaces, pad with spaces."""
+    return " " + _NONWORD.sub(" ", text.lower()).strip() + " "
+
+
+def text_matches_avoid_terms(text: str | None, terms: list[str]) -> bool:
+    """True if text contains any avoid term as a whole word or phrase."""
+    if not text or not terms:
+        return False
+    haystack = _normalize(text)
+    for term in terms:
+        needle = _normalize(term).strip()
+        if needle and f" {needle} " in haystack:
+            return True
+    return False
+
+
+@dataclass
+class Pause:
+    kind: str  # "topic" or "flavor"
+    value: str
+    until: str  # ISO 8601 UTC timestamp
+
+    def active(self, now: datetime) -> bool:
+        try:
+            until = datetime.fromisoformat(self.until.replace("Z", "+00:00"))
+        except (ValueError, AttributeError):
+            return False
+        return until > now
+
+
+@dataclass
+class FilterPrefs:
+    include_topics: list[str] = field(default_factory=list)
+    include_flavors: list[str] = field(default_factory=list)
+    mute_topics: list[str] = field(default_factory=list)
+    mute_flavors: list[str] = field(default_factory=list)
+    avoid_terms: list[str] = field(default_factory=list)
+    pauses: list[Pause] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict | None) -> "FilterPrefs":
+        data = data or {}
+        return cls(
+            include_topics=list(data.get("include_topics") or []),
+            include_flavors=list(data.get("include_flavors") or []),
+            mute_topics=list(data.get("mute_topics") or []),
+            mute_flavors=list(data.get("mute_flavors") or []),
+            avoid_terms=list(data.get("avoid_terms") or []),
+            pauses=[Pause(**p) for p in (data.get("pauses") or [])],
+        )
+
+    def muted_topics(self, now: datetime) -> set[str]:
+        """Standing mutes plus any topic currently paused."""
+        muted = set(self.mute_topics)
+        muted |= {p.value for p in self.pauses if p.kind == "topic" and p.active(now)}
+        return muted
+
+    def muted_flavors(self, now: datetime) -> set[str]:
+        muted = set(self.mute_flavors)
+        muted |= {p.value for p in self.pauses if p.kind == "flavor" and p.active(now)}
+        return muted
+
+    def is_empty(self) -> bool:
+        return not (
+            self.include_topics
+            or self.include_flavors
+            or self.mute_topics
+            or self.mute_flavors
+            or self.avoid_terms
+            or self.pauses
+        )
+
+
+def allows(article: dict, prefs: FilterPrefs, now: datetime) -> bool:
+    """True if an article (a feed/brief row dict) survives the preferences."""
+    topic = article.get("topic")
+    flavor = article.get("flavor")
+
+    if prefs.include_topics and topic not in prefs.include_topics:
+        return False
+    if prefs.include_flavors and flavor not in prefs.include_flavors:
+        return False
+    if topic in prefs.muted_topics(now):
+        return False
+    if flavor in prefs.muted_flavors(now):
+        return False
+    blob = f"{article.get('title') or ''} {article.get('description') or ''}"
+    if text_matches_avoid_terms(blob, prefs.avoid_terms):
+        return False
+    return True
+
+
+def filter_articles(articles: list[dict], prefs: FilterPrefs, now: datetime) -> list[dict]:
+    """Apply preferences to a list of article rows, preserving order."""
+    if prefs.is_empty():
+        return articles
+    return [a for a in articles if allows(a, prefs, now)]

goodnews/llm.py

@@ -220,6 +220,14 @@ class LocalModelClient:
         return parse_classifier_json(content)
 
 
+@dataclass
+class ClassifyReport:
+    results: list[tuple[int, dict]]
+    attempted: int
+    succeeded: int
+    skipped: int
+
+
 def classify_articles(
     conn: sqlite3.Connection,
     client: LocalModelClient,
@@ -228,17 +236,19 @@ def classify_articles(
     dry_run: bool = False,
     only_unclassified: bool = False,
     progress: "Callable[[int, int, int], None] | None" = None,
-) -> list[tuple[int, dict]]:
+) -> ClassifyReport:
     rows = _classification_candidates(
         conn, limit=limit, include_rejected=include_rejected, only_unclassified=only_unclassified
     )
     results = []
+    skipped = 0
     for index, row in enumerate(rows, start=1):
         try:
             scores = client.classify(row)
         except RuntimeError as exc:
             # One slow/failed article (timeout, bad response) shouldn't sink the
             # whole batch or discard work already committed. Skip and continue.
+            skipped += 1
             print(f"[{row['id']}] skipped: {exc}")
             continue
         scores = normalize_scores(scores, model_name=client.model)
@@ -248,7 +258,7 @@ def classify_articles(
             conn.commit()
         if progress is not None:
             progress(index, len(rows), row["id"])
-    return results
+    return ClassifyReport(results=results, attempted=len(rows), succeeded=len(results), skipped=skipped)
 
 
 def parse_classifier_json(content: str) -> dict:

goodnews/text.py

@@ -26,7 +26,8 @@ def clean_text(value: str | None, max_len: int = 1000) -> str | None:
     text = html.unescape(text)
     text = WHITESPACE_RE.sub(" ", text).strip()
     if len(text) > max_len:
-        return text[: max_len - 1].rstrip() + "..."
+        # Keep the ellipsis inside max_len rather than overshooting by 3.
+        return text[: max_len - 3].rstrip() + "..."
     return text or None