Add semantic cross-source dedup via local embeddings

- LocalModelClient.embed() calls the OpenAI-compatible /embeddings endpoint
  (local nomic model); base_url shared with chat, model via GOODNEWS_EMBED_MODEL.
- New article_embeddings table and articles.duplicate_of column (+ migration).
- dedup module: embeds missing articles, clusters near-identical stories within
  a date window by cosine similarity (pure-stdlib, vectors normalised once), and
  marks all but the highest-ranked member of each cluster as a duplicate.
- 'dedup' CLI command; cycle now runs poll -> classify -> dedup -> brief.
- Feed and brief queries hide duplicates, so a story carried by multiple
  outlets shows once.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

goodnews/cli.py

@@ -8,6 +8,7 @@ from pathlib import Path
 
 from .briefs import build_daily_brief, show_brief
 from .db import connect, init_db
+from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, dedup as run_dedup
 from .feeds import fetch_feed, parse_feed, poll_all_sources, poll_due_sources, poll_source
 from .llm import LocalModelClient, classify_articles
 from .scoring import score_article
@@ -68,11 +69,19 @@ def main() -> None:
     )
     cycle_parser.add_argument("--classify-limit", type=int, default=40)
     cycle_parser.add_argument("--no-classify", action="store_true", help="Skip the LLM classify step")
+    cycle_parser.add_argument("--no-dedup", action="store_true", help="Skip the embedding dedup step")
     cycle_parser.add_argument("--no-brief", action="store_true", help="Skip rebuilding today's brief")
     cycle_parser.add_argument("--force", action="store_true", help="Poll all active sources, ignoring intervals")
     cycle_parser.add_argument("--base-url", help="OpenAI-compatible base URL for classify")
     cycle_parser.add_argument("--model", help="Local model name for classify")
 
+    dedup_parser = subparsers.add_parser("dedup", help="Cluster near-duplicate stories via local embeddings")
+    dedup_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help="Cosine similarity cutoff")
+    dedup_parser.add_argument("--window-days", type=int, default=DEFAULT_WINDOW_DAYS)
+    dedup_parser.add_argument("--embed-limit", type=int, help="Cap how many missing embeddings to compute")
+    dedup_parser.add_argument("--base-url", help="OpenAI-compatible base URL")
+    dedup_parser.add_argument("--model", help="Chat model name (unused for embeddings)")
+
     check_llm_parser = subparsers.add_parser("check-llm", help="Check local OpenAI-compatible model endpoint")
     check_llm_parser.add_argument("--base-url", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
     check_llm_parser.add_argument("--model", help="Expected local model name")
@@ -153,6 +162,17 @@ def main() -> None:
             print("Dry run only; database was not updated.")
     elif args.command == "cycle":
         run_cycle(conn, args)
+    elif args.command == "dedup":
+        init_db(conn)
+        client = llm_client_from_args(args)
+        stats = run_dedup(
+            conn, client, threshold=args.threshold, window_days=args.window_days, embed_limit=args.embed_limit
+        )
+        print(
+            f"dedup: embedded={stats['embedded']} articles={stats['articles']} "
+            f"clusters={stats['clusters']} duplicate_clusters={stats['duplicate_clusters']} "
+            f"duplicates_hidden={stats['duplicates']}"
+        )
     elif args.command == "check-llm":
         client = llm_client_from_args(args)
         try:
@@ -256,6 +276,13 @@ def run_cycle(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
         except Exception as exc:  # endpoint down, timeout, etc. — keep going
             print(f"classify: skipped ({exc})")
 
+    if not args.no_dedup:
+        try:
+            stats = run_dedup(conn, llm_client_from_args(args))
+            print(f"dedup: embedded={stats['embedded']} duplicates_hidden={stats['duplicates']}")
+        except Exception as exc:
+            print(f"dedup: skipped ({exc})")
+
     if not args.no_brief:
         today = date.today().isoformat()
         try: