Sync repo to deployed state: SEO recovery, Publishing Desk, Play games, emoji picker

The deploy pipeline runs from the working tree, so a wave of shipped features had never been committed. This snapshots git to what's actually running. SEO impression recovery (live + verified): - Duplicate /a/{id} now 301-redirect to their canonical twin instead of 404 (a hard 404 silently dropped already-indexed URLs and tanked impressions). - Dedup representative selection reworked: accepted/serveable -> established rep (URL stability) -> quality score, so an accepted page never retires to a rejected rep and an indexed canonical doesn't churn when a newer twin arrives. - HEAD /a/{id} returns the same status as GET (api_route GET+HEAD) instead of falling through to the static mount and 404ing. - `dedup --force-recluster`: cycle-locked, model-free re-cluster to re-apply the policy to the existing corpus (shared cycle_lock context manager). - CLI honors GOODNEWS_DB for its default --db (was silently ignored). Publishing Desk (admin tool to post highlights to X via Web Intents): - publishing.py queue/rank/handle-resolution; admin UI; full searchable emoji picker (bundled data, no CDN) for the blurb editor. Play games + site: - Bloom (word-wheel), Memory Match, daily ritual set, Zen Den (dev-gated). - English-only language gate; source prospecting; paywall + dedup hardening. Tests: full suite green (349). Ignores tightened (node_modules, data/*.db). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-18 11:32:27 -04:00
parent 2dbe73430c
commit 89c0fbe1f6
66 changed files with 6138 additions and 109 deletions
@@ -1,6 +1,7 @@
 from __future__ import annotations

 import argparse
+import contextlib
 import os
 import sqlite3
 from pathlib import Path
@@ -10,7 +11,7 @@ from .db import connect, init_db
 from .digest import send_due_digests
 from .games import generate_daily_puzzles
 from .localtime import local_today
-from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, dedup as run_dedup
+from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, cluster_duplicates, dedup as run_dedup
 from .enrich import enrich_brief_images, enrich_recent_images, enrich_summarized_images
 from .summarize import generate_summary, get_summary
 from .feeds import (
@@ -39,9 +40,17 @@ DEFAULT_DB = ROOT / "data" / "goodnews.sqlite3"
 DEFAULT_SOURCES = ROOT / "config" / "sources.toml"


+def _default_db() -> Path:
+    # Honor GOODNEWS_DB like the rest of the app (db.connect) does, so `GOODNEWS_DB=… `
+    # actually targets that DB instead of being silently ignored — otherwise a copy-DB
+    # maintenance run (e.g. dedup --force-recluster) can land on production by surprise.
+    return Path(os.environ.get("GOODNEWS_DB") or DEFAULT_DB)
+
+
 def main() -> None:
    parser = argparse.ArgumentParser(prog="goodnews")
-    parser.add_argument("--db", type=Path, default=DEFAULT_DB, help="SQLite database path")
+    parser.add_argument("--db", type=Path, default=_default_db(),
+                        help="SQLite database path (defaults to $GOODNEWS_DB, else the bundled data/ DB)")
    subparsers = parser.add_subparsers(dest="command", required=True)

    subparsers.add_parser("init-db", help="Create or update the SQLite schema")
@@ -144,6 +153,9 @@ def main() -> None:
    dedup_parser.add_argument("--embed-limit", type=int, help="Cap how many missing embeddings to compute")
    dedup_parser.add_argument("--base-url", help="OpenAI-compatible base URL")
    dedup_parser.add_argument("--model", help="Chat model name (unused for embeddings)")
+    dedup_parser.add_argument("--force-recluster", action="store_true",
+                              help="Re-cluster the EXISTING corpus even if no new embeddings "
+                                   "(re-applies representative policy; cycle-locked, no model needed)")

    check_llm_parser = subparsers.add_parser("check-llm", help="Check local OpenAI-compatible model endpoint")
    check_llm_parser.add_argument("--base-url", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
@@ -221,7 +233,9 @@ def main() -> None:
                import json as _json

                p = _json.loads(r["preview_json"])
-                line += f"  (accept {round(p.get('acceptance_rate', 0) * 100)}%, sampled {p.get('sampled', 0)})"
+                _rate = p.get("acceptance_rate")
+                _rate_str = f"{round(_rate * 100)}%" if _rate is not None else "—"
+                line += f"  (accept {_rate_str}, sampled {p.get('sampled', 0)})"
            print(line)
    elif args.command == "promote-candidate":
        init_db(conn)
@@ -286,15 +300,31 @@ def main() -> None:
        print(f"enrich-images: {found} new image(s) for summarized articles")
    elif args.command == "dedup":
        init_db(conn)
-        client = llm_client_from_args(args)
-        stats = run_dedup(
-            conn, client, threshold=args.threshold, window_days=args.window_days, embed_limit=args.embed_limit
-        )
-        print(
-            f"dedup: embedded={stats['embedded']} articles={stats['articles']} "
-            f"clusters={stats['clusters']} duplicate_clusters={stats['duplicate_clusters']} "
-            f"duplicates_hidden={stats['duplicates']}"
-        )
+        if args.force_recluster:
+            # Re-apply representative policy to the EXISTING corpus. The normal path
+            # fast-skips when no new embeddings exist, so it would NOT pick up a policy
+            # change. Cycle-locked so it can't overlap the scheduled timer; no model
+            # needed (pure re-cluster over stored embeddings).
+            with cycle_lock(args.db) as acquired:
+                if not acquired:
+                    print("dedup: a cycle is already running; re-run --force-recluster after it finishes")
+                    return
+                stats = cluster_duplicates(conn, threshold=args.threshold, window_days=args.window_days)
+            print(
+                f"dedup (forced recluster): articles={stats['articles']} "
+                f"clusters={stats['clusters']} duplicate_clusters={stats['duplicate_clusters']} "
+                f"duplicates_hidden={stats['duplicates']}"
+            )
+        else:
+            client = llm_client_from_args(args)
+            stats = run_dedup(
+                conn, client, threshold=args.threshold, window_days=args.window_days, embed_limit=args.embed_limit
+            )
+            print(
+                f"dedup: embedded={stats['embedded']} articles={stats['articles']} "
+                f"clusters={stats['clusters']} duplicate_clusters={stats['duplicate_clusters']} "
+                f"duplicates_hidden={stats['duplicates']}"
+            )
    elif args.command == "check-llm":
        client = llm_client_from_args(args)
        try:
@@ -368,7 +398,9 @@ def list_recent(conn: sqlite3.Connection, limit: int, accepted_only: bool) -> No
 def print_preview(p: dict) -> None:
    mode = "model" if p["classified"] else "heuristic"
    print(f"Preview of {p['url']}  ({mode})")
-    print(f"  sampled={p['sampled']} accepted={p['accepted']} ({p['acceptance_rate']*100:.0f}%)")
+    rate = p.get("acceptance_rate")
+    rate_str = f"{rate * 100:.0f}%" if rate is not None else "— (all held)"
+    print(f"  sampled={p['sampled']} accepted={p['accepted']} ({rate_str})")
    print(f"  freshness: newest={p['newest_published'] or 'unknown'} in_last_7d={p['recent_7d']}")
    print(f"  averages: cortisol={p['avg_cortisol']} ragebait={p['avg_ragebait']} pr_risk={p['avg_pr_risk']}")
    if p["topic_mix"]:
@@ -398,6 +430,28 @@ def check_feeds(conn: sqlite3.Connection, include_inactive: bool = False) -> Non
    print(f"--- {ok}/{len(rows)} feeds healthy ---")


+@contextlib.contextmanager
+def cycle_lock(db_path):
+    """Exclusive, non-blocking lock shared by the scheduled cycle and any manual job
+    that mutates the corpus (e.g. a forced dedup re-cluster), so they can never overlap
+    and contend on the database/model. Yields True if acquired, False if already held."""
+    import fcntl
+
+    lock_path = Path(db_path).parent / ".goodnews-cycle.lock"
+    lock_file = open(lock_path, "w")
+    try:
+        fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        lock_file.close()
+        yield False
+        return
+    try:
+        yield True
+    finally:
+        fcntl.flock(lock_file, fcntl.LOCK_UN)
+        lock_file.close()
+
+
 def run_cycle(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
    """One end-to-end pass for a scheduler: poll due sources, classify the new
    arrivals, dedup, rebuild today's brief. Each step is independent and
@@ -406,21 +460,11 @@ def run_cycle(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
    Holds an exclusive lock so a manual run and the systemd timer (or two timer
    ticks) can never overlap and contend on the database and model.
    """
-    import fcntl
-
-    lock_path = Path(args.db).parent / ".goodnews-cycle.lock"
-    lock_file = open(lock_path, "w")
-    try:
-        fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except OSError:
-        print("cycle: another cycle is already running; skipping")
-        lock_file.close()
-        return
-    try:
+    with cycle_lock(args.db) as acquired:
+        if not acquired:
+            print("cycle: another cycle is already running; skipping")
+            return
        _run_cycle_locked(conn, args)
-    finally:
-        fcntl.flock(lock_file, fcntl.LOCK_UN)
-        lock_file.close()


 def _run_cycle_locked(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
@@ -505,6 +549,12 @@ def _run_cycle_locked(conn: sqlite3.Connection, args: argparse.Namespace) -> Non
        except Exception as exc:
            print(f"review: skipped ({exc})")

+    try:
+        from .queries import reindex_search
+        print(f"search: indexed {reindex_search(conn)} articles")
+    except Exception as exc:  # noqa: BLE001 — search index is non-critical
+        print(f"search: skipped ({exc})")
+
    if not args.no_digest:
        try:
            sent = send_due_digests(conn)  # morning-gated + deduped internally