Sync repo to deployed state: SEO recovery, Publishing Desk, Play games, emoji picker

The deploy pipeline runs from the working tree, so a wave of shipped features
had never been committed. This snapshots git to what's actually running.

SEO impression recovery (live + verified):
- Duplicate /a/{id} now 301-redirect to their canonical twin instead of 404
  (a hard 404 silently dropped already-indexed URLs and tanked impressions).
- Dedup representative selection reworked: accepted/serveable -> established
  rep (URL stability) -> quality score, so an accepted page never retires to a
  rejected rep and an indexed canonical doesn't churn when a newer twin arrives.
- HEAD /a/{id} returns the same status as GET (api_route GET+HEAD) instead of
  falling through to the static mount and 404ing.
- `dedup --force-recluster`: cycle-locked, model-free re-cluster to re-apply the
  policy to the existing corpus (shared cycle_lock context manager).
- CLI honors GOODNEWS_DB for its default --db (was silently ignored).

Publishing Desk (admin tool to post highlights to X via Web Intents):
- publishing.py queue/rank/handle-resolution; admin UI; full searchable emoji
  picker (bundled data, no CDN) for the blurb editor.

Play games + site:
- Bloom (word-wheel), Memory Match, daily ritual set, Zen Den (dev-gated).
- English-only language gate; source prospecting; paywall + dedup hardening.

Tests: full suite green (349). Ignores tightened (node_modules, data/*.db).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-18 11:32:27 -04:00
parent 2dbe73430c
commit 89c0fbe1f6
66 changed files with 6138 additions and 109 deletions
+352 -17
View File
@@ -18,10 +18,13 @@ import hashlib
import hmac
import io
import json
import logging
import os
import re
import secrets
import sqlite3
import threading
import time
from collections import Counter
from contextlib import contextmanager
from datetime import datetime, timezone
@@ -33,7 +36,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from . import auth, email_send, feeds, games, oauth_google, queries, share, sources, summarize
from . import auth, bloom, email_send, feeds, games, oauth_google, publishing, queries, share, sources, summarize
from .localtime import local_today
from .markup import reply_html_to_text, sanitize_reply_html
from .db import connect
@@ -55,6 +58,8 @@ _EDGE_DERIVED = "public, max-age=0, s-maxage=120, stale-while-revalidate=120"
_EDGE_FEED = "public, max-age=0, s-maxage=45, stale-while-revalidate=30" # global feed (URL-keyed, shareable only)
_PRIVATE = "private, no-store" # never share across users
log = logging.getLogger("goodnews.api")
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_DB = ROOT / "data" / "goodnews.sqlite3"
# Prefer the built SvelteKit site; fall back to the legacy single-page harness.
@@ -147,6 +152,32 @@ def _user_out(user: sqlite3.Row) -> dict:
# scrapers don't each kick off a duplicate LLM call.
_summarizing: set[int] = set()
# In-process cache of fully-rendered /a/{id} share pages. We're direct-origin (no
# CDN), so Cache-Control alone can't shield the box from crawler bursts hitting the
# sitemap's article URLs while the LAN LLM / cycle is loading it. Only COMPLETE
# pages (summary + explanation present) are cached, so a "still generating" page is
# never pinned; a short TTL still picks up edits. Per-process (fine across workers).
# INVARIANT: the share page is PUBLIC/anonymous — the cache key is article_id alone.
# If /a/{id} ever personalizes (per-viewer content), key by viewer or drop the cache,
# or one visitor's variant would be served to another.
_SHARE_CACHE: dict[int, tuple[float, str]] = {}
_SHARE_TTL = 900.0 # 15 min
_SHARE_CACHE_MAX = 512
def _share_cache_get(aid: int) -> str | None:
hit = _SHARE_CACHE.get(aid)
if hit and (time.monotonic() - hit[0]) < _SHARE_TTL:
return hit[1]
return None
def _share_cache_put(aid: int, html: str) -> None:
if len(_SHARE_CACHE) >= _SHARE_CACHE_MAX:
oldest = min(_SHARE_CACHE, key=lambda k: _SHARE_CACHE[k][0])
_SHARE_CACHE.pop(oldest, None)
_SHARE_CACHE[aid] = (time.monotonic(), html)
def _run_summary(article_id: int) -> None:
try:
@@ -158,6 +189,29 @@ def _run_summary(article_id: int) -> None:
_summarizing.discard(article_id)
# Publishing Desk: the "Build queue" job runs in the background (one bounded
# comparative LLM call can be slow); the admin polls the queue endpoint. Mirrors the
# summary-kick pattern — never holds an HTTP request open on the model. The lock makes
# the check-and-set atomic so two rapid clicks can't launch two expensive jobs.
_publish_build: dict = {"building": False, "result": None, "error": None}
_publish_build_lock = threading.Lock()
def _run_publish_build() -> None:
try:
try:
client = LocalModelClient.from_env()
except Exception: # noqa: BLE001 — model down → deterministic fallback inside build_queue
client = None
with get_conn() as conn:
res = publishing.build_queue(conn, PUBLIC_BASE_URL, client=client)
_publish_build.update(result=res, error=None)
except Exception as exc: # noqa: BLE001 — surface, don't crash the worker
_publish_build.update(error=str(exc)[:300])
finally:
_publish_build["building"] = False
def _kick_summary(article_id: int, background_tasks: BackgroundTasks) -> None:
if article_id in _summarizing:
return
@@ -332,7 +386,7 @@ class SourcePreview(BaseModel):
sampled: int
classified: bool
accepted: int
acceptance_rate: float
acceptance_rate: float | None # None when there are no English items to judge (all held)
avg_cortisol: float
avg_ragebait: float
avg_pr_risk: float
@@ -357,6 +411,54 @@ class GameStateBody(BaseModel):
state: dict = {}
class PublishStatusBody(BaseModel):
status: str
draft_text: str | None = None
final_text: str | None = None
post_url: str | None = None
snooze_until: str | None = None
class PublishDraftBody(BaseModel):
draft_text: str = ""
class EntityHandleBody(BaseModel):
entity_name: str
handle: str
profile_url: str | None = None
class GameStateItem(BaseModel):
game: str
variant: str
state: dict = {}
class GameStateBatchBody(BaseModel):
date: str
items: list[GameStateItem] = []
class BloomReportBody(BaseModel):
word: str = ""
date: str | None = None
mode: str | None = None
format: str | None = None
letters: str | None = None
reason: str | None = None
class BloomOverrideBody(BaseModel):
word: str = ""
action: str = "allow" # 'allow' | 'block'
reason: str | None = None
class BloomReportActionBody(BaseModel):
action: str = "" # 'approve' | 'block' | 'dismiss'
class WordPoolBody(BaseModel):
word: str
@@ -495,6 +597,13 @@ _EVENT_KINDS = {
}
def _fts_query(q: str) -> str:
"""Raw search box → safe FTS5 query: alnum terms only (no operator/quote
injection), each prefix-matched and AND'd together. '' when nothing usable."""
terms = re.findall(r"[A-Za-z0-9]+", q or "")[:8]
return " ".join(f"{t}*" for t in terms)
def _visitor_hash(token: str | None) -> str:
token = (token or "").strip()[:200]
if not token:
@@ -660,22 +769,38 @@ def create_app() -> FastAPI:
state: str | None = None,
error: str | None = None,
) -> RedirectResponse:
fail = RedirectResponse(f"{PUBLIC_BASE_URL}/auth/verify?error=google", status_code=302)
if error or not code or not state:
return fail
# The user always sees the same generic error=google (no detail leaked),
# but we log WHY internally so device/host-specific failures (e.g. a www
# vs apex cookie loss, a state mismatch, a token-exchange error) are
# diagnosable instead of all looking identical.
def fail(reason: str, exc: Exception | None = None) -> RedirectResponse:
host = request.headers.get("Host", "?")
if exc is not None:
log.warning("google callback failed: %s (host=%s): %s", reason, host, exc)
else:
log.warning("google callback failed: %s (host=%s)", reason, host)
return RedirectResponse(f"{PUBLIC_BASE_URL}/auth/verify?error=google", status_code=302)
if error:
return fail(f"provider_error:{error}")
if not code or not state:
return fail("missing_code_or_state")
saved = _unsign(request.cookies.get(OAUTH_COOKIE))
if not saved:
return fail
# Most likely the host-only ub_oauth cookie was set on a different
# host than this callback (www vs apex). Canonicalizing www→apex at
# the edge prevents this.
return fail("missing_oauth_cookie")
saved_state, _, verifier = saved.partition(":")
if not hmac.compare_digest(saved_state, state):
return fail
return fail("state_mismatch")
try:
tokens = oauth_google.exchange_code(code, _google_redirect_uri(), verifier)
info = oauth_google.verify_id_token(tokens["id_token"])
if not info.get("picture") and tokens.get("access_token"):
info["picture"] = oauth_google.fetch_userinfo(tokens["access_token"]).get("picture")
except Exception:
return fail
except Exception as exc: # noqa: BLE001 — log reason, show generic error
return fail("token_exchange_or_verify", exc)
with get_conn() as conn:
user_id = auth.find_or_create_user(
conn, info["email"], "google", info["sub"],
@@ -925,13 +1050,19 @@ def create_app() -> FastAPI:
# --- Public share/landing page for an article -------------------------
@app.get("/a/{article_id}", response_class=HTMLResponse)
# GET + HEAD: FastAPI's @app.get registers GET only (no auto-HEAD), so a HEAD would
# fall through to the catch-all StaticFiles mount at "/" and 404. Register both so
# HEAD returns the same status (200/301/404) as GET, sans body.
@app.api_route("/a/{article_id}", methods=["GET", "HEAD"], response_class=HTMLResponse)
def share_page(article_id: str, background_tasks: BackgroundTasks) -> HTMLResponse:
not_found = HTMLResponse(share.render_not_found(PUBLIC_BASE_URL), status_code=404)
try:
aid = int(article_id)
except (TypeError, ValueError):
return not_found # malformed id → calm 404, no stack trace
cached = _share_cache_get(aid)
if cached is not None: # serve a rendered page without touching SQLite/render
return HTMLResponse(cached, headers={"Cache-Control": "public, max-age=300"})
with get_conn() as conn:
row = conn.execute(
"SELECT a.id, a.title, a.description, a.image_url, a.canonical_url, "
@@ -941,16 +1072,45 @@ def create_app() -> FastAPI:
"LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
(aid,),
).fetchone()
# Only render real, accepted, non-duplicate stories.
if not row or row["duplicate_of"] is not None or not row["accepted"]:
if not row:
return not_found
# A duplicate's URL may already be indexed by Google. A hard 404 silently
# drops it (and any newer twin that arrives later retires the OLDER, already
# indexed URL) — that's what tanked impressions. So 301 to the canonical twin
# instead: Google consolidates the page onto the survivor. dedup stores a star
# (dup -> rep, rep.duplicate_of IS NULL); we still follow a short chain with a
# cycle guard as cheap insurance.
if row["duplicate_of"] is not None:
seen, cur, target = {aid}, row["duplicate_of"], None
for _ in range(8):
if cur in seen:
break
seen.add(cur)
r2 = conn.execute(
"SELECT a.id, a.duplicate_of, s.accepted FROM articles a "
"LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
(cur,),
).fetchone()
if not r2:
break
if r2["duplicate_of"] is None:
target = r2 if r2["accepted"] else None
break
cur = r2["duplicate_of"]
if target is not None:
return RedirectResponse(f"/a/{target['id']}", status_code=301)
return not_found # canonical itself is gone/rejected → genuinely 404
if not row["accepted"]:
return not_found
summary = summarize.get_summary(conn, aid)
explanation = summarize.get_explanation(conn, aid)
if not summary or not explanation:
complete = bool(summary and explanation)
if not complete:
_kick_summary(aid, background_tasks) # generate/top-up for next time; page polls
return HTMLResponse(
share.render_share_page(dict(row), PUBLIC_BASE_URL, summary=summary, explanation=explanation)
)
html = share.render_share_page(dict(row), PUBLIC_BASE_URL, summary=summary, explanation=explanation)
if complete:
_share_cache_put(aid, html) # cache only the finished page (never the "generating" state)
return HTMLResponse(html, headers={"Cache-Control": "public, max-age=300" if complete else "no-cache"})
# --- Privacy-respecting first-party analytics -------------------------
@@ -1305,6 +1465,76 @@ def create_app() -> FastAPI:
cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone()
return _candidate_dict(cand)
@app.post("/api/admin/candidates/{cid}/restore")
def admin_candidate_restore(cid: int, request: Request) -> dict:
# Send a rejected candidate back to staging for another look.
with get_conn() as conn:
_require_admin(conn, request)
if not sources.restore_candidate(conn, cid):
raise HTTPException(status_code=404, detail="no rejected candidate with that id")
cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone()
return _candidate_dict(cand)
# --- Publishing Desk (admin): outbound-share queue for X (platform-neutral) ---
@app.post("/api/admin/publishing/build")
def admin_publishing_build(request: Request, background_tasks: BackgroundTasks) -> dict:
# Kick the queue build in the background (the comparative LLM call can be slow);
# the client polls /queue. No-op if a build is already running.
with get_conn() as conn:
_require_admin(conn, request)
with _publish_build_lock: # atomic check-and-set: one job at a time
if not _publish_build["building"]:
_publish_build.update(building=True, result=None, error=None)
background_tasks.add_task(_run_publish_build)
return {"building": True}
@app.get("/api/admin/publishing/queue")
def admin_publishing_queue(request: Request, archived: bool = False) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
items = publishing.list_queue(conn, include_archived=archived)
return {"building": _publish_build["building"], "last": _publish_build.get("result"),
"error": _publish_build.get("error"), "items": items}
@app.post("/api/admin/publishing/{sid}/status")
def admin_publishing_status(sid: int, body: PublishStatusBody, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
ok = publishing.set_status(conn, sid, body.status, draft_text=body.draft_text,
final_text=body.final_text, post_url=body.post_url,
snooze_until=body.snooze_until)
if not ok:
raise HTTPException(status_code=400, detail="bad status or id")
return {"ok": True}
@app.post("/api/admin/publishing/{sid}/draft")
def admin_publishing_draft(sid: int, body: PublishDraftBody, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
ok = publishing.save_draft(conn, sid, body.draft_text)
if not ok:
raise HTTPException(status_code=404, detail="no such share")
return {"ok": True}
@app.post("/api/admin/publishing/{sid}/restore")
def admin_publishing_restore(sid: int, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
ok = publishing.restore(conn, sid)
if not ok:
raise HTTPException(status_code=400, detail="not a restorable (skipped/snoozed) share")
return {"ok": True}
@app.post("/api/admin/publishing/handles")
def admin_publishing_add_handle(body: EntityHandleBody, request: Request) -> dict:
# Save a verified handle (e.g. after confirming one via 'Find on X').
with get_conn() as conn:
_require_admin(conn, request)
ok = publishing.add_entity_handle(conn, body.entity_name, body.handle, body.profile_url)
if not ok:
raise HTTPException(status_code=400, detail="bad entity or handle")
return {"ok": True}
# --- CSV exports (admin-gated, for inspection / archiving) ---------------
def _csv_cell(v):
@@ -1593,6 +1823,32 @@ def create_app() -> FastAPI:
items=[Article.from_row(r) for r in rows],
)
@app.get("/api/search", response_model=FeedResponse)
def search(response: Response, q: str = Query("", max_length=120),
prefs: str | None = Query(None), limit: int = Query(30, ge=1, le=60),
offset: int = Query(0, ge=0)) -> FeedResponse:
# Public article search across the visitor-facing corpus. Mirrors the feed's
# boundaries (accepted/visible/non-duplicate + the reader's Calm Filters /
# avoid-terms) but NOT a lane scope — you searched on purpose. Ranked by
# relevance (bm25), recency as a tie-break. Per-reader → never edge-cached.
response.headers["Cache-Control"] = _PRIVATE
fts = _fts_query(q)
if not fts:
return FeedResponse(topic=None, flavor=None, count=0, items=[])
fp = prefs_from_json(prefs)
now = datetime.now(timezone.utc)
kw = _prefs_sql_kw(fp, now)
with get_conn() as conn:
if not conn.execute("SELECT 1 FROM article_search LIMIT 1").fetchone():
queries.reindex_search(conn) # lazy build (fresh deploy / before first cycle)
fetch_n = min(2000, (offset + limit) * 4 + 40) if fp.avoid_terms else (offset + limit)
raw = queries.feed(conn, accepted_only=True, limit=fetch_n, offset=0, match=fts, **kw)
kept = filter_articles(raw, fp, now) if fp.avoid_terms else raw # word-boundary avoid-terms
items = kept[offset:offset + limit]
# Keep relevance order (don't paywall-reorder); the badge still shows true status.
return FeedResponse(topic=None, flavor=None, count=len(items),
items=[Article.from_row(r) for r in items])
@app.get("/api/puzzle/{game}")
def daily_puzzle(game: str, variant: str = Query("5")) -> dict:
with get_conn() as conn:
@@ -1600,8 +1856,29 @@ def create_app() -> FastAPI:
return games.word_puzzle_response(conn, local_today(), variant)
if game == "wordsearch":
return games.wordsearch_response(conn, local_today(), variant)
if game == "bloom":
return bloom.bloom_response(conn, local_today())
raise HTTPException(status_code=404, detail="no such puzzle")
@app.get("/api/puzzle/bloom/free")
def bloom_free(response: Response, format: str = "center", seed: str | None = None) -> dict:
# A free-play wheel: deterministic by `seed` (client stores it to resume),
# random when none is given. Center Circle or Wild Bloom. No DB, no sync.
fmt = "wild" if format == "wild" else "center"
s = seed if (seed and re.fullmatch(r"[A-Za-z0-9_-]{1,32}", seed)) else secrets.token_urlsafe(6)
response.headers["Cache-Control"] = "no-store"
with get_conn() as conn:
return bloom.bloom_free_response(conn, s, fmt)
@app.post("/api/bloom/report")
def bloom_report(body: BloomReportBody) -> dict:
# A player flagging a rejected word as "should count". Public + deduped;
# lands in the admin queue (approve→allow / block / dismiss).
with get_conn() as conn:
ok = bloom.add_report(conn, body.word, body.date, body.mode, body.format,
body.letters, body.reason)
return {"ok": bool(ok)}
@app.post("/api/puzzle/word/guess")
def word_guess(body: WordGuessRequest) -> dict:
if body.variant not in games.WORD_VARIANTS:
@@ -1615,7 +1892,9 @@ def create_app() -> FastAPI:
# --- Cross-device game state sync (signed-in only; merged server-side) ---
def _game_ok(game: str, variant: str) -> bool:
return (game == "word" and variant in games.WORD_VARIANTS) or \
(game == "wordsearch" and variant in games.WS_TIERS)
(game == "wordsearch" and variant in games.WS_TIERS) or \
(game == "bloom" and variant == "") or \
(game == "match" and variant in games.MATCH_VARIANTS) # "<tier>-<format>"
def _valid_pdate(d: str) -> bool:
return bool(re.match(r"^\d{4}-\d{2}-\d{2}$", d or "")) # plain YYYY-MM-DD, no junk rows
@@ -1647,6 +1926,27 @@ def create_app() -> FastAPI:
merged = games.save_game_state(conn, user["id"], body.game, body.variant, body.date, body.state or {})
return {"state": merged}
@app.put("/api/games/state/batch")
def game_state_put_batch(body: GameStateBatchBody, request: Request) -> dict:
"""Reconcile many (game, variant) states for one date in a SINGLE request, so
the hub doesn't fan out a dozen calls on every /play load. Each item is
validated/sanitized/merged exactly like the single PUT; unknown or oversized
items are dropped (not fatal). Signed-out → echo (no sync), same as the single
endpoint, so cross-device pull is preserved for signed-in users."""
if not _valid_pdate(body.date):
raise HTTPException(status_code=400, detail="bad date")
items = [it for it in body.items[:32]
if _game_ok(it.game, it.variant) and len(json.dumps(it.state)) <= 20000]
with get_conn() as conn:
user = _current_user(conn, request)
if not user:
return {"states": [{"game": it.game, "variant": it.variant, "state": it.state} for it in items]}
out = []
for it in items:
merged = games.save_game_state(conn, user["id"], it.game, it.variant, body.date, it.state or {})
out.append({"game": it.game, "variant": it.variant, "state": merged})
return {"states": out}
@app.get("/api/games/stats")
def game_stats_get(game: str, variant: str, request: Request) -> dict:
if not _game_ok(game, variant):
@@ -1656,6 +1956,41 @@ def create_app() -> FastAPI:
return {"stats": games.game_stats(conn, user["id"], game, variant) if user else None}
# --- Admin: Daily Word pool curation ---
# --- Admin: Bloom word curation (runtime, no deploy) ---
@app.get("/api/admin/bloom/reports")
def admin_bloom_reports(request: Request, status: str = "pending") -> dict:
with get_conn() as conn:
_require_admin(conn, request)
st = status if status in ("pending", "approved", "blocked", "dismissed") else "pending"
return {"status": st, "reports": bloom.list_reports(conn, st),
"overrides": bloom.list_overrides(conn)}
@app.post("/api/admin/bloom/reports/{report_id}")
def admin_bloom_resolve(report_id: int, body: BloomReportActionBody, request: Request) -> dict:
with get_conn() as conn:
admin = _require_admin(conn, request)
ok = bloom.resolve_report(conn, report_id, body.action, by=admin["email"])
if not ok:
raise HTTPException(status_code=400, detail="bad report or action")
return {"ok": True}
@app.post("/api/admin/bloom/overrides")
def admin_bloom_override(body: BloomOverrideBody, request: Request) -> dict:
with get_conn() as conn:
admin = _require_admin(conn, request)
ok = bloom.set_override(conn, body.word, body.action, reason=body.reason, by=admin["email"])
if not ok:
raise HTTPException(status_code=422,
detail="allow needs a real ≥4-letter word with no 'S'; block accepts any word")
return {"ok": True}
@app.delete("/api/admin/bloom/overrides/{word}")
def admin_bloom_override_clear(word: str, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
bloom.clear_override(conn, word)
return {"ok": True}
@app.get("/api/admin/word/lookup")
def admin_word_lookup(word: str, request: Request) -> dict:
with get_conn() as conn: