thejayman77/upbeatBytes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync repo to deployed state: SEO recovery, Publishing Desk, Play games, emoji picker

Browse Source 
The deploy pipeline runs from the working tree, so a wave of shipped features
had never been committed. This snapshots git to what's actually running.

SEO impression recovery (live + verified):
- Duplicate /a/{id} now 301-redirect to their canonical twin instead of 404
  (a hard 404 silently dropped already-indexed URLs and tanked impressions).
- Dedup representative selection reworked: accepted/serveable -> established
  rep (URL stability) -> quality score, so an accepted page never retires to a
  rejected rep and an indexed canonical doesn't churn when a newer twin arrives.
- HEAD /a/{id} returns the same status as GET (api_route GET+HEAD) instead of
  falling through to the static mount and 404ing.
- `dedup --force-recluster`: cycle-locked, model-free re-cluster to re-apply the
  policy to the existing corpus (shared cycle_lock context manager).
- CLI honors GOODNEWS_DB for its default --db (was silently ignored).

Publishing Desk (admin tool to post highlights to X via Web Intents):
- publishing.py queue/rank/handle-resolution; admin UI; full searchable emoji
  picker (bundled data, no CDN) for the blurb editor.

Play games + site:
- Bloom (word-wheel), Memory Match, daily ritual set, Zen Den (dev-gated).
- English-only language gate; source prospecting; paywall + dedup hardening.

Tests: full suite green (349). Ignores tightened (node_modules, data/*.db).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-18 11:32:27 -04:00
parent 2dbe73430c
commit 89c0fbe1f6
66 changed files with 6138 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
goodnews/queries.py
+62 -9
View File
@@ -11,6 +11,7 @@ import sqlite3
from datetime import UTC, datetime, timedelta
from .feeds import MAX_BACKOFF_MINUTES
from .localtime import local_now
from .paywall import is_paywalled, is_paywalled_for_source
# UA substrings that mark automated clients. Crawlers run JS on a throttled
@@ -78,6 +79,7 @@ def feed(
follow_sources: list[int] | None = None,
follow_tags: list[str] | None = None,
since: str | None = None,
match: str | None = None,
) -> list[dict]:
"""Return articles with categorical filters applied in SQL.
@@ -92,6 +94,14 @@ def feed(
"""
clauses = ["a.duplicate_of IS NULL", "src.content_visible = 1"]
params: list = []
# Full-text search: join the FTS index and MATCH first, so its bound param
# leads and relevance can drive the ordering. All the boundary clauses below
# still apply, so search mirrors exactly what the visitor feed would show.
fts_join = ""
if match:
fts_join = "JOIN article_search ON article_search.article_id = a.id"
clauses.append("article_search MATCH ?")
params.append(match)
if accepted_only:
clauses.append("s.accepted = 1")
if topic:
@@ -155,17 +165,19 @@ def feed(
where = "WHERE " + " AND ".join(clauses)
params.extend([limit, offset])
order_by = (
"COALESCE(a.published_at, a.discovered_at) DESC, rank_score DESC"
if sort == "latest"
else "rank_score DESC, COALESCE(a.published_at, a.discovered_at) DESC"
)
if match:
order_by = "bm25(article_search), COALESCE(a.published_at, a.discovered_at) DESC" # relevance, then recency
elif sort == "latest":
order_by = "COALESCE(a.published_at, a.discovered_at) DESC, rank_score DESC"
else:
order_by = "rank_score DESC, COALESCE(a.published_at, a.discovered_at) DESC"
rows = conn.execute(
f"""
SELECT {_ARTICLE_COLUMNS}
FROM articles a
JOIN sources src ON src.id = a.source_id
JOIN article_scores s ON s.article_id = a.id
{fts_join}
{where}
ORDER BY {order_by}
LIMIT ? OFFSET ?
@@ -175,6 +187,27 @@ def feed(
return [dict(row) for row in rows]
def reindex_search(conn: sqlite3.Connection) -> int:
"""Rebuild the article_search FTS index from the accepted, non-duplicate corpus
(title/description/source name/tags). A cheap full rebuild (a few thousand
rows); run on each ingest cycle and lazily on first search. Live visibility /
boundary filtering is applied at query time, so it doesn't need reindexing."""
conn.execute("DELETE FROM article_search")
conn.execute(
"""
INSERT INTO article_search (article_id, title, body, source_name, tags)
SELECT a.id, a.title, COALESCE(a.description, ''), src.name,
COALESCE((SELECT group_concat(t.tag, ' ') FROM article_tags t WHERE t.article_id = a.id), '')
FROM articles a
JOIN sources src ON src.id = a.source_id
JOIN article_scores s ON s.article_id = a.id
WHERE s.accepted = 1 AND a.duplicate_of IS NULL
"""
)
conn.commit()
return conn.execute("SELECT COUNT(*) FROM article_search").fetchone()[0]
def brief(conn: sqlite3.Connection, brief_date: str | None = None, limit: int = 10) -> dict:
"""Return a stored daily brief (latest if no date) with its ranked items."""
target_date = brief_date or _latest_brief_date(conn)
@@ -344,6 +377,8 @@ def source_health(conn: sqlite3.Connection) -> list[dict]:
(SELECT COUNT(*) FROM articles a WHERE a.source_id = s.id) AS total_articles,
(SELECT COUNT(*) FROM articles a JOIN article_scores sc ON sc.article_id = a.id
WHERE a.source_id = s.id AND sc.accepted = 1) AS accepted_total,
(SELECT COUNT(*) FROM articles a JOIN article_scores sc ON sc.article_id = a.id
WHERE a.source_id = s.id AND sc.reason_code = 'non_english') AS non_english,
(SELECT COUNT(*) FROM articles a WHERE a.source_id = s.id AND a.duplicate_of IS NOT NULL) AS duplicates,
(SELECT COUNT(*) FROM articles a JOIN article_scores sc ON sc.article_id = a.id
WHERE a.source_id = s.id AND sc.accepted = 1 AND a.duplicate_of IS NULL) AS served,
@@ -365,7 +400,14 @@ def source_health(conn: sqlite3.Connection) -> list[dict]:
d = dict(r)
total = d["total_articles"] or 0
accepted = d["accepted_total"] or 0
d["acceptance_rate"] = round(100 * accepted / total) if total else None
non_english = d.get("non_english") or 0
# Acceptance is judged over articles actually scored in English — non-English
# items are HELD (awaiting translation), not calm-filter rejections, so they
# don't drag a multilingual source's rate down.
judged = total - non_english
d["acceptance_rate"] = round(100 * accepted / judged) if judged else None
d["non_english"] = non_english
d["non_english_rate"] = round(100 * non_english / total) if total else None
d["duplicate_rate"] = round(100 * d["duplicates"] / total) if total else None
# Curation quality: of what this source got ACCEPTED, how much was a
# duplicate of content already served (accepted_total − served = accepted dupes).
@@ -459,7 +501,9 @@ def _attention(content: dict, sources: list[dict], feedback_unread: int, now: da
_SRC_ART_FILTERS = {
"accepted": "AND s.accepted = 1",
"rejected": "AND s.accepted = 0",
# 'rejected' = calm-filter rejections only; non-English is HELD, its own bucket.
"rejected": "AND s.accepted = 0 AND COALESCE(s.reason_code,'') != 'non_english'",
"held": "AND s.reason_code = 'non_english'",
"no_image": "AND (a.image_url IS NULL OR a.image_url = '')",
"duplicates": "AND a.duplicate_of IS NOT NULL",
}
@@ -493,6 +537,7 @@ def source_articles(conn: sqlite3.Connection, source_id: int, filter: str = "all
"published_at": r["published_at"] or r["discovered_at"],
"accepted": r["accepted"],
"reason": r["reason_text"] or r["reason_code"], # the "why" behind accept/reject
"held": r["reason_code"] == "non_english", # held for language, not rejected
"topic": r["topic"],
"flavor": r["flavor"],
"paywalled": is_paywalled_for_source(r["canonical_url"], override), # effective (domain rule + override)
@@ -510,7 +555,8 @@ def source_articles_summary(conn: sqlite3.Connection, source_id: int) -> dict:
"""
SELECT COUNT(*) total,
COALESCE(SUM(s.accepted = 1), 0) accepted,
COALESCE(SUM(s.accepted = 0), 0) rejected,
COALESCE(SUM(s.accepted = 0 AND COALESCE(s.reason_code,'') != 'non_english'), 0) rejected,
COALESCE(SUM(s.reason_code = 'non_english'), 0) non_english,
COALESCE(SUM(a.image_url IS NULL OR a.image_url = ''), 0) no_image,
COALESCE(SUM(a.duplicate_of IS NOT NULL), 0) duplicates
FROM articles a LEFT JOIN article_scores s ON s.article_id = a.id
@@ -523,6 +569,7 @@ def source_articles_summary(conn: sqlite3.Connection, source_id: int) -> dict:
url = (srow["homepage_url"] or srow["feed_url"]) if srow else None
return {
"total": agg["total"], "accepted": agg["accepted"], "rejected": agg["rejected"],
"non_english": agg["non_english"], # held for language (not a calm-filter rejection)
"no_image": agg["no_image"], "duplicates": agg["duplicates"],
"paywalled": is_paywalled_for_source(url, override), # effective
"paywall_domain": is_paywalled(url), # what the domain rule alone says
@@ -533,6 +580,11 @@ def source_articles_summary(conn: sqlite3.Connection, source_id: int) -> dict:
def admin_stats(conn: sqlite3.Connection, days: int = 30) -> dict:
"""Aggregate, non-personal usage stats for the admin dashboard."""
since = f"-{days} days"
# "Today" for timestamp-based counters is the SITE-LOCAL day (GOODNEWS_TZ), not
# UTC: otherwise an evening error (e.g. 22:53 local) lands on the next UTC day and
# reads as a fresh "today" the following morning — the exact false-alarm we hit.
local_day_start = (local_now().replace(hour=0, minute=0, second=0, microsecond=0)
.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S"))
def scalar(sql, params=()):
return conn.execute(sql, params).fetchone()[0] or 0
@@ -658,7 +710,8 @@ def admin_stats(conn: sqlite3.Connection, days: int = 30) -> dict:
# check routinely and would read as real users seeing blank screens.
"client_errors": {
"today": scalar(
f"SELECT COUNT(*) FROM client_errors WHERE date(created_at)=date('now') AND {_NOT_BOT_SQL}"
f"SELECT COUNT(*) FROM client_errors WHERE created_at >= ? AND {_NOT_BOT_SQL}",
(local_day_start,),
),
"window": scalar(
f"SELECT COUNT(*) FROM client_errors WHERE created_at>=date('now',?) AND {_NOT_BOT_SQL}",