Sparse-day-proof briefs, feed health check, and 16 new sources

- Briefs now fill from a rolling window (prefer today, backfill up to
  window_days) and exclude anything featured in the last 7 days of briefs, so
  slow days still produce five items without stories lingering day to day.
- New 'check-feeds' command fetches and parses every feed to catch dead ones.
- Added 16 validated sources (science, environment, animals, culture),
  expanding coverage from 12 to 28 feeds to reduce staleness.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jay
2026-05-30 15:30:03 +00:00
parent cef272a8fc
commit 2a9c49e2a9
3 changed files with 192 additions and 6 deletions
+20 -1
View File
@@ -8,7 +8,7 @@ from pathlib import Path
from .briefs import build_daily_brief, show_brief
from .db import connect, init_db
from .feeds import poll_all_sources, poll_due_sources, poll_source
from .feeds import fetch_feed, parse_feed, poll_all_sources, poll_due_sources, poll_source
from .llm import LocalModelClient, classify_articles
from .scoring import score_article
from .sources import load_sources, upsert_sources
@@ -48,6 +48,9 @@ def main() -> None:
subparsers.add_parser("source-report", help="Show source-level ingestion and scoring stats")
check_feeds_parser = subparsers.add_parser("check-feeds", help="Fetch and parse each feed, reporting health")
check_feeds_parser.add_argument("--all", action="store_true", help="Include inactive sources")
runs_parser = subparsers.add_parser("list-runs", help="Show recent ingest runs")
runs_parser.add_argument("--limit", type=int, default=20)
@@ -122,6 +125,8 @@ def main() -> None:
list_category(conn, topic=args.topic, flavor=args.flavor, limit=args.limit, accepted_only=not args.all)
elif args.command == "source-report":
source_report(conn)
elif args.command == "check-feeds":
check_feeds(conn, include_inactive=args.all)
elif args.command == "list-runs":
list_runs(conn, limit=args.limit)
elif args.command == "rescore":
@@ -214,6 +219,20 @@ def list_recent(conn: sqlite3.Connection, limit: int, accepted_only: bool) -> No
print(f" {row['canonical_url']}")
def check_feeds(conn: sqlite3.Connection, include_inactive: bool = False) -> None:
where = "" if include_inactive else "WHERE active = 1"
rows = conn.execute(f"SELECT name, feed_url FROM sources {where} ORDER BY name").fetchall()
ok = 0
for row in rows:
try:
items = parse_feed(fetch_feed(row["feed_url"]))
ok += 1
print(f"OK {row['name']}: {len(items)} items")
except Exception as exc:
print(f"FAIL {row['name']}: {exc}")
print(f"--- {ok}/{len(rows)} feeds healthy ---")
def run_cycle(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
"""One end-to-end pass for a scheduler: poll due sources, classify the new
arrivals, rebuild today's brief. Each step is independent and non-fatal so a