Track 3: read-only source preview (vet a feed before adding)

- feeds.preview_feed(): fetch + score a sample WITHOUT persisting; returns
  freshness, acceptance rate, cortisol/ragebait/PR averages, and example
  accepted/rejected items. With an LLM client it also returns topic/flavor mix
  and the model's (accurate) acceptance view.
- CLI 'preview-source URL [--sample] [--classify]'.
- API 'GET /api/source-preview?url=&sample=&classify=' with an http(s)-only
  guard (SSRF note left for go-public hardening).
- Site 'Suggest a source' panel with Quick check (heuristic, instant) and Deep
  check (model, accurate), rendered DOM-safely.
- Tests: network-free preview_feed tests via monkeypatched fetch (45 total).
- README documents the command, endpoint, and updated roadmap.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jay
2026-05-30 19:37:34 +00:00
parent cabe0b6049
commit 95195daff8
6 changed files with 304 additions and 5 deletions
+38 -1
View File
@@ -9,7 +9,14 @@ from pathlib import Path
from .briefs import build_daily_brief, show_brief
from .db import connect, init_db
from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, dedup as run_dedup
from .feeds import fetch_feed, parse_feed, poll_all_sources, poll_due_sources, poll_source
from .feeds import (
fetch_feed,
parse_feed,
poll_all_sources,
poll_due_sources,
poll_source,
preview_feed,
)
from .llm import LocalModelClient, classify_articles
from .scoring import score_article
from .sources import load_sources, upsert_sources
@@ -52,6 +59,13 @@ def main() -> None:
check_feeds_parser = subparsers.add_parser("check-feeds", help="Fetch and parse each feed, reporting health")
check_feeds_parser.add_argument("--all", action="store_true", help="Include inactive sources")
preview_parser = subparsers.add_parser("preview-source", help="Score a sample of a feed without adding it")
preview_parser.add_argument("url", help="Feed URL to preview")
preview_parser.add_argument("--sample", type=int, default=25)
preview_parser.add_argument("--classify", action="store_true", help="Also classify with the local model (slower)")
preview_parser.add_argument("--base-url", help="OpenAI-compatible base URL (with --classify)")
preview_parser.add_argument("--model", help="Local model name (with --classify)")
runs_parser = subparsers.add_parser("list-runs", help="Show recent ingest runs")
runs_parser.add_argument("--limit", type=int, default=20)
@@ -136,6 +150,10 @@ def main() -> None:
source_report(conn)
elif args.command == "check-feeds":
check_feeds(conn, include_inactive=args.all)
elif args.command == "preview-source":
client = llm_client_from_args(args) if args.classify else None
preview = preview_feed(args.url, sample=args.sample, client=client)
print_preview(preview)
elif args.command == "list-runs":
list_runs(conn, limit=args.limit)
elif args.command == "rescore":
@@ -243,6 +261,25 @@ def list_recent(conn: sqlite3.Connection, limit: int, accepted_only: bool) -> No
print(f" {row['canonical_url']}")
def print_preview(p: dict) -> None:
mode = "model" if p["classified"] else "heuristic"
print(f"Preview of {p['url']} ({mode})")
print(f" sampled={p['sampled']} accepted={p['accepted']} ({p['acceptance_rate']*100:.0f}%)")
print(f" freshness: newest={p['newest_published'] or 'unknown'} in_last_7d={p['recent_7d']}")
print(f" averages: cortisol={p['avg_cortisol']} ragebait={p['avg_ragebait']} pr_risk={p['avg_pr_risk']}")
if p["topic_mix"]:
print(f" topics: {p['topic_mix']}")
print(f" flavors: {p['flavor_mix']}")
if p["examples_accepted"]:
print(" would accept:")
for t in p["examples_accepted"]:
print(f" + {t[:80]}")
if p["examples_rejected"]:
print(" would skip:")
for ex in p["examples_rejected"]:
print(f" - {ex['title'][:70]} ({ex['reason']})")
def check_feeds(conn: sqlite3.Connection, include_inactive: bool = False) -> None:
where = "" if include_inactive else "WHERE active = 1"
rows = conn.execute(f"SELECT name, feed_url FROM sources {where} ORDER BY name").fetchall()