Track 3: read-only source preview (vet a feed before adding)
- feeds.preview_feed(): fetch + score a sample WITHOUT persisting; returns freshness, acceptance rate, cortisol/ragebait/PR averages, and example accepted/rejected items. With an LLM client it also returns topic/flavor mix and the model's (accurate) acceptance view. - CLI 'preview-source URL [--sample] [--classify]'. - API 'GET /api/source-preview?url=&sample=&classify=' with an http(s)-only guard (SSRF note left for go-public hardening). - Site 'Suggest a source' panel with Quick check (heuristic, instant) and Deep check (model, accurate), rendered DOM-safely. - Tests: network-free preview_feed tests via monkeypatched fetch (45 total). - README documents the command, endpoint, and updated roadmap. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+38
-1
@@ -9,7 +9,14 @@ from pathlib import Path
|
||||
from .briefs import build_daily_brief, show_brief
|
||||
from .db import connect, init_db
|
||||
from .dedup import DEFAULT_THRESHOLD, DEFAULT_WINDOW_DAYS, dedup as run_dedup
|
||||
from .feeds import fetch_feed, parse_feed, poll_all_sources, poll_due_sources, poll_source
|
||||
from .feeds import (
|
||||
fetch_feed,
|
||||
parse_feed,
|
||||
poll_all_sources,
|
||||
poll_due_sources,
|
||||
poll_source,
|
||||
preview_feed,
|
||||
)
|
||||
from .llm import LocalModelClient, classify_articles
|
||||
from .scoring import score_article
|
||||
from .sources import load_sources, upsert_sources
|
||||
@@ -52,6 +59,13 @@ def main() -> None:
|
||||
check_feeds_parser = subparsers.add_parser("check-feeds", help="Fetch and parse each feed, reporting health")
|
||||
check_feeds_parser.add_argument("--all", action="store_true", help="Include inactive sources")
|
||||
|
||||
preview_parser = subparsers.add_parser("preview-source", help="Score a sample of a feed without adding it")
|
||||
preview_parser.add_argument("url", help="Feed URL to preview")
|
||||
preview_parser.add_argument("--sample", type=int, default=25)
|
||||
preview_parser.add_argument("--classify", action="store_true", help="Also classify with the local model (slower)")
|
||||
preview_parser.add_argument("--base-url", help="OpenAI-compatible base URL (with --classify)")
|
||||
preview_parser.add_argument("--model", help="Local model name (with --classify)")
|
||||
|
||||
runs_parser = subparsers.add_parser("list-runs", help="Show recent ingest runs")
|
||||
runs_parser.add_argument("--limit", type=int, default=20)
|
||||
|
||||
@@ -136,6 +150,10 @@ def main() -> None:
|
||||
source_report(conn)
|
||||
elif args.command == "check-feeds":
|
||||
check_feeds(conn, include_inactive=args.all)
|
||||
elif args.command == "preview-source":
|
||||
client = llm_client_from_args(args) if args.classify else None
|
||||
preview = preview_feed(args.url, sample=args.sample, client=client)
|
||||
print_preview(preview)
|
||||
elif args.command == "list-runs":
|
||||
list_runs(conn, limit=args.limit)
|
||||
elif args.command == "rescore":
|
||||
@@ -243,6 +261,25 @@ def list_recent(conn: sqlite3.Connection, limit: int, accepted_only: bool) -> No
|
||||
print(f" {row['canonical_url']}")
|
||||
|
||||
|
||||
def print_preview(p: dict) -> None:
|
||||
mode = "model" if p["classified"] else "heuristic"
|
||||
print(f"Preview of {p['url']} ({mode})")
|
||||
print(f" sampled={p['sampled']} accepted={p['accepted']} ({p['acceptance_rate']*100:.0f}%)")
|
||||
print(f" freshness: newest={p['newest_published'] or 'unknown'} in_last_7d={p['recent_7d']}")
|
||||
print(f" averages: cortisol={p['avg_cortisol']} ragebait={p['avg_ragebait']} pr_risk={p['avg_pr_risk']}")
|
||||
if p["topic_mix"]:
|
||||
print(f" topics: {p['topic_mix']}")
|
||||
print(f" flavors: {p['flavor_mix']}")
|
||||
if p["examples_accepted"]:
|
||||
print(" would accept:")
|
||||
for t in p["examples_accepted"]:
|
||||
print(f" + {t[:80]}")
|
||||
if p["examples_rejected"]:
|
||||
print(" would skip:")
|
||||
for ex in p["examples_rejected"]:
|
||||
print(f" - {ex['title'][:70]} ({ex['reason']})")
|
||||
|
||||
|
||||
def check_feeds(conn: sqlite3.Connection, include_inactive: bool = False) -> None:
|
||||
where = "" if include_inactive else "WHERE active = 1"
|
||||
rows = conn.execute(f"SELECT name, feed_url FROM sources {where} ORDER BY name").fetchall()
|
||||
|
||||
Reference in New Issue
Block a user