Track 3: read-only source preview (vet a feed before adding)
- feeds.preview_feed(): fetch + score a sample WITHOUT persisting; returns freshness, acceptance rate, cortisol/ragebait/PR averages, and example accepted/rejected items. With an LLM client it also returns topic/flavor mix and the model's (accurate) acceptance view. - CLI 'preview-source URL [--sample] [--classify]'. - API 'GET /api/source-preview?url=&sample=&classify=' with an http(s)-only guard (SSRF note left for go-public hardening). - Site 'Suggest a source' panel with Quick check (heuristic, instant) and Deep check (model, accurate), rendered DOM-safely. - Tests: network-free preview_feed tests via monkeypatched fetch (45 total). - README documents the command, endpoint, and updated roadmap. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import sqlite3
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -133,6 +134,106 @@ def poll_source(conn: sqlite3.Connection, source: sqlite3.Row) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None) -> dict:
|
||||
"""Fetch and score a sample of a feed WITHOUT persisting anything.
|
||||
|
||||
Read-only: lets a user vet a candidate source before it is ever added. By
|
||||
default it uses the fast heuristic; pass an LLM client to also get the
|
||||
topic/flavor mix and the model's acceptance view (slower).
|
||||
"""
|
||||
items = parse_feed(fetch_feed(url))
|
||||
rows = []
|
||||
for item in items[:sample]:
|
||||
title = clean_text(item.title, max_len=500)
|
||||
if not title:
|
||||
continue
|
||||
description = clean_text(item.description, max_len=1000)
|
||||
s = score_article(title, description, pr_risk_default)
|
||||
rows.append(
|
||||
{
|
||||
"title": title,
|
||||
"description": description,
|
||||
"url": canonicalize_url(item.url),
|
||||
"published_at": item.published_at,
|
||||
"accepted": bool(s["accepted"]),
|
||||
"cortisol": s["cortisol_score"],
|
||||
"ragebait": s["ragebait_score"],
|
||||
"pr_risk": s["pr_risk_score"],
|
||||
"reason_code": s["reason_code"],
|
||||
"topic": None,
|
||||
"flavor": None,
|
||||
}
|
||||
)
|
||||
|
||||
classified = False
|
||||
if client and rows:
|
||||
from .llm import normalize_scores
|
||||
|
||||
classified = True
|
||||
for r in rows:
|
||||
try:
|
||||
raw = client.classify(
|
||||
{
|
||||
"source_name": "preview",
|
||||
"default_category": None,
|
||||
"source_trust_score": 5,
|
||||
"source_pr_risk_score": pr_risk_default,
|
||||
"published_at": r["published_at"],
|
||||
"title": r["title"],
|
||||
"description": r["description"] or "",
|
||||
"canonical_url": r["url"],
|
||||
}
|
||||
)
|
||||
ns = normalize_scores(raw, model_name=client.model)
|
||||
r.update(
|
||||
accepted=bool(ns["accepted"]),
|
||||
topic=ns["topic"],
|
||||
flavor=ns["flavor"],
|
||||
cortisol=ns["cortisol_score"],
|
||||
ragebait=ns["ragebait_score"],
|
||||
pr_risk=ns["pr_risk_score"],
|
||||
)
|
||||
except Exception:
|
||||
pass # one bad item shouldn't sink the whole preview
|
||||
|
||||
total = len(rows)
|
||||
accepted = sum(1 for r in rows if r["accepted"])
|
||||
|
||||
def _avg(key: str) -> float:
|
||||
return round(sum(r[key] for r in rows) / total, 1) if total else 0.0
|
||||
|
||||
# Freshness: newest item and how many landed in the last week.
|
||||
now = datetime.now(UTC)
|
||||
dates = []
|
||||
for r in rows:
|
||||
if r["published_at"]:
|
||||
try:
|
||||
dates.append(datetime.fromisoformat(r["published_at"]))
|
||||
except ValueError:
|
||||
pass
|
||||
newest = max(dates).isoformat() if dates else None
|
||||
recent_7d = sum(1 for d in dates if (now - d).days <= 7)
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"sampled": total,
|
||||
"classified": classified,
|
||||
"accepted": accepted,
|
||||
"acceptance_rate": round(accepted / total, 2) if total else 0.0,
|
||||
"avg_cortisol": _avg("cortisol"),
|
||||
"avg_ragebait": _avg("ragebait"),
|
||||
"avg_pr_risk": _avg("pr_risk"),
|
||||
"newest_published": newest,
|
||||
"recent_7d": recent_7d,
|
||||
"topic_mix": dict(Counter(r["topic"] for r in rows if r["topic"])),
|
||||
"flavor_mix": dict(Counter(r["flavor"] for r in rows if r["flavor"])),
|
||||
"examples_accepted": [r["title"] for r in rows if r["accepted"]][:5],
|
||||
"examples_rejected": [
|
||||
{"title": r["title"], "reason": r["reason_code"]} for r in rows if not r["accepted"]
|
||||
][:5],
|
||||
}
|
||||
|
||||
|
||||
def fetch_feed(url: str, timeout: int = 20) -> bytes:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user