Promote-candidate UI: add-a-source pipeline in the admin console

Bring the supervised source-candidate flow into Sources (Codex's v1 scope), so
adding feeds no longer needs the CLI.

* feeds.safe_fetch_feed: SSRF-safe fetch for UNTRUSTED (admin-pasted) URLs —
  http(s) only, every redirect hop re-validated via enrich._host_is_public,
  body size-capped, bounded redirects, no cookies. preview_feed gains a
  `fetcher` param; the API path passes safe_fetch_feed (NOT the raw fetch_feed
  used for already-vetted polling).
* API (admin-gated): GET /candidates; POST /candidates (suggest+preview, gated
  before the outbound fetch, no DB conn held during network); /{id}/preview
  (explicit re-preview); /{id}/promote (paused by default, returns the new
  source + updated candidate); /{id}/reject. rejected stays on candidates only.
* Admin Sources tab: "Add a source" field + a candidate queue showing the
  preview (pass rate, recent count, example headlines) with Promote (as paused,
  or Activate immediately) / Re-preview / Reject.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-09 10:28:00 -04:00
parent 35aaeece6d
commit 1a8d1b3bf1
4 changed files with 276 additions and 4 deletions
+41 -3
View File
@@ -9,12 +9,15 @@ import xml.etree.ElementTree as ET
from collections import Counter
from dataclasses import dataclass
from datetime import UTC, datetime
from urllib.parse import urljoin, urlsplit
from .enrich import MAX_REDIRECTS, _NoRedirect, _host_is_public
from .scoring import score_article
from .text import canonicalize_url, clean_text, sha256_text
USER_AGENT = "goodNews/0.1 (+local constructive news prototype)"
FEED_MAX_BYTES = 2_000_000 # cap on a fetched feed body (SSRF-safe preview path)
@dataclass
@@ -176,14 +179,15 @@ def poll_source(conn: sqlite3.Connection, source: sqlite3.Row) -> dict:
}
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None) -> dict:
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None, fetcher=None) -> dict:
"""Fetch and score a sample of a feed WITHOUT persisting anything.
Read-only: lets a user vet a candidate source before it is ever added. By
default it uses the fast heuristic; pass an LLM client to also get the
topic/flavor mix and the model's acceptance view (slower).
topic/flavor mix and the model's acceptance view (slower). Pass
fetcher=safe_fetch_feed for untrusted (admin-pasted) URLs.
"""
items = parse_feed(fetch_feed(url))
items = parse_feed((fetcher or fetch_feed)(url))
rows = []
for item in items[:sample]:
title = clean_text(item.title, max_len=500)
@@ -287,6 +291,40 @@ def fetch_feed(url: str, timeout: int = 20) -> bytes:
raise RuntimeError(f"failed fetching {url}: {exc.reason}") from exc
def safe_fetch_feed(url: str, timeout: int = 10) -> bytes:
"""SSRF-safe feed fetch for UNTRUSTED (admin-pasted) URLs.
Unlike fetch_feed (used for already-vetted, curated feeds), this re-validates
every redirect hop against public IPs (reusing enrich._host_is_public),
allows only http(s), caps the body, follows a bounded number of redirects,
and sends no cookies/credentials. Use this for the preview/candidate path.
"""
opener = urllib.request.build_opener(_NoRedirect)
current = url
for _ in range(MAX_REDIRECTS + 1):
parts = urlsplit(current)
if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
raise RuntimeError("refusing to fetch a non-public or non-http(s) URL")
request = urllib.request.Request(current, headers={"User-Agent": USER_AGENT})
try:
response = opener.open(request, timeout=timeout)
except (urllib.error.URLError, OSError, ValueError) as exc:
raise RuntimeError(f"failed fetching {current}: {exc}") from exc
status = getattr(response, "status", 200) or 200
if status in (301, 302, 303, 307, 308):
location = response.headers.get("Location")
response.close()
if not location:
raise RuntimeError("redirect without a location")
current = urljoin(current, location)
continue
try:
return response.read(FEED_MAX_BYTES)
finally:
response.close()
raise RuntimeError("too many redirects")
def parse_feed(xml: bytes) -> list[FeedItem]:
root = ET.fromstring(xml)
root_name = _local_name(root.tag)