Promote-candidate UI: add-a-source pipeline in the admin console
Bring the supervised source-candidate flow into Sources (Codex's v1 scope), so
adding feeds no longer needs the CLI.
* feeds.safe_fetch_feed: SSRF-safe fetch for UNTRUSTED (admin-pasted) URLs —
http(s) only, every redirect hop re-validated via enrich._host_is_public,
body size-capped, bounded redirects, no cookies. preview_feed gains a
`fetcher` param; the API path passes safe_fetch_feed (NOT the raw fetch_feed
used for already-vetted polling).
* API (admin-gated): GET /candidates; POST /candidates (suggest+preview, gated
before the outbound fetch, no DB conn held during network); /{id}/preview
(explicit re-preview); /{id}/promote (paused by default, returns the new
source + updated candidate); /{id}/reject. rejected stays on candidates only.
* Admin Sources tab: "Add a source" field + a candidate queue showing the
preview (pass rate, recent count, example headlines) with Promote (as paused,
or Activate immediately) / Re-preview / Reject.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+41
-3
@@ -9,12 +9,15 @@ import xml.etree.ElementTree as ET
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from .enrich import MAX_REDIRECTS, _NoRedirect, _host_is_public
|
||||
from .scoring import score_article
|
||||
from .text import canonicalize_url, clean_text, sha256_text
|
||||
|
||||
|
||||
USER_AGENT = "goodNews/0.1 (+local constructive news prototype)"
|
||||
FEED_MAX_BYTES = 2_000_000 # cap on a fetched feed body (SSRF-safe preview path)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -176,14 +179,15 @@ def poll_source(conn: sqlite3.Connection, source: sqlite3.Row) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None) -> dict:
|
||||
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None, fetcher=None) -> dict:
|
||||
"""Fetch and score a sample of a feed WITHOUT persisting anything.
|
||||
|
||||
Read-only: lets a user vet a candidate source before it is ever added. By
|
||||
default it uses the fast heuristic; pass an LLM client to also get the
|
||||
topic/flavor mix and the model's acceptance view (slower).
|
||||
topic/flavor mix and the model's acceptance view (slower). Pass
|
||||
fetcher=safe_fetch_feed for untrusted (admin-pasted) URLs.
|
||||
"""
|
||||
items = parse_feed(fetch_feed(url))
|
||||
items = parse_feed((fetcher or fetch_feed)(url))
|
||||
rows = []
|
||||
for item in items[:sample]:
|
||||
title = clean_text(item.title, max_len=500)
|
||||
@@ -287,6 +291,40 @@ def fetch_feed(url: str, timeout: int = 20) -> bytes:
|
||||
raise RuntimeError(f"failed fetching {url}: {exc.reason}") from exc
|
||||
|
||||
|
||||
def safe_fetch_feed(url: str, timeout: int = 10) -> bytes:
|
||||
"""SSRF-safe feed fetch for UNTRUSTED (admin-pasted) URLs.
|
||||
|
||||
Unlike fetch_feed (used for already-vetted, curated feeds), this re-validates
|
||||
every redirect hop against public IPs (reusing enrich._host_is_public),
|
||||
allows only http(s), caps the body, follows a bounded number of redirects,
|
||||
and sends no cookies/credentials. Use this for the preview/candidate path.
|
||||
"""
|
||||
opener = urllib.request.build_opener(_NoRedirect)
|
||||
current = url
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
parts = urlsplit(current)
|
||||
if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
|
||||
raise RuntimeError("refusing to fetch a non-public or non-http(s) URL")
|
||||
request = urllib.request.Request(current, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
response = opener.open(request, timeout=timeout)
|
||||
except (urllib.error.URLError, OSError, ValueError) as exc:
|
||||
raise RuntimeError(f"failed fetching {current}: {exc}") from exc
|
||||
status = getattr(response, "status", 200) or 200
|
||||
if status in (301, 302, 303, 307, 308):
|
||||
location = response.headers.get("Location")
|
||||
response.close()
|
||||
if not location:
|
||||
raise RuntimeError("redirect without a location")
|
||||
current = urljoin(current, location)
|
||||
continue
|
||||
try:
|
||||
return response.read(FEED_MAX_BYTES)
|
||||
finally:
|
||||
response.close()
|
||||
raise RuntimeError("too many redirects")
|
||||
|
||||
|
||||
def parse_feed(xml: bytes) -> list[FeedItem]:
|
||||
root = ET.fromstring(xml)
|
||||
root_name = _local_name(root.tag)
|
||||
|
||||
Reference in New Issue
Block a user