From 1a8d1b3bf1216284a4326a6a9ccb4c89352c686c Mon Sep 17 00:00:00 2001 From: jay Date: Tue, 9 Jun 2026 10:28:00 -0400 Subject: [PATCH] Promote-candidate UI: add-a-source pipeline in the admin console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the supervised source-candidate flow into Sources (Codex's v1 scope), so adding feeds no longer needs the CLI. * feeds.safe_fetch_feed: SSRF-safe fetch for UNTRUSTED (admin-pasted) URLs — http(s) only, every redirect hop re-validated via enrich._host_is_public, body size-capped, bounded redirects, no cookies. preview_feed gains a `fetcher` param; the API path passes safe_fetch_feed (NOT the raw fetch_feed used for already-vetted polling). * API (admin-gated): GET /candidates; POST /candidates (suggest+preview, gated before the outbound fetch, no DB conn held during network); /{id}/preview (explicit re-preview); /{id}/promote (paused by default, returns the new source + updated candidate); /{id}/reject. rejected stays on candidates only. * Admin Sources tab: "Add a source" field + a candidate queue showing the preview (pass rate, recent count, example headlines) with Promote (as paused, or Activate immediately) / Re-preview / Reject. Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend/src/routes/admin/+page.svelte | 105 +++++++++++++++++++++++++ goodnews/api.py | 94 +++++++++++++++++++++- goodnews/feeds.py | 44 ++++++++++- tests/test_admin.py | 37 +++++++++ 4 files changed, 276 insertions(+), 4 deletions(-) diff --git a/frontend/src/routes/admin/+page.svelte b/frontend/src/routes/admin/+page.svelte index 61f58f7..dc17fa7 100644 --- a/frontend/src/routes/admin/+page.svelte +++ b/frontend/src/routes/admin/+page.svelte @@ -30,6 +30,7 @@ try { await loadStats(); feedback = await getJSON('/api/admin/feedback'); + candidates = await getJSON('/api/admin/candidates'); } catch { error = "Couldn't load stats."; } @@ -116,6 +117,51 @@ catch { s.review_flag = 1; s.review_reason = prevR; } } + // --- Source candidates: supervised "add a source" pipeline --- + let candidates = $state([]); + let newFeedUrl = $state(''); + let newFeedName = $state(''); + let addBusy = $state(false); + let addErr = $state(''); + let pendingCandidates = $derived(candidates.filter((c) => c.status !== 'promoted' && c.status !== 'rejected')); + + async function addCandidate() { + const url = newFeedUrl.trim(); + if (!url || addBusy) return; + addBusy = true; addErr = ''; + try { + const c = await postJSON('/api/admin/candidates', { feed_url: url, name: newFeedName.trim() || null }); + candidates = [c, ...candidates.filter((x) => x.id !== c.id)]; + newFeedUrl = ''; newFeedName = ''; + } catch (e) { + addErr = e?.message || "Couldn't preview that feed."; + } finally { + addBusy = false; + } + } + async function repreviewCandidate(c) { + c._err = ''; + try { const u = await postJSON(`/api/admin/candidates/${c.id}/preview`); c.preview = u.preview; } + catch (e) { c._err = e?.message || 'Re-preview failed.'; } + } + async function promoteCandidate(c) { + c._err = ''; + try { + await postJSON(`/api/admin/candidates/${c.id}/promote`, { + default_category: (c._cat || '').trim() || null, + active: !!c._activate, + }); + c.status = 'promoted'; + await loadStats(); // surface the new (paused) source in the table below + } catch (e) { + c._err = e?.message || 'Promote failed.'; + } + } + async function rejectCandidate(c) { + try { await postJSON(`/api/admin/candidates/${c.id}/reject`); c.status = 'rejected'; } + catch { /* leave as-is */ } + } + // Feedback inbox: filter + read/unread + delete. let fbCat = $state('all'); // 'all' | 'unread' | a category let shownFeedback = $derived( @@ -338,6 +384,48 @@ {:else if section === 'sources'} +
+

Add a source

+
+ e.key === 'Enter' && addCandidate()} /> + + +
+ {#if addErr}

{addErr}

{/if} +

Fetches a sample (safely) and stages it as a candidate — nothing is added live until you Promote.

+
+ + {#if pendingCandidates.length} +
+

Candidate queue ({pendingCandidates.length})

+
    + {#each pendingCandidates as c (c.id)} +
  • +
    + {c.name || c.feed_url} + {c.status} +
    +
    {c.feed_url}
    + {#if c.preview} +
    + {c.preview.accepted ?? 0}/{c.preview.sampled ?? 0} sampled would pass{#if c.preview.acceptance_rate != null} · {Math.round(c.preview.acceptance_rate * 100)}% accept{/if}{#if c.preview.recent_7d != null} · {c.preview.recent_7d} in last 7d{/if} + {#if c.preview.examples_accepted?.length}
    e.g. {c.preview.examples_accepted.slice(0, 3).join(' · ')}
    {/if} +
    + {/if} + {#if c._err}

    {c._err}

    {/if} +
    + + + + + +
    +
    • + {/each} +
+
+ {/if} +

Sources

{healthy} healthy · {resting} resting · {flagged} flagged · {paused} paused · {retired} retired · {sources.length} total

@@ -664,6 +752,23 @@ .filterchips .chip:hover { border-color: var(--accent); } .filterchips .chip.on { background: var(--accent); border-color: var(--accent); color: #fff; } + /* Add a source + candidate queue */ + .addsrc { background: var(--surface); border: 1px solid var(--line); border-radius: 14px; padding: 14px 16px; margin-bottom: 6px; } + .addrow { display: flex; gap: 8px; flex-wrap: wrap; } + .addrow input { flex: 1 1 200px; box-sizing: border-box; font: inherit; font-size: 0.9rem; padding: 8px 11px; border: 1px solid var(--line); border-radius: 9px; background: var(--bg); color: var(--ink); } + .addrow input:focus { outline: none; border-color: var(--accent); } + ul.candlist { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 10px; } + ul.candlist li { background: var(--surface); border: 1px solid var(--line); border-radius: 12px; padding: 12px 14px; } + .chead { display: flex; align-items: baseline; gap: 10px; } + .chead .cname { font-weight: 600; color: var(--ink); } + .chead .cstatus { font-size: 0.72rem; color: var(--muted); text-transform: capitalize; } + .curl { font-size: 0.76rem; color: var(--muted); word-break: break-all; margin-top: 2px; } + .cprev { font-size: 0.84rem; color: var(--ink); margin-top: 7px; } + .cprev .cex { color: var(--muted); font-size: 0.8rem; margin-top: 2px; font-style: italic; } + .cactions { display: flex; gap: 9px; align-items: center; flex-wrap: wrap; margin-top: 10px; } + .cactions .ccat { font: inherit; font-size: 0.8rem; padding: 5px 9px; border: 1px solid var(--line); border-radius: 8px; background: var(--bg); color: var(--ink); width: 150px; } + .cactions .cchk { font-size: 0.8rem; color: var(--muted); display: inline-flex; align-items: center; gap: 5px; } + /* Source health table */ .tablewrap { overflow-x: auto; } .srctable { width: 100%; border-collapse: collapse; font-size: 0.86rem; min-width: 560px; } diff --git a/goodnews/api.py b/goodnews/api.py index 37785de..dd7d7b7 100644 --- a/goodnews/api.py +++ b/goodnews/api.py @@ -31,7 +31,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel -from . import auth, email_send, feeds, oauth_google, queries, share, summarize +from . import auth, email_send, feeds, oauth_google, queries, share, sources, summarize from .markup import reply_html_to_text, sanitize_reply_html from .db import connect from .filters import filter_articles, prefs_from_json @@ -394,6 +394,19 @@ class SourceVisibilityBody(BaseModel): visible: bool = True +class CandidateSuggestBody(BaseModel): + feed_url: str = "" + name: str | None = None + + +class CandidatePromoteBody(BaseModel): + default_category: str | None = None + active: bool = False # promote-as-paused by default; opt in to activate + trust_score: int = 5 + pr_risk_score: int = 3 + poll_interval_minutes: int = 180 + + class SourceReviewBody(BaseModel): flag: bool = False reason: str | None = None @@ -918,6 +931,85 @@ def create_app() -> FastAPI: conn.commit() return {"ok": True, "visible": body.visible} + # --- Source candidates (supervised add-a-source pipeline) ---------------- + + def _candidate_dict(row) -> dict: + d = dict(row) + raw = d.pop("preview_json", None) + try: + d["preview"] = json.loads(raw) if raw else None + except (ValueError, TypeError): + d["preview"] = None + return d + + @app.get("/api/admin/candidates") + def admin_candidates(request: Request) -> list[dict]: + with get_conn() as conn: + _require_admin(conn, request) + rows = sources.list_candidates(conn) + return [_candidate_dict(r) for r in rows] + + def _preview_or_502(url: str) -> dict: + # SSRF-safe fetch (admin-pasted URL is untrusted); heuristic-only (fast). + try: + return feeds.preview_feed(url, sample=20, fetcher=feeds.safe_fetch_feed) + except Exception as exc: # noqa: BLE001 — surface a readable reason + raise HTTPException(status_code=502, detail=f"Couldn't preview that feed: {exc}") + + @app.post("/api/admin/candidates") + def admin_candidate_suggest(body: CandidateSuggestBody, request: Request) -> dict: + url = (body.feed_url or "").strip() + if not url: + raise HTTPException(status_code=422, detail="feed_url is required") + with get_conn() as conn: # gate BEFORE the outbound fetch + _require_admin(conn, request) + preview = _preview_or_502(url) # no DB connection held during network I/O + with get_conn() as conn: + _require_admin(conn, request) + row = sources.save_candidate(conn, url, preview=preview, name=(body.name or None), status="suggested") + return _candidate_dict(row) + + @app.post("/api/admin/candidates/{cid}/preview") + def admin_candidate_repreview(cid: int, request: Request) -> dict: + with get_conn() as conn: + _require_admin(conn, request) + cand = conn.execute("SELECT feed_url FROM source_candidates WHERE id = ?", (cid,)).fetchone() + if not cand: + raise HTTPException(status_code=404, detail="candidate not found") + url = cand["feed_url"] + preview = _preview_or_502(url) + with get_conn() as conn: + _require_admin(conn, request) + row = sources.save_candidate(conn, url, preview=preview) + return _candidate_dict(row) + + @app.post("/api/admin/candidates/{cid}/promote") + def admin_candidate_promote(cid: int, body: CandidatePromoteBody, request: Request) -> dict: + with get_conn() as conn: + _require_admin(conn, request) + try: + source_id = sources.promote_candidate( + conn, cid, active=body.active, default_category=body.default_category, + trust_score=body.trust_score, pr_risk_score=body.pr_risk_score, + poll_interval_minutes=body.poll_interval_minutes, + ) + except ValueError: + raise HTTPException(status_code=404, detail="candidate not found") + src = conn.execute( + "SELECT id, name, status, active, content_visible FROM sources WHERE id = ?", (source_id,) + ).fetchone() + cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone() + return {"ok": True, "source_id": source_id, "source": dict(src), "candidate": _candidate_dict(cand)} + + @app.post("/api/admin/candidates/{cid}/reject") + def admin_candidate_reject(cid: int, request: Request) -> dict: + with get_conn() as conn: + _require_admin(conn, request) + if not sources.reject_candidate(conn, cid): + raise HTTPException(status_code=404, detail="candidate not found") + cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone() + return _candidate_dict(cand) + @app.post("/api/admin/sources/{sid}/review") def admin_source_review(sid: int, body: SourceReviewBody, request: Request) -> dict: with get_conn() as conn: diff --git a/goodnews/feeds.py b/goodnews/feeds.py index f49f797..6f05ff1 100644 --- a/goodnews/feeds.py +++ b/goodnews/feeds.py @@ -9,12 +9,15 @@ import xml.etree.ElementTree as ET from collections import Counter from dataclasses import dataclass from datetime import UTC, datetime +from urllib.parse import urljoin, urlsplit +from .enrich import MAX_REDIRECTS, _NoRedirect, _host_is_public from .scoring import score_article from .text import canonicalize_url, clean_text, sha256_text USER_AGENT = "goodNews/0.1 (+local constructive news prototype)" +FEED_MAX_BYTES = 2_000_000 # cap on a fetched feed body (SSRF-safe preview path) @dataclass @@ -176,14 +179,15 @@ def poll_source(conn: sqlite3.Connection, source: sqlite3.Row) -> dict: } -def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None) -> dict: +def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None, fetcher=None) -> dict: """Fetch and score a sample of a feed WITHOUT persisting anything. Read-only: lets a user vet a candidate source before it is ever added. By default it uses the fast heuristic; pass an LLM client to also get the - topic/flavor mix and the model's acceptance view (slower). + topic/flavor mix and the model's acceptance view (slower). Pass + fetcher=safe_fetch_feed for untrusted (admin-pasted) URLs. """ - items = parse_feed(fetch_feed(url)) + items = parse_feed((fetcher or fetch_feed)(url)) rows = [] for item in items[:sample]: title = clean_text(item.title, max_len=500) @@ -287,6 +291,40 @@ def fetch_feed(url: str, timeout: int = 20) -> bytes: raise RuntimeError(f"failed fetching {url}: {exc.reason}") from exc +def safe_fetch_feed(url: str, timeout: int = 10) -> bytes: + """SSRF-safe feed fetch for UNTRUSTED (admin-pasted) URLs. + + Unlike fetch_feed (used for already-vetted, curated feeds), this re-validates + every redirect hop against public IPs (reusing enrich._host_is_public), + allows only http(s), caps the body, follows a bounded number of redirects, + and sends no cookies/credentials. Use this for the preview/candidate path. + """ + opener = urllib.request.build_opener(_NoRedirect) + current = url + for _ in range(MAX_REDIRECTS + 1): + parts = urlsplit(current) + if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname): + raise RuntimeError("refusing to fetch a non-public or non-http(s) URL") + request = urllib.request.Request(current, headers={"User-Agent": USER_AGENT}) + try: + response = opener.open(request, timeout=timeout) + except (urllib.error.URLError, OSError, ValueError) as exc: + raise RuntimeError(f"failed fetching {current}: {exc}") from exc + status = getattr(response, "status", 200) or 200 + if status in (301, 302, 303, 307, 308): + location = response.headers.get("Location") + response.close() + if not location: + raise RuntimeError("redirect without a location") + current = urljoin(current, location) + continue + try: + return response.read(FEED_MAX_BYTES) + finally: + response.close() + raise RuntimeError("too many redirects") + + def parse_feed(xml: bytes) -> list[FeedItem]: root = ET.fromstring(xml) root_name = _local_name(root.tag) diff --git a/tests/test_admin.py b/tests/test_admin.py index ba153a4..4f701bd 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -123,3 +123,40 @@ def test_admin_stats_days_param_clamped(tmp_path, monkeypatch): assert tc.get("/api/admin/stats?days=90").json()["days"] == 90 assert tc.get("/api/admin/stats?days=999").json()["days"] == 30 # clamped assert tc.get("/api/admin/stats").json()["days"] == 30 # default + + +def test_candidate_suggest_promote_paused(tmp_path, monkeypatch): + app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com") + monkeypatch.setattr(api.feeds, "preview_feed", + lambda url, **k: {"url": url, "sampled": 5, "accepted": 4, "examples_accepted": ["A", "B"]}) + tc = _signin(app, api, "boss@x.com") + cand = tc.post("/api/admin/candidates", json={"feed_url": "http://good/feed", "name": "Good Feed"}).json() + assert cand["status"] == "suggested" and cand["preview"]["accepted"] == 4 + cid = cand["id"] + assert any(c["id"] == cid for c in tc.get("/api/admin/candidates").json()) + # promote defaults to paused (active-on-approval off) — no mirror drift + res = tc.post(f"/api/admin/candidates/{cid}/promote", json={}).json() + assert res["source"]["status"] == "paused" and res["source"]["active"] == 0 + assert res["candidate"]["status"] == "promoted" + assert any(s["name"] == "Good Feed" for s in tc.get("/api/admin/stats").json()["sources"]) + + +def test_candidate_reject_and_gating(tmp_path, monkeypatch): + app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com") + monkeypatch.setattr(api.feeds, "preview_feed", lambda url, **k: {"url": url, "sampled": 1, "accepted": 0}) + tc = _signin(app, api, "boss@x.com") + cand = tc.post("/api/admin/candidates", json={"feed_url": "http://meh/feed"}).json() + assert tc.post(f"/api/admin/candidates/{cand['id']}/reject").json()["status"] == "rejected" + anon = TestClient(app) + assert anon.get("/api/admin/candidates").status_code == 401 + assert anon.post("/api/admin/candidates", json={"feed_url": "http://x/f"}).status_code == 401 + assert anon.post("/api/admin/candidates/1/promote", json={}).status_code == 401 + + +def test_safe_fetch_feed_blocks_ssrf(): + import pytest + from goodnews.feeds import safe_fetch_feed + for bad in ("http://127.0.0.1/x", "http://localhost/x", "file:///etc/passwd", + "http://169.254.169.254/latest", "ftp://x/y"): + with pytest.raises(RuntimeError): + safe_fetch_feed(bad, timeout=2)