thejayman77/upbeatBytes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Promote-candidate UI: add-a-source pipeline in the admin console

Browse Source 
Bring the supervised source-candidate flow into Sources (Codex's v1 scope), so
adding feeds no longer needs the CLI.

* feeds.safe_fetch_feed: SSRF-safe fetch for UNTRUSTED (admin-pasted) URLs —
  http(s) only, every redirect hop re-validated via enrich._host_is_public,
  body size-capped, bounded redirects, no cookies. preview_feed gains a
  `fetcher` param; the API path passes safe_fetch_feed (NOT the raw fetch_feed
  used for already-vetted polling).
* API (admin-gated): GET /candidates; POST /candidates (suggest+preview, gated
  before the outbound fetch, no DB conn held during network); /{id}/preview
  (explicit re-preview); /{id}/promote (paused by default, returns the new
  source + updated candidate); /{id}/reject. rejected stays on candidates only.
* Admin Sources tab: "Add a source" field + a candidate queue showing the
  preview (pass rate, recent count, example headlines) with Promote (as paused,
  or Activate immediately) / Re-preview / Reject.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-09 10:28:00 -04:00
parent 35aaeece6d
commit 1a8d1b3bf1
4 changed files with 276 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/routes/admin/+page.svelte
+105
View File
@@ -30,6 +30,7 @@
try {
await loadStats();
feedback = await getJSON('/api/admin/feedback');
candidates = await getJSON('/api/admin/candidates');
} catch {
error = "Couldn't load stats.";
}
@@ -116,6 +117,51 @@
catch { s.review_flag = 1; s.review_reason = prevR; }
}
// --- Source candidates: supervised "add a source" pipeline ---
let candidates = $state([]);
let newFeedUrl = $state('');
let newFeedName = $state('');
let addBusy = $state(false);
let addErr = $state('');
let pendingCandidates = $derived(candidates.filter((c) => c.status !== 'promoted' && c.status !== 'rejected'));
async function addCandidate() {
const url = newFeedUrl.trim();
if (!url || addBusy) return;
addBusy = true; addErr = '';
try {
const c = await postJSON('/api/admin/candidates', { feed_url: url, name: newFeedName.trim() || null });
candidates = [c, ...candidates.filter((x) => x.id !== c.id)];
newFeedUrl = ''; newFeedName = '';
} catch (e) {
addErr = e?.message || "Couldn't preview that feed.";
} finally {
addBusy = false;
}
}
async function repreviewCandidate(c) {
c._err = '';
try { const u = await postJSON(`/api/admin/candidates/${c.id}/preview`); c.preview = u.preview; }
catch (e) { c._err = e?.message || 'Re-preview failed.'; }
}
async function promoteCandidate(c) {
c._err = '';
try {
await postJSON(`/api/admin/candidates/${c.id}/promote`, {
default_category: (c._cat || '').trim() || null,
active: !!c._activate,
});
c.status = 'promoted';
await loadStats(); // surface the new (paused) source in the table below
} catch (e) {
c._err = e?.message || 'Promote failed.';
}
}
async function rejectCandidate(c) {
try { await postJSON(`/api/admin/candidates/${c.id}/reject`); c.status = 'rejected'; }
catch { /* leave as-is */ }
}
// Feedback inbox: filter + read/unread + delete.
let fbCat = $state('all'); // 'all' | 'unread' | a category
let shownFeedback = $derived(
@@ -338,6 +384,48 @@
</div>
{:else if section === 'sources'}
<section class="addsrc">
<h2>Add a source</h2>
<div class="addrow">
<input type="url" placeholder="Feed URL (RSS / Atom)" bind:value={newFeedUrl} onkeydown={(e) => e.key === 'Enter' && addCandidate()} />
<input type="text" placeholder="Name (optional)" bind:value={newFeedName} />
<button class="csend" onclick={addCandidate} disabled={addBusy || !newFeedUrl.trim()}>{addBusy ? 'Previewing…' : 'Preview & add'}</button>
</div>
{#if addErr}<p class="cerr">{addErr}</p>{/if}
<p class="legend2">Fetches a sample (safely) and stages it as a candidate — nothing is added live until you Promote.</p>
</section>
{#if pendingCandidates.length}
<section>
<h2>Candidate queue <span class="count">({pendingCandidates.length})</span></h2>
<ul class="candlist">
{#each pendingCandidates as c (c.id)}
<li>
<div class="chead">
<span class="cname">{c.name || c.feed_url}</span>
<span class="cstatus">{c.status}</span>
</div>
<div class="curl">{c.feed_url}</div>
{#if c.preview}
<div class="cprev">
{c.preview.accepted ?? 0}/{c.preview.sampled ?? 0} sampled would pass{#if c.preview.acceptance_rate != null} · {Math.round(c.preview.acceptance_rate * 100)}% accept{/if}{#if c.preview.recent_7d != null} · {c.preview.recent_7d} in last 7d{/if}
{#if c.preview.examples_accepted?.length}<div class="cex">e.g. {c.preview.examples_accepted.slice(0, 3).join(' · ')}</div>{/if}
</div>
{/if}
{#if c._err}<p class="cerr">{c._err}</p>{/if}
<div class="cactions">
<input class="ccat" type="text" placeholder="category (optional)" bind:value={c._cat} />
<label class="cchk"><input type="checkbox" bind:checked={c._activate} /> Activate immediately</label>
<button class="csend" onclick={() => promoteCandidate(c)}>Promote{c._activate ? '' : ' as paused'}</button>
<button class="act" onclick={() => repreviewCandidate(c)}>Re-preview</button>
<button class="act del" onclick={() => rejectCandidate(c)}>Reject</button>
</div>
</li>
{/each}
</ul>
</section>
{/if}
<h2>Sources</h2>
<p class="sub2">{healthy} healthy · {resting} resting · {flagged} flagged · {paused} paused · {retired} retired · {sources.length} total</p>
<div class="filterchips">
@@ -664,6 +752,23 @@
.filterchips .chip:hover { border-color: var(--accent); }
.filterchips .chip.on { background: var(--accent); border-color: var(--accent); color: #fff; }
/* Add a source + candidate queue */
.addsrc { background: var(--surface); border: 1px solid var(--line); border-radius: 14px; padding: 14px 16px; margin-bottom: 6px; }
.addrow { display: flex; gap: 8px; flex-wrap: wrap; }
.addrow input { flex: 1 1 200px; box-sizing: border-box; font: inherit; font-size: 0.9rem; padding: 8px 11px; border: 1px solid var(--line); border-radius: 9px; background: var(--bg); color: var(--ink); }
.addrow input:focus { outline: none; border-color: var(--accent); }
ul.candlist { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 10px; }
ul.candlist li { background: var(--surface); border: 1px solid var(--line); border-radius: 12px; padding: 12px 14px; }
.chead { display: flex; align-items: baseline; gap: 10px; }
.chead .cname { font-weight: 600; color: var(--ink); }
.chead .cstatus { font-size: 0.72rem; color: var(--muted); text-transform: capitalize; }
.curl { font-size: 0.76rem; color: var(--muted); word-break: break-all; margin-top: 2px; }
.cprev { font-size: 0.84rem; color: var(--ink); margin-top: 7px; }
.cprev .cex { color: var(--muted); font-size: 0.8rem; margin-top: 2px; font-style: italic; }
.cactions { display: flex; gap: 9px; align-items: center; flex-wrap: wrap; margin-top: 10px; }
.cactions .ccat { font: inherit; font-size: 0.8rem; padding: 5px 9px; border: 1px solid var(--line); border-radius: 8px; background: var(--bg); color: var(--ink); width: 150px; }
.cactions .cchk { font-size: 0.8rem; color: var(--muted); display: inline-flex; align-items: center; gap: 5px; }
/* Source health table */
.tablewrap { overflow-x: auto; }
.srctable { width: 100%; border-collapse: collapse; font-size: 0.86rem; min-width: 560px; }
goodnews/api.py
+93 -1
View File
@@ -31,7 +31,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from . import auth, email_send, feeds, oauth_google, queries, share, summarize
from . import auth, email_send, feeds, oauth_google, queries, share, sources, summarize
from .markup import reply_html_to_text, sanitize_reply_html
from .db import connect
from .filters import filter_articles, prefs_from_json
@@ -394,6 +394,19 @@ class SourceVisibilityBody(BaseModel):
visible: bool = True
class CandidateSuggestBody(BaseModel):
feed_url: str = ""
name: str | None = None
class CandidatePromoteBody(BaseModel):
default_category: str | None = None
active: bool = False # promote-as-paused by default; opt in to activate
trust_score: int = 5
pr_risk_score: int = 3
poll_interval_minutes: int = 180
class SourceReviewBody(BaseModel):
flag: bool = False
reason: str | None = None
@@ -918,6 +931,85 @@ def create_app() -> FastAPI:
conn.commit()
return {"ok": True, "visible": body.visible}
# --- Source candidates (supervised add-a-source pipeline) ----------------
def _candidate_dict(row) -> dict:
d = dict(row)
raw = d.pop("preview_json", None)
try:
d["preview"] = json.loads(raw) if raw else None
except (ValueError, TypeError):
d["preview"] = None
return d
@app.get("/api/admin/candidates")
def admin_candidates(request: Request) -> list[dict]:
with get_conn() as conn:
_require_admin(conn, request)
rows = sources.list_candidates(conn)
return [_candidate_dict(r) for r in rows]
def _preview_or_502(url: str) -> dict:
# SSRF-safe fetch (admin-pasted URL is untrusted); heuristic-only (fast).
try:
return feeds.preview_feed(url, sample=20, fetcher=feeds.safe_fetch_feed)
except Exception as exc: # noqa: BLE001 — surface a readable reason
raise HTTPException(status_code=502, detail=f"Couldn't preview that feed: {exc}")
@app.post("/api/admin/candidates")
def admin_candidate_suggest(body: CandidateSuggestBody, request: Request) -> dict:
url = (body.feed_url or "").strip()
if not url:
raise HTTPException(status_code=422, detail="feed_url is required")
with get_conn() as conn: # gate BEFORE the outbound fetch
_require_admin(conn, request)
preview = _preview_or_502(url) # no DB connection held during network I/O
with get_conn() as conn:
_require_admin(conn, request)
row = sources.save_candidate(conn, url, preview=preview, name=(body.name or None), status="suggested")
return _candidate_dict(row)
@app.post("/api/admin/candidates/{cid}/preview")
def admin_candidate_repreview(cid: int, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
cand = conn.execute("SELECT feed_url FROM source_candidates WHERE id = ?", (cid,)).fetchone()
if not cand:
raise HTTPException(status_code=404, detail="candidate not found")
url = cand["feed_url"]
preview = _preview_or_502(url)
with get_conn() as conn:
_require_admin(conn, request)
row = sources.save_candidate(conn, url, preview=preview)
return _candidate_dict(row)
@app.post("/api/admin/candidates/{cid}/promote")
def admin_candidate_promote(cid: int, body: CandidatePromoteBody, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
try:
source_id = sources.promote_candidate(
conn, cid, active=body.active, default_category=body.default_category,
trust_score=body.trust_score, pr_risk_score=body.pr_risk_score,
poll_interval_minutes=body.poll_interval_minutes,
)
except ValueError:
raise HTTPException(status_code=404, detail="candidate not found")
src = conn.execute(
"SELECT id, name, status, active, content_visible FROM sources WHERE id = ?", (source_id,)
).fetchone()
cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone()
return {"ok": True, "source_id": source_id, "source": dict(src), "candidate": _candidate_dict(cand)}
@app.post("/api/admin/candidates/{cid}/reject")
def admin_candidate_reject(cid: int, request: Request) -> dict:
with get_conn() as conn:
_require_admin(conn, request)
if not sources.reject_candidate(conn, cid):
raise HTTPException(status_code=404, detail="candidate not found")
cand = conn.execute("SELECT * FROM source_candidates WHERE id = ?", (cid,)).fetchone()
return _candidate_dict(cand)
@app.post("/api/admin/sources/{sid}/review")
def admin_source_review(sid: int, body: SourceReviewBody, request: Request) -> dict:
with get_conn() as conn:
goodnews/feeds.py
+41 -3
View File
@@ -9,12 +9,15 @@ import xml.etree.ElementTree as ET
from collections import Counter
from dataclasses import dataclass
from datetime import UTC, datetime
from urllib.parse import urljoin, urlsplit
from .enrich import MAX_REDIRECTS, _NoRedirect, _host_is_public
from .scoring import score_article
from .text import canonicalize_url, clean_text, sha256_text
USER_AGENT = "goodNews/0.1 (+local constructive news prototype)"
FEED_MAX_BYTES = 2_000_000 # cap on a fetched feed body (SSRF-safe preview path)
@dataclass
@@ -176,14 +179,15 @@ def poll_source(conn: sqlite3.Connection, source: sqlite3.Row) -> dict:
}
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None) -> dict:
def preview_feed(url: str, sample: int = 25, pr_risk_default: int = 3, client=None, fetcher=None) -> dict:
"""Fetch and score a sample of a feed WITHOUT persisting anything.
Read-only: lets a user vet a candidate source before it is ever added. By
default it uses the fast heuristic; pass an LLM client to also get the
topic/flavor mix and the model's acceptance view (slower).
topic/flavor mix and the model's acceptance view (slower). Pass
fetcher=safe_fetch_feed for untrusted (admin-pasted) URLs.
"""
items = parse_feed(fetch_feed(url))
items = parse_feed((fetcher or fetch_feed)(url))
rows = []
for item in items[:sample]:
title = clean_text(item.title, max_len=500)
@@ -287,6 +291,40 @@ def fetch_feed(url: str, timeout: int = 20) -> bytes:
raise RuntimeError(f"failed fetching {url}: {exc.reason}") from exc
def safe_fetch_feed(url: str, timeout: int = 10) -> bytes:
"""SSRF-safe feed fetch for UNTRUSTED (admin-pasted) URLs.
Unlike fetch_feed (used for already-vetted, curated feeds), this re-validates
every redirect hop against public IPs (reusing enrich._host_is_public),
allows only http(s), caps the body, follows a bounded number of redirects,
and sends no cookies/credentials. Use this for the preview/candidate path.
"""
opener = urllib.request.build_opener(_NoRedirect)
current = url
for _ in range(MAX_REDIRECTS + 1):
parts = urlsplit(current)
if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
raise RuntimeError("refusing to fetch a non-public or non-http(s) URL")
request = urllib.request.Request(current, headers={"User-Agent": USER_AGENT})
try:
response = opener.open(request, timeout=timeout)
except (urllib.error.URLError, OSError, ValueError) as exc:
raise RuntimeError(f"failed fetching {current}: {exc}") from exc
status = getattr(response, "status", 200) or 200
if status in (301, 302, 303, 307, 308):
location = response.headers.get("Location")
response.close()
if not location:
raise RuntimeError("redirect without a location")
current = urljoin(current, location)
continue
try:
return response.read(FEED_MAX_BYTES)
finally:
response.close()
raise RuntimeError("too many redirects")
def parse_feed(xml: bytes) -> list[FeedItem]:
root = ET.fromstring(xml)
root_name = _local_name(root.tag)
tests/test_admin.py
+37
View File
@@ -123,3 +123,40 @@ def test_admin_stats_days_param_clamped(tmp_path, monkeypatch):
assert tc.get("/api/admin/stats?days=90").json()["days"] == 90
assert tc.get("/api/admin/stats?days=999").json()["days"] == 30 # clamped
assert tc.get("/api/admin/stats").json()["days"] == 30 # default
def test_candidate_suggest_promote_paused(tmp_path, monkeypatch):
app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com")
monkeypatch.setattr(api.feeds, "preview_feed",
lambda url, **k: {"url": url, "sampled": 5, "accepted": 4, "examples_accepted": ["A", "B"]})
tc = _signin(app, api, "boss@x.com")
cand = tc.post("/api/admin/candidates", json={"feed_url": "http://good/feed", "name": "Good Feed"}).json()
assert cand["status"] == "suggested" and cand["preview"]["accepted"] == 4
cid = cand["id"]
assert any(c["id"] == cid for c in tc.get("/api/admin/candidates").json())
# promote defaults to paused (active-on-approval off) — no mirror drift
res = tc.post(f"/api/admin/candidates/{cid}/promote", json={}).json()
assert res["source"]["status"] == "paused" and res["source"]["active"] == 0
assert res["candidate"]["status"] == "promoted"
assert any(s["name"] == "Good Feed" for s in tc.get("/api/admin/stats").json()["sources"])
def test_candidate_reject_and_gating(tmp_path, monkeypatch):
app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com")
monkeypatch.setattr(api.feeds, "preview_feed", lambda url, **k: {"url": url, "sampled": 1, "accepted": 0})
tc = _signin(app, api, "boss@x.com")
cand = tc.post("/api/admin/candidates", json={"feed_url": "http://meh/feed"}).json()
assert tc.post(f"/api/admin/candidates/{cand['id']}/reject").json()["status"] == "rejected"
anon = TestClient(app)
assert anon.get("/api/admin/candidates").status_code == 401
assert anon.post("/api/admin/candidates", json={"feed_url": "http://x/f"}).status_code == 401
assert anon.post("/api/admin/candidates/1/promote", json={}).status_code == 401
def test_safe_fetch_feed_blocks_ssrf():
import pytest
from goodnews.feeds import safe_fetch_feed
for bad in ("http://127.0.0.1/x", "http://localhost/x", "file:///etc/passwd",
"http://169.254.169.254/latest", "ftp://x/y"):
with pytest.raises(RuntimeError):
safe_fetch_feed(bad, timeout=2)