Source Retire lifecycle (Phase 1: status + content_visible, active mirrored)

Per Codex's plan — introduce a lifecycle without a risky "change the source of truth everywhere" moment. * Schema: sources.status (active|paused|retired) + content_visible; migration backfills status from active (active=1→active, else paused), content_visible=1. * `active` is kept as a SYNCED MIRROR: status active→active=1, paused/retired→0, so the scheduler/CLI/legacy code keep working unchanged. * Retire stops polling but keeps articles visible (non-destructive). Hiding is a separate, reversible lever: content_visible=0 drops a source's articles from the public feed + brief (read AND build), behind a confirm. Personal saved/ history are untouched. * API: /sources/{id}/status (validates, mirrors active) + /visibility, replacing /active. source_health returns status + content_visible. * Admin: status column (active/paused/retired + "hidden"), Retired filter, Pause/Resume · Retire/Restore · Hide/Show actions. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-09 09:58:15 -04:00
parent ba92c0a04b
commit 9ed817c051
6 changed files with 124 additions and 49 deletions
@@ -61,28 +61,40 @@
  }

  let sources = $derived(stats?.sources ?? []);
-  let healthy = $derived(sources.filter((s) => s.active && !s.failures && !s.review_flag).length);
-  let resting = $derived(sources.filter((s) => s.active && s.failures > 0).length);
+  // status falls back to the legacy active flag during the migration window.
+  const sstatus = (s) => s.status || (s.active ? 'active' : 'paused');
+  let healthy = $derived(sources.filter((s) => sstatus(s) === 'active' && !s.failures && !s.review_flag).length);
+  let resting = $derived(sources.filter((s) => sstatus(s) === 'active' && s.failures > 0).length);
  let flagged = $derived(sources.filter((s) => s.review_flag).length);
-  let paused = $derived(sources.filter((s) => !s.active).length);
+  let paused = $derived(sources.filter((s) => sstatus(s) === 'paused').length);
+  let retired = $derived(sources.filter((s) => sstatus(s) === 'retired').length);

  // Sources table filter.
  let srcFilter = $state('all');
  let shownSources = $derived(
-    srcFilter === 'healthy' ? sources.filter((s) => s.active && !s.failures && !s.review_flag)
-      : srcFilter === 'resting' ? sources.filter((s) => s.active && s.failures > 0)
+    srcFilter === 'healthy' ? sources.filter((s) => sstatus(s) === 'active' && !s.failures && !s.review_flag)
+      : srcFilter === 'resting' ? sources.filter((s) => sstatus(s) === 'active' && s.failures > 0)
        : srcFilter === 'flagged' ? sources.filter((s) => s.review_flag)
-          : srcFilter === 'paused' ? sources.filter((s) => !s.active)
-            : sources
+          : srcFilter === 'paused' ? sources.filter((s) => sstatus(s) === 'paused')
+            : srcFilter === 'retired' ? sources.filter((s) => sstatus(s) === 'retired')
+              : sources
  );

-  // Pause/resume (stops polling only; existing articles stay live) + flag/clear.
-  // Optimistic against the deep-reactive stats; revert on failure.
-  async function toggleActive(s) {
-    const next = !s.active;
-    s.active = next ? 1 : 0;
-    try { await postJSON(`/api/admin/sources/${s.id}/active`, { active: next }); }
-    catch { s.active = next ? 0 : 1; }
+  // Lifecycle (status: active/paused/retired) keeps `active` mirrored server-side;
+  // visibility hides existing articles. Both optimistic with revert-on-failure.
+  async function setStatus(s, status) {
+    const prev = { status: s.status, active: s.active };
+    s.status = status; s.active = status === 'active' ? 1 : 0;
+    try { await postJSON(`/api/admin/sources/${s.id}/status`, { status }); }
+    catch { s.status = prev.status; s.active = prev.active; }
+  }
+  async function toggleVisible(s) {
+    const next = !s.content_visible;
+    if (!next && !confirm(`Hide all of “${s.name}”’s articles from the public feed and brief?\n(Reversible — nothing is deleted.)`)) return;
+    const prev = s.content_visible;
+    s.content_visible = next ? 1 : 0;
+    try { await postJSON(`/api/admin/sources/${s.id}/visibility`, { visible: next }); }
+    catch { s.content_visible = prev; }
  }
  // Flagging opens a small inline popover for the reason; clearing is immediate.
  let flagging = $state(null); // the source being flagged
@@ -327,9 +339,9 @@

    {:else if section === 'sources'}
      <h2>Sources</h2>
-      <p class="sub2">{healthy} healthy · {resting} resting · {flagged} flagged · {paused} paused · {sources.length} total</p>
+      <p class="sub2">{healthy} healthy · {resting} resting · {flagged} flagged · {paused} paused · {retired} retired · {sources.length} total</p>
      <div class="filterchips">
-        {#each [['all', 'All'], ['healthy', 'Healthy'], ['resting', 'Resting'], ['flagged', 'Flagged'], ['paused', 'Paused']] as [key, label] (key)}
+        {#each [['all', 'All'], ['healthy', 'Healthy'], ['resting', 'Resting'], ['flagged', 'Flagged'], ['paused', 'Paused'], ['retired', 'Retired']] as [key, label] (key)}
          <button class="chip" class:on={srcFilter === key} onclick={() => (srcFilter = key)}>{label}</button>
        {/each}
      </div>
@@ -343,7 +355,8 @@
          </thead>
          <tbody>
            {#each shownSources as s (s.id)}
-              <tr class:warn={s.active && s.failures > 0} class:flag={s.review_flag} class:paused={!s.active}>
+              {@const st = sstatus(s)}
+              <tr class:warn={st === 'active' && s.failures > 0} class:flag={s.review_flag} class:paused={st !== 'active'}>
                <td class="sname">
                  {s.name}{#if s.category}<span class="cat">{s.category}</span>{/if}
                  {#if s.review_flag && s.review_reason}<span class="rr">“{s.review_reason}”</span>{/if}
@@ -352,17 +365,25 @@
                <td class="num">{s.acceptance_rate != null ? s.acceptance_rate + '%' : '—'}</td>
                <td class="num" title={s.accepted_dup_rate != null ? `${s.accepted_dup_rate}% of accepted were duplicates` : ''}>{s.duplicate_rate != null ? s.duplicate_rate + '%' : '—'}</td>
                <td class="dim">{s.last_success_at ? fwhen(s.last_success_at) : '—'}</td>
-                <td class="dim">{s.active ? fwhen(s.next_due_at) : '—'}</td>
+                <td class="dim">{st === 'active' ? fwhen(s.next_due_at) : '—'}</td>
                <td class="num">{s.failures || ''}</td>
                <td class="status">
-                  {#if !s.active}<span class="paustxt">paused</span>
+                  {#if st === 'retired'}<span class="paustxt">retired</span>
+                  {:else if st === 'paused'}<span class="paustxt">paused</span>
                  {:else if s.failures > 0}<span class="bad" title={s.last_error || ''}>⚠ resting{#if s.review_flag} · review{/if}</span>
                  {:else if s.review_flag}<span class="flagtxt">⚑ review</span>
                  {:else}<span class="good">✓ ok</span>{/if}
+                  {#if !s.content_visible}<span class="hidtxt" title="Articles hidden from the public feed">· hidden</span>{/if}
                </td>
                <td class="rowactions">
-                  <button class="act" onclick={() => toggleActive(s)}>{s.active ? 'Pause' : 'Resume'}</button>
+                  {#if st === 'retired'}
+                    <button class="act" onclick={() => setStatus(s, 'active')}>Restore</button>
+                  {:else}
+                    <button class="act" onclick={() => setStatus(s, st === 'active' ? 'paused' : 'active')}>{st === 'active' ? 'Pause' : 'Resume'}</button>
+                    <button class="act" onclick={() => setStatus(s, 'retired')}>Retire</button>
+                  {/if}
                  <button class="act" onclick={() => (s.review_flag ? clearReview(s) : openFlag(s))}>{s.review_flag ? 'Clear' : 'Flag'}</button>
+                  <button class="act" onclick={() => toggleVisible(s)}>{s.content_visible ? 'Hide' : 'Show'}</button>
                </td>
              </tr>
            {/each}
@@ -663,6 +684,7 @@
  .srctable .status .flagtxt { color: var(--accent-deep); }
  .srctable .status .good { color: #3f7048; }
  .srctable .status .paustxt { color: var(--muted); font-style: italic; }
+  .srctable .status .hidtxt { color: #9a3b3b; font-size: 0.78rem; }
  .srctable .rowactions { white-space: nowrap; }
  .srctable .rowactions .act {
    background: none; border: 1px solid var(--line); color: var(--accent-deep);
@@ -386,8 +386,12 @@ class FeedbackReplyBody(BaseModel):
    html: str = ""  # raw editor HTML — sanitized server-side before use


-class SourceActiveBody(BaseModel):
-    active: bool = True
+class SourceStatusBody(BaseModel):
+    status: str = "active"  # active | paused | retired
+
+
+class SourceVisibilityBody(BaseModel):
+    visible: bool = True


 class SourceReviewBody(BaseModel):
@@ -882,17 +886,37 @@ def create_app() -> FastAPI:
            ).fetchone()
        return {"ok": True, "reply": dict(reply)}

-    @app.post("/api/admin/sources/{sid}/active")
-    def admin_source_active(sid: int, body: SourceActiveBody, request: Request) -> dict:
-        # Pause/resume a feed. Pausing only stops future polling — its already-
-        # accepted articles stay live (the feed query doesn't filter on active).
+    @app.post("/api/admin/sources/{sid}/status")
+    def admin_source_status(sid: int, body: SourceStatusBody, request: Request) -> dict:
+        # Lifecycle: active | paused | retired. Stops/resumes polling only —
+        # existing articles stay visible (use /visibility to hide). `active` is
+        # kept mirrored so the scheduler/CLI keep working off the legacy flag.
+        if body.status not in ("active", "paused", "retired"):
+            raise HTTPException(status_code=422, detail="invalid status")
+        active = 1 if body.status == "active" else 0
        with get_conn() as conn:
            _require_admin(conn, request)
-            cur = conn.execute("UPDATE sources SET active = ? WHERE id = ?", (1 if body.active else 0, sid))
+            cur = conn.execute(
+                "UPDATE sources SET status = ?, active = ? WHERE id = ?", (body.status, active, sid)
+            )
            if cur.rowcount == 0:
                raise HTTPException(status_code=404, detail="source not found")
            conn.commit()
-        return {"ok": True, "active": body.active}
+        return {"ok": True, "status": body.status, "active": active}
+
+    @app.post("/api/admin/sources/{sid}/visibility")
+    def admin_source_visibility(sid: int, body: SourceVisibilityBody, request: Request) -> dict:
+        # Pull a source's existing articles out of (or back into) the public feed
+        # and brief. Reversible; never deletes anything.
+        with get_conn() as conn:
+            _require_admin(conn, request)
+            cur = conn.execute(
+                "UPDATE sources SET content_visible = ? WHERE id = ?", (1 if body.visible else 0, sid)
+            )
+            if cur.rowcount == 0:
+                raise HTTPException(status_code=404, detail="source not found")
+            conn.commit()
+        return {"ok": True, "visible": body.visible}

    @app.post("/api/admin/sources/{sid}/review")
    def admin_source_review(sid: int, body: SourceReviewBody, request: Request) -> dict:
@@ -139,6 +139,7 @@ def _candidate_articles(
        JOIN sources src ON src.id = a.source_id
        JOIN article_scores s ON s.article_id = a.id
        WHERE s.accepted = 1
+          AND src.content_visible = 1
          AND a.duplicate_of IS NULL
          AND date(COALESCE(a.published_at, a.discovered_at)) <= date(?)
          AND date(COALESCE(a.published_at, a.discovered_at)) > date(?, '-' || ? || ' days')
@@ -17,6 +17,8 @@ CREATE TABLE IF NOT EXISTS sources (
    trust_score INTEGER NOT NULL DEFAULT 5,
    pr_risk_score INTEGER NOT NULL DEFAULT 3,
    active INTEGER NOT NULL DEFAULT 1,
+    status TEXT NOT NULL DEFAULT 'active',
+    content_visible INTEGER NOT NULL DEFAULT 1,
    poll_interval_minutes INTEGER NOT NULL DEFAULT 60,
    notes TEXT,
    last_success_at TEXT,
@@ -317,6 +319,14 @@ def _migrate(conn: sqlite3.Connection) -> None:
        if column not in source_cols:
            conn.execute(f"ALTER TABLE sources ADD COLUMN {column} {decl}")

+    # Lifecycle: status (active/paused/retired) + content_visible. `active` is
+    # kept as a synced mirror so legacy code (scheduler/CLI) keeps working.
+    if "status" not in source_cols:
+        conn.execute("ALTER TABLE sources ADD COLUMN status TEXT NOT NULL DEFAULT 'active'")
+        conn.execute("UPDATE sources SET status = CASE WHEN active = 1 THEN 'active' ELSE 'paused' END")
+    if "content_visible" not in source_cols:
+        conn.execute("ALTER TABLE sources ADD COLUMN content_visible INTEGER NOT NULL DEFAULT 1")
+
    # feedback.read_at (admin inbox read/unread) added later.
    fb_cols = {row["name"] for row in conn.execute("PRAGMA table_info(feedback)")}
    if fb_cols and "read_at" not in fb_cols:
@@ -72,7 +72,7 @@ def feed(
    items (e.g. 'discovery' for a Wonder lane) fall outside any over-fetch window.
    Word-boundary avoid-terms remain a Python pass on the caller side.
    """
-    clauses = ["a.duplicate_of IS NULL"]
+    clauses = ["a.duplicate_of IS NULL", "src.content_visible = 1"]
    params: list = []
    if accepted_only:
        clauses.append("s.accepted = 1")
@@ -157,7 +157,7 @@ def brief(conn: sqlite3.Connection, brief_date: str | None = None, limit: int =
        JOIN articles a ON a.id = bi.article_id
        JOIN sources src ON src.id = a.source_id
        LEFT JOIN article_scores s ON s.article_id = a.id
-        WHERE b.brief_date = ?
+        WHERE b.brief_date = ? AND src.content_visible = 1
        ORDER BY bi.rank
        LIMIT ?
        """,
@@ -295,6 +295,7 @@ def source_health(conn: sqlite3.Connection) -> list[dict]:
        """
        SELECT
            s.id, s.name, s.default_category AS category, s.active,
+            s.status, s.content_visible,
            s.consecutive_failures AS failures, s.review_flag, s.review_reason,
            s.poll_interval_minutes AS interval_minutes,
            s.last_success_at, s.last_error_at, substr(s.last_error, 1, 160) AS last_error,
@@ -57,31 +57,48 @@ def test_admin_stats_shape(tmp_path, monkeypatch):
    assert any(g["tag"] == "science" for g in stats["top_groupings"])


-def test_source_actions_pause_resume_flag(tmp_path, monkeypatch):
+def _src(tc, sid=1):
+    return next(s for s in tc.get("/api/admin/stats").json()["sources"] if s["id"] == sid)
+
+
+def test_source_lifecycle_status_and_visibility(tmp_path, monkeypatch):
    app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com")
    tc = _signin(app, api, "boss@x.com")
-    # pause
-    assert tc.post("/api/admin/sources/1/active", json={"active": False}).json()["active"] is False
-    st = tc.get("/api/admin/stats").json()
-    src = next(s for s in st["sources"] if s["id"] == 1)
-    assert src["active"] == 0  # paused source still listed (manageable)
-    # resume
-    tc.post("/api/admin/sources/1/active", json={"active": True})
-    # flag with reason, then clear
-    tc.post("/api/admin/sources/1/review", json={"flag": True, "reason": "spammy lately"})
-    src = next(s for s in tc.get("/api/admin/stats").json()["sources"] if s["id"] == 1)
-    assert src["review_flag"] == 1 and src["review_reason"] == "spammy lately"
-    tc.post("/api/admin/sources/1/review", json={"flag": False})
-    src = next(s for s in tc.get("/api/admin/stats").json()["sources"] if s["id"] == 1)
-    assert src["review_flag"] == 0 and src["review_reason"] is None
-    # 404 on missing
-    assert tc.post("/api/admin/sources/999/active", json={"active": False}).status_code == 404
+    # pause → status paused + active mirror 0
+    assert tc.post("/api/admin/sources/1/status", json={"status": "paused"}).json()["active"] == 0
+    assert _src(tc)["status"] == "paused" and _src(tc)["active"] == 0
+    # retire → status retired, still active=0, articles stay visible
+    tc.post("/api/admin/sources/1/status", json={"status": "retired"})
+    assert _src(tc)["status"] == "retired" and _src(tc)["active"] == 0
+    assert _src(tc)["content_visible"] == 1  # retire does NOT hide content
+    # restore → active again, mirror 1
+    tc.post("/api/admin/sources/1/status", json={"status": "active"})
+    assert _src(tc)["status"] == "active" and _src(tc)["active"] == 1
+    # hide content → feed excludes it; show → back
+    tc.post("/api/admin/sources/1/visibility", json={"visible": False})
+    assert _src(tc)["content_visible"] == 0
+    from goodnews import queries
+    import sqlite3, os
+    c = sqlite3.connect(os.environ["GOODNEWS_DB"]); c.row_factory = sqlite3.Row
+    assert queries.feed(c) == []  # hidden source's article drops out of the feed
+    tc.post("/api/admin/sources/1/visibility", json={"visible": True})
+    c2 = sqlite3.connect(os.environ["GOODNEWS_DB"]); c2.row_factory = sqlite3.Row
+    assert len(queries.feed(c2)) == 1  # back in the feed
+    # validation + 404
+    assert tc.post("/api/admin/sources/1/status", json={"status": "bogus"}).status_code == 422
+    assert tc.post("/api/admin/sources/999/status", json={"status": "paused"}).status_code == 404


-def test_source_actions_gated(tmp_path, monkeypatch):
+def test_source_flag_and_gating(tmp_path, monkeypatch):
    app, api = _make(tmp_path, monkeypatch, admin_email="boss@x.com")
+    tc = _signin(app, api, "boss@x.com")
+    tc.post("/api/admin/sources/1/review", json={"flag": True, "reason": "spammy lately"})
+    assert _src(tc)["review_flag"] == 1 and _src(tc)["review_reason"] == "spammy lately"
+    tc.post("/api/admin/sources/1/review", json={"flag": False})
+    assert _src(tc)["review_flag"] == 0 and _src(tc)["review_reason"] is None
    anon = TestClient(app)
-    assert anon.post("/api/admin/sources/1/active", json={"active": False}).status_code == 401
+    assert anon.post("/api/admin/sources/1/status", json={"status": "paused"}).status_code == 401
+    assert anon.post("/api/admin/sources/1/visibility", json={"visible": False}).status_code == 401
    assert anon.post("/api/admin/sources/1/review", json={"flag": True}).status_code == 401