thejayman77
38abc26ddd
Per Codex's spec — a publisher saying "slow down" shouldn't make a feed look broken, but repeated 429s stay visible via last_success_at / stale-source. * Schema: sources.retry_after_at (nullable) + migration. * feeds.parse_retry_after: delta-seconds OR HTTP-date → UTC stamp; ignores invalid/negative/past; caps at now + MAX_BACKOFF_MINUTES. * fetch_feed raises RateLimited (carrying the parsed time) on a 429. * poll_source: on 429 set retry_after_at + last_error, status='rate_limited', and do NOT increment consecutive_failures; on success clear retry_after_at; non-429 failures unchanged. * due_source_rows requires BOTH the streak backoff elapsed AND retry_after_at passed (i.e. the later of the two). * Admin: source_health returns retry_after_at; status reads "rate-limited · rests until …" rather than "failed/resting". Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
66 lines
2.7 KiB
Python
66 lines
2.7 KiB
Python
from datetime import UTC, datetime, timedelta
|
|
|
|
from goodnews import feeds
|
|
from goodnews.db import connect, init_db
|
|
|
|
|
|
NOW = datetime(2026, 6, 9, 12, 0, 0, tzinfo=UTC)
|
|
|
|
|
|
def test_parse_delta_seconds():
|
|
assert feeds.parse_retry_after("120", now=NOW) == "2026-06-09 12:02:00"
|
|
|
|
|
|
def test_parse_http_date():
|
|
assert feeds.parse_retry_after("Tue, 09 Jun 2026 12:30:00 GMT", now=NOW) == "2026-06-09 12:30:00"
|
|
|
|
|
|
def test_parse_ignores_invalid_and_past():
|
|
assert feeds.parse_retry_after("", now=NOW) is None
|
|
assert feeds.parse_retry_after("soon", now=NOW) is None
|
|
assert feeds.parse_retry_after("-30", now=NOW) is None # negative
|
|
assert feeds.parse_retry_after("Tue, 09 Jun 2020 00:00:00 GMT", now=NOW) is None # past
|
|
|
|
|
|
def test_parse_caps_at_max_backoff():
|
|
capped = feeds.parse_retry_after(str(60 * 60 * 24 * 30), now=NOW) # 30 days
|
|
assert capped == (NOW + timedelta(minutes=feeds.MAX_BACKOFF_MINUTES)).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def _src(c):
|
|
c.execute("INSERT INTO sources (id,name,feed_url,active) VALUES (1,'S','http://s/f',1)")
|
|
c.commit()
|
|
return c.execute("SELECT * FROM sources WHERE id=1").fetchone()
|
|
|
|
|
|
def test_429_sets_retry_after_without_streak(monkeypatch):
|
|
c = connect(":memory:"); init_db(c); src = _src(c)
|
|
def boom(url, timeout=20):
|
|
raise feeds.RateLimited("HTTP 429", retry_after_at="2030-01-01 00:00:00")
|
|
monkeypatch.setattr(feeds, "fetch_feed", boom)
|
|
res = feeds.poll_source(c, src)
|
|
assert res["status"] == "rate_limited"
|
|
row = c.execute("SELECT consecutive_failures, retry_after_at FROM sources WHERE id=1").fetchone()
|
|
assert row["consecutive_failures"] == 0 # NOT inflated
|
|
assert row["retry_after_at"] == "2030-01-01 00:00:00"
|
|
# and it's not due while resting
|
|
assert [s["id"] for s in feeds.due_source_rows(c)] == []
|
|
|
|
|
|
def test_success_clears_retry_after(monkeypatch):
|
|
c = connect(":memory:"); init_db(c); src = _src(c)
|
|
c.execute("UPDATE sources SET retry_after_at='2030-01-01 00:00:00', consecutive_failures=2 WHERE id=1")
|
|
c.commit()
|
|
monkeypatch.setattr(feeds, "fetch_feed", lambda url, timeout=20: b"<rss><channel></channel></rss>")
|
|
feeds.poll_source(c, src)
|
|
row = c.execute("SELECT consecutive_failures, retry_after_at FROM sources WHERE id=1").fetchone()
|
|
assert row["retry_after_at"] is None and row["consecutive_failures"] == 0
|
|
|
|
|
|
def test_non_429_failure_still_increments_streak(monkeypatch):
|
|
c = connect(":memory:"); init_db(c); src = _src(c)
|
|
monkeypatch.setattr(feeds, "fetch_feed", lambda url, timeout=20: (_ for _ in ()).throw(RuntimeError("HTTP 500")))
|
|
res = feeds.poll_source(c, src)
|
|
assert res["status"] == "failed"
|
|
assert c.execute("SELECT consecutive_failures FROM sources WHERE id=1").fetchone()[0] == 1
|