Honor Retry-After on HTTP 429 (polite rest, not a failure)

Per Codex's spec — a publisher saying "slow down" shouldn't make a feed look
broken, but repeated 429s stay visible via last_success_at / stale-source.

* Schema: sources.retry_after_at (nullable) + migration.
* feeds.parse_retry_after: delta-seconds OR HTTP-date → UTC stamp; ignores
  invalid/negative/past; caps at now + MAX_BACKOFF_MINUTES.
* fetch_feed raises RateLimited (carrying the parsed time) on a 429.
* poll_source: on 429 set retry_after_at + last_error, status='rate_limited',
  and do NOT increment consecutive_failures; on success clear retry_after_at;
  non-429 failures unchanged.
* due_source_rows requires BOTH the streak backoff elapsed AND retry_after_at
  passed (i.e. the later of the two).
* Admin: source_health returns retry_after_at; status reads
  "rate-limited · rests until …" rather than "failed/resting".

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jay
2026-06-09 10:47:40 -04:00
parent 373571b476
commit 38abc26ddd
5 changed files with 146 additions and 4 deletions
+65
View File
@@ -0,0 +1,65 @@
from datetime import UTC, datetime, timedelta
from goodnews import feeds
from goodnews.db import connect, init_db
NOW = datetime(2026, 6, 9, 12, 0, 0, tzinfo=UTC)
def test_parse_delta_seconds():
assert feeds.parse_retry_after("120", now=NOW) == "2026-06-09 12:02:00"
def test_parse_http_date():
assert feeds.parse_retry_after("Tue, 09 Jun 2026 12:30:00 GMT", now=NOW) == "2026-06-09 12:30:00"
def test_parse_ignores_invalid_and_past():
assert feeds.parse_retry_after("", now=NOW) is None
assert feeds.parse_retry_after("soon", now=NOW) is None
assert feeds.parse_retry_after("-30", now=NOW) is None # negative
assert feeds.parse_retry_after("Tue, 09 Jun 2020 00:00:00 GMT", now=NOW) is None # past
def test_parse_caps_at_max_backoff():
capped = feeds.parse_retry_after(str(60 * 60 * 24 * 30), now=NOW) # 30 days
assert capped == (NOW + timedelta(minutes=feeds.MAX_BACKOFF_MINUTES)).strftime("%Y-%m-%d %H:%M:%S")
def _src(c):
c.execute("INSERT INTO sources (id,name,feed_url,active) VALUES (1,'S','http://s/f',1)")
c.commit()
return c.execute("SELECT * FROM sources WHERE id=1").fetchone()
def test_429_sets_retry_after_without_streak(monkeypatch):
c = connect(":memory:"); init_db(c); src = _src(c)
def boom(url, timeout=20):
raise feeds.RateLimited("HTTP 429", retry_after_at="2030-01-01 00:00:00")
monkeypatch.setattr(feeds, "fetch_feed", boom)
res = feeds.poll_source(c, src)
assert res["status"] == "rate_limited"
row = c.execute("SELECT consecutive_failures, retry_after_at FROM sources WHERE id=1").fetchone()
assert row["consecutive_failures"] == 0 # NOT inflated
assert row["retry_after_at"] == "2030-01-01 00:00:00"
# and it's not due while resting
assert [s["id"] for s in feeds.due_source_rows(c)] == []
def test_success_clears_retry_after(monkeypatch):
c = connect(":memory:"); init_db(c); src = _src(c)
c.execute("UPDATE sources SET retry_after_at='2030-01-01 00:00:00', consecutive_failures=2 WHERE id=1")
c.commit()
monkeypatch.setattr(feeds, "fetch_feed", lambda url, timeout=20: b"<rss><channel></channel></rss>")
feeds.poll_source(c, src)
row = c.execute("SELECT consecutive_failures, retry_after_at FROM sources WHERE id=1").fetchone()
assert row["retry_after_at"] is None and row["consecutive_failures"] == 0
def test_non_429_failure_still_increments_streak(monkeypatch):
c = connect(":memory:"); init_db(c); src = _src(c)
monkeypatch.setattr(feeds, "fetch_feed", lambda url, timeout=20: (_ for _ in ()).throw(RuntimeError("HTTP 500")))
res = feeds.poll_source(c, src)
assert res["status"] == "failed"
assert c.execute("SELECT consecutive_failures FROM sources WHERE id=1").fetchone()[0] == 1