Add interval-aware polling and a 'cycle' command for scheduling
- poll_due_sources(): polls only sources whose last successful poll is older than their poll_interval_minutes (or never polled), finally giving that config field meaning. - classify gains only_unclassified to spend the LLM solely on new (heuristic) articles, so a frequent scheduled run stays cheap. - 'cycle' command runs poll-due -> classify-new -> rebuild-today's-brief, with each step non-fatal so a down model endpoint or empty day never aborts it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+33
-6
@@ -28,13 +28,40 @@ class FeedItem:
|
||||
|
||||
|
||||
def poll_all_sources(conn: sqlite3.Connection, limit: int | None = None) -> dict:
|
||||
query = """
|
||||
SELECT *
|
||||
FROM sources
|
||||
WHERE active = 1
|
||||
ORDER BY id
|
||||
return _poll_rows(conn, conn.execute(
|
||||
"SELECT * FROM sources WHERE active = 1 ORDER BY id"
|
||||
).fetchall(), limit)
|
||||
|
||||
|
||||
def poll_due_sources(conn: sqlite3.Connection, limit: int | None = None) -> dict:
|
||||
"""Poll only active sources whose last successful poll is older than their
|
||||
poll_interval_minutes (or that have never been polled successfully).
|
||||
|
||||
This is what makes poll_interval_minutes meaningful and lets a scheduler run
|
||||
frequently without re-hitting feeds that are not yet due.
|
||||
"""
|
||||
rows = conn.execute(query).fetchall()
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT s.*
|
||||
FROM sources s
|
||||
WHERE s.active = 1
|
||||
AND (
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM ingest_runs r
|
||||
WHERE r.source_id = s.id AND r.status = 'ok'
|
||||
)
|
||||
OR (
|
||||
SELECT MAX(r.finished_at) FROM ingest_runs r
|
||||
WHERE r.source_id = s.id AND r.status = 'ok'
|
||||
) <= datetime('now', '-' || s.poll_interval_minutes || ' minutes')
|
||||
)
|
||||
ORDER BY s.id
|
||||
"""
|
||||
).fetchall()
|
||||
return _poll_rows(conn, rows, limit)
|
||||
|
||||
|
||||
def _poll_rows(conn: sqlite3.Connection, rows: list[sqlite3.Row], limit: int | None) -> dict:
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user