Add interval-aware polling and a 'cycle' command for scheduling
- poll_due_sources(): polls only sources whose last successful poll is older than their poll_interval_minutes (or never polled), finally giving that config field meaning. - classify gains only_unclassified to spend the LLM solely on new (heuristic) articles, so a frequent scheduled run stays cheap. - 'cycle' command runs poll-due -> classify-new -> rebuild-today's-brief, with each step non-fatal so a down model endpoint or empty day never aborts it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+46
-1
@@ -3,11 +3,12 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from .briefs import build_daily_brief, show_brief
|
||||
from .db import connect, init_db
|
||||
from .feeds import poll_all_sources, poll_source
|
||||
from .feeds import poll_all_sources, poll_due_sources, poll_source
|
||||
from .llm import LocalModelClient, classify_articles
|
||||
from .scoring import score_article
|
||||
from .sources import load_sources, upsert_sources
|
||||
@@ -59,6 +60,16 @@ def main() -> None:
|
||||
classify_parser.add_argument("--base-url", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
|
||||
classify_parser.add_argument("--model", help="Local model name")
|
||||
|
||||
cycle_parser = subparsers.add_parser(
|
||||
"cycle", help="Poll due sources, classify new articles, rebuild today's brief (for scheduling)"
|
||||
)
|
||||
cycle_parser.add_argument("--classify-limit", type=int, default=40)
|
||||
cycle_parser.add_argument("--no-classify", action="store_true", help="Skip the LLM classify step")
|
||||
cycle_parser.add_argument("--no-brief", action="store_true", help="Skip rebuilding today's brief")
|
||||
cycle_parser.add_argument("--force", action="store_true", help="Poll all active sources, ignoring intervals")
|
||||
cycle_parser.add_argument("--base-url", help="OpenAI-compatible base URL for classify")
|
||||
cycle_parser.add_argument("--model", help="Local model name for classify")
|
||||
|
||||
check_llm_parser = subparsers.add_parser("check-llm", help="Check local OpenAI-compatible model endpoint")
|
||||
check_llm_parser.add_argument("--base-url", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
|
||||
check_llm_parser.add_argument("--model", help="Expected local model name")
|
||||
@@ -135,6 +146,8 @@ def main() -> None:
|
||||
print(f" {scores['reason_text']}")
|
||||
if args.dry_run:
|
||||
print("Dry run only; database was not updated.")
|
||||
elif args.command == "cycle":
|
||||
run_cycle(conn, args)
|
||||
elif args.command == "check-llm":
|
||||
client = llm_client_from_args(args)
|
||||
try:
|
||||
@@ -201,6 +214,38 @@ def list_recent(conn: sqlite3.Connection, limit: int, accepted_only: bool) -> No
|
||||
print(f" {row['canonical_url']}")
|
||||
|
||||
|
||||
def run_cycle(conn: sqlite3.Connection, args: argparse.Namespace) -> None:
|
||||
"""One end-to-end pass for a scheduler: poll due sources, classify the new
|
||||
arrivals, rebuild today's brief. Each step is independent and non-fatal so a
|
||||
down model endpoint or empty day never aborts the whole cycle.
|
||||
"""
|
||||
init_db(conn)
|
||||
|
||||
if args.force:
|
||||
poll_result = poll_all_sources(conn)
|
||||
else:
|
||||
poll_result = poll_due_sources(conn)
|
||||
print(f"poll: {_format_result(poll_result)}")
|
||||
|
||||
if not args.no_classify:
|
||||
client = llm_client_from_args(args)
|
||||
try:
|
||||
results = classify_articles(
|
||||
conn, client, limit=args.classify_limit, include_rejected=True, only_unclassified=True
|
||||
)
|
||||
print(f"classify: {len(results)} new article(s) scored by {client.model}")
|
||||
except Exception as exc: # endpoint down, timeout, etc. — keep going
|
||||
print(f"classify: skipped ({exc})")
|
||||
|
||||
if not args.no_brief:
|
||||
today = date.today().isoformat()
|
||||
try:
|
||||
brief_id = build_daily_brief(conn, brief_date=today, limit=5, replace=True)
|
||||
print(f"brief: rebuilt {today} (id {brief_id})")
|
||||
except Exception as exc:
|
||||
print(f"brief: skipped ({exc})")
|
||||
|
||||
|
||||
def serve(args: argparse.Namespace) -> None:
|
||||
try:
|
||||
import uvicorn
|
||||
|
||||
Reference in New Issue
Block a user