Durability pass: tests, clearer diversity/classify behavior, Calm Filters foundation
- Add pytest suite (34 tests) covering scoring thresholds, dedup clustering + representative selection + time window, brief source/category diversity, avoid-term phrase matching, and text canonicalization/truncation. - Rewrite _select_diverse with an explicit, tested contract (best-first, one per source, backfill, then inject a second category by evicting the lowest-ranked pick). - classify_articles now returns attempted/succeeded/skipped (ClassifyReport) so silent model failures are visible in both the cycle and classify output. - Fix clean_text truncation to stay within max_len (ellipsis no longer overshoots). - New filters.py: canonical FilterPrefs shape (include/mute topics+flavors, avoid_terms, pauses) and pure word/phrase-boundary matching engine seeding Calm Filters. Not yet wired into the API. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+12
-2
@@ -220,6 +220,14 @@ class LocalModelClient:
|
||||
return parse_classifier_json(content)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassifyReport:
|
||||
results: list[tuple[int, dict]]
|
||||
attempted: int
|
||||
succeeded: int
|
||||
skipped: int
|
||||
|
||||
|
||||
def classify_articles(
|
||||
conn: sqlite3.Connection,
|
||||
client: LocalModelClient,
|
||||
@@ -228,17 +236,19 @@ def classify_articles(
|
||||
dry_run: bool = False,
|
||||
only_unclassified: bool = False,
|
||||
progress: "Callable[[int, int, int], None] | None" = None,
|
||||
) -> list[tuple[int, dict]]:
|
||||
) -> ClassifyReport:
|
||||
rows = _classification_candidates(
|
||||
conn, limit=limit, include_rejected=include_rejected, only_unclassified=only_unclassified
|
||||
)
|
||||
results = []
|
||||
skipped = 0
|
||||
for index, row in enumerate(rows, start=1):
|
||||
try:
|
||||
scores = client.classify(row)
|
||||
except RuntimeError as exc:
|
||||
# One slow/failed article (timeout, bad response) shouldn't sink the
|
||||
# whole batch or discard work already committed. Skip and continue.
|
||||
skipped += 1
|
||||
print(f"[{row['id']}] skipped: {exc}")
|
||||
continue
|
||||
scores = normalize_scores(scores, model_name=client.model)
|
||||
@@ -248,7 +258,7 @@ def classify_articles(
|
||||
conn.commit()
|
||||
if progress is not None:
|
||||
progress(index, len(rows), row["id"])
|
||||
return results
|
||||
return ClassifyReport(results=results, attempted=len(rows), succeeded=len(results), skipped=skipped)
|
||||
|
||||
|
||||
def parse_classifier_json(content: str) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user