"""On-demand, cached article summaries. Lazy: generated only for articles that actually get shared/viewed, then cached in article_summaries forever. We fetch the article text *transiently* and ask the local LLM for a short, ORIGINAL summary in our own words. We store only that summary — never the publisher's article body — and the page always credits and links to the source. """ from __future__ import annotations import re import sqlite3 import urllib.error import urllib.request from urllib.parse import urljoin, urlsplit from .enrich import USER_AGENT, _NoRedirect, _host_is_public from .llm import LocalModelClient _FETCH_BYTES = 600_000 _FETCH_TIMEOUT = 8 _MAX_REDIRECTS = 3 _DROP = re.compile(rb"<(script|style|noscript|template|svg)[^>]*>.*?", re.IGNORECASE | re.DOTALL) _TAGS = re.compile(rb"<[^>]+>") _WS = re.compile(r"\s+") _SYSTEM = ( "You write a short, original summary of a news story for a calm, constructive news " "site. Use your own words — never copy sentences from the source. 2 to 4 plain " "sentences: what happened and why it's encouraging. No preamble, no markdown, no " "headline — just the summary." ) def _fetch_text(url: str) -> str: """SSRF-guarded fetch of an article page, reduced to plain text (capped).""" opener = urllib.request.build_opener(_NoRedirect) for _ in range(_MAX_REDIRECTS + 1): if not url: return "" parts = urlsplit(url) if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname): return "" request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"}) try: response = opener.open(request, timeout=_FETCH_TIMEOUT) except (urllib.error.URLError, OSError, ValueError): return "" status = getattr(response, "status", 200) or 200 if status in (301, 302, 303, 307, 308): location = response.headers.get("Location") response.close() if not location: return "" url = urljoin(url, location) continue if "html" not in response.headers.get("Content-Type", "").lower(): response.close() return "" try: raw = response.read(_FETCH_BYTES) finally: response.close() raw = _DROP.sub(b" ", raw) text = _TAGS.sub(b" ", raw).decode("utf-8", "replace") return _WS.sub(" ", text).strip()[:4000] return "" def summarize_article(client: LocalModelClient, title: str, snippet: str, body_text: str) -> str: material = (body_text or snippet or title or "")[:4000] user = f"Title: {title}\n\nArticle text:\n{material}\n\nWrite the summary." messages = [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": user}] resp = client._chat(client._build_payload(messages, None)) content = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or "" return content.strip()[:1200] def get_summary(conn: sqlite3.Connection, article_id: int) -> str | None: row = conn.execute( "SELECT summary FROM article_summaries WHERE article_id = ?", (article_id,) ).fetchone() return row["summary"] if row else None def generate_summary(conn: sqlite3.Connection, article_id: int, client: LocalModelClient | None = None) -> str | None: """Generate + cache a summary for one article. Returns it, or None if skipped.""" if get_summary(conn, article_id): return get_summary(conn, article_id) # already cached row = conn.execute( "SELECT a.title, a.description, a.canonical_url, a.duplicate_of, s.accepted " "FROM articles a LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?", (article_id,), ).fetchone() if not row or row["duplicate_of"] is not None or not row["accepted"]: return None client = client or LocalModelClient.from_env() body = _fetch_text(row["canonical_url"]) summary = summarize_article(client, row["title"], row["description"] or "", body) if not summary: return None conn.execute( "INSERT OR REPLACE INTO article_summaries (article_id, summary, model) VALUES (?, ?, ?)", (article_id, summary, client.model), ) conn.commit() return summary