thejayman77
1d71575982
The /a/<id> page now carries an original short summary so it stands on its own,
without republishing the publisher's article:
- summarize.py: transient SSRF-guarded fetch of the article text → local LLM
writes a 2-4 sentence ORIGINAL summary (our words). Cached in article_summaries
forever; we store only our summary, never the body. Generated lazily (only for
shared/viewed articles), de-duped so concurrent hits don't double-generate.
- /a serves cached-or-pending; when pending it shows a calm "summary on its way,
read at {source}" note and self-polls /api/summary/<id>, swapping the summary
in the moment it's ready (never blocks the page on the batch-tier LLM).
- Share menu warms generation on open so recipients usually get the rich version.
- Container reaches the arbiter at arbiter:8080 over caddy_web (LLM env added to
the API container). 124 tests pass.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
109 lines
4.3 KiB
Python
109 lines
4.3 KiB
Python
"""On-demand, cached article summaries.
|
|
|
|
Lazy: generated only for articles that actually get shared/viewed, then cached in
|
|
article_summaries forever. We fetch the article text *transiently* and ask the
|
|
local LLM for a short, ORIGINAL summary in our own words. We store only that
|
|
summary — never the publisher's article body — and the page always credits and
|
|
links to the source.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sqlite3
|
|
import urllib.error
|
|
import urllib.request
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
from .enrich import USER_AGENT, _NoRedirect, _host_is_public
|
|
from .llm import LocalModelClient
|
|
|
|
_FETCH_BYTES = 600_000
|
|
_FETCH_TIMEOUT = 8
|
|
_MAX_REDIRECTS = 3
|
|
_DROP = re.compile(rb"<(script|style|noscript|template|svg)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
|
|
_TAGS = re.compile(rb"<[^>]+>")
|
|
_WS = re.compile(r"\s+")
|
|
|
|
_SYSTEM = (
|
|
"You write a short, original summary of a news story for a calm, constructive news "
|
|
"site. Use your own words — never copy sentences from the source. 2 to 4 plain "
|
|
"sentences: what happened and why it's encouraging. No preamble, no markdown, no "
|
|
"headline — just the summary."
|
|
)
|
|
|
|
|
|
def _fetch_text(url: str) -> str:
|
|
"""SSRF-guarded fetch of an article page, reduced to plain text (capped)."""
|
|
opener = urllib.request.build_opener(_NoRedirect)
|
|
for _ in range(_MAX_REDIRECTS + 1):
|
|
if not url:
|
|
return ""
|
|
parts = urlsplit(url)
|
|
if parts.scheme not in ("http", "https") or not _host_is_public(parts.hostname):
|
|
return ""
|
|
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
|
|
try:
|
|
response = opener.open(request, timeout=_FETCH_TIMEOUT)
|
|
except (urllib.error.URLError, OSError, ValueError):
|
|
return ""
|
|
status = getattr(response, "status", 200) or 200
|
|
if status in (301, 302, 303, 307, 308):
|
|
location = response.headers.get("Location")
|
|
response.close()
|
|
if not location:
|
|
return ""
|
|
url = urljoin(url, location)
|
|
continue
|
|
if "html" not in response.headers.get("Content-Type", "").lower():
|
|
response.close()
|
|
return ""
|
|
try:
|
|
raw = response.read(_FETCH_BYTES)
|
|
finally:
|
|
response.close()
|
|
raw = _DROP.sub(b" ", raw)
|
|
text = _TAGS.sub(b" ", raw).decode("utf-8", "replace")
|
|
return _WS.sub(" ", text).strip()[:4000]
|
|
return ""
|
|
|
|
|
|
def summarize_article(client: LocalModelClient, title: str, snippet: str, body_text: str) -> str:
|
|
material = (body_text or snippet or title or "")[:4000]
|
|
user = f"Title: {title}\n\nArticle text:\n{material}\n\nWrite the summary."
|
|
messages = [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": user}]
|
|
resp = client._chat(client._build_payload(messages, None))
|
|
content = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or ""
|
|
return content.strip()[:1200]
|
|
|
|
|
|
def get_summary(conn: sqlite3.Connection, article_id: int) -> str | None:
|
|
row = conn.execute(
|
|
"SELECT summary FROM article_summaries WHERE article_id = ?", (article_id,)
|
|
).fetchone()
|
|
return row["summary"] if row else None
|
|
|
|
|
|
def generate_summary(conn: sqlite3.Connection, article_id: int, client: LocalModelClient | None = None) -> str | None:
|
|
"""Generate + cache a summary for one article. Returns it, or None if skipped."""
|
|
if get_summary(conn, article_id):
|
|
return get_summary(conn, article_id) # already cached
|
|
row = conn.execute(
|
|
"SELECT a.title, a.description, a.canonical_url, a.duplicate_of, s.accepted "
|
|
"FROM articles a LEFT JOIN article_scores s ON s.article_id = a.id WHERE a.id = ?",
|
|
(article_id,),
|
|
).fetchone()
|
|
if not row or row["duplicate_of"] is not None or not row["accepted"]:
|
|
return None
|
|
client = client or LocalModelClient.from_env()
|
|
body = _fetch_text(row["canonical_url"])
|
|
summary = summarize_article(client, row["title"], row["description"] or "", body)
|
|
if not summary:
|
|
return None
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO article_summaries (article_id, summary, model) VALUES (?, ?, ?)",
|
|
(article_id, summary, client.model),
|
|
)
|
|
conn.commit()
|
|
return summary
|