thejayman77
bfd612eb9b
- paywall.py: conservative domain-level paywall detection (New Scientist, Nature, and common hard/soft paywalls). Never fetches pages — an honest hint. - API: Article gains a 'paywalled' flag; the brief now leads with a gentle AND readable story (paywalled/charged stories stay in the five, just not first). - New GET /api/replacement returns the next-best readable, unshown article (honors mood+prefs via the merged prefs param; gentle=true for hero swaps). - UI: paywalled cards show 'May need a subscription'; a Replace / 'Find one I can read' action (always visible, while tuning actions stay tucked) swaps the card for a readable alternative, with a gentle notice when none remain. - Tests: paywall detection + replacement behavior (77 total). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
42 lines
1.1 KiB
Python
42 lines
1.1 KiB
Python
"""Domain-level paywall hints.
|
|
|
|
We never fetch article pages, so a paywall can only be inferred from the host.
|
|
This is a curated, conservative list of hard/soft paywalls — enough to label a
|
|
card "subscription may be required" and to prefer readable stories for the lead
|
|
and for replacements. It will never be perfect; it's an honest hint, not a gate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from urllib.parse import urlsplit
|
|
|
|
# Host suffixes considered paywalled. Subdomains match (news.nature.com → nature.com).
|
|
PAYWALL_DOMAINS = {
|
|
"newscientist.com",
|
|
"nature.com",
|
|
"nytimes.com",
|
|
"wsj.com",
|
|
"ft.com",
|
|
"economist.com",
|
|
"wired.com",
|
|
"theatlantic.com",
|
|
"washingtonpost.com",
|
|
"bloomberg.com",
|
|
"technologyreview.com",
|
|
"newyorker.com",
|
|
"scientificamerican.com",
|
|
"nationalgeographic.com",
|
|
"thetimes.co.uk",
|
|
"telegraph.co.uk",
|
|
"foreignpolicy.com",
|
|
"hbr.org",
|
|
"harpers.org",
|
|
}
|
|
|
|
|
|
def is_paywalled(url: str | None) -> bool:
|
|
host = urlsplit(url or "").netloc.lower()
|
|
if host.startswith("www."):
|
|
host = host[4:]
|
|
return any(host == d or host.endswith("." + d) for d in PAYWALL_DOMAINS)
|