Add semantic cross-source dedup via local embeddings
- LocalModelClient.embed() calls the OpenAI-compatible /embeddings endpoint (local nomic model); base_url shared with chat, model via GOODNEWS_EMBED_MODEL. - New article_embeddings table and articles.duplicate_of column (+ migration). - dedup module: embeds missing articles, clusters near-identical stories within a date window by cosine similarity (pure-stdlib, vectors normalised once), and marks all but the highest-ranked member of each cluster as a duplicate. - 'dedup' CLI command; cycle now runs poll -> classify -> dedup -> brief. - Feed and brief queries hide duplicates, so a story carried by multiple outlets shows once. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+19
-2
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS articles (
|
||||
raw_guid TEXT,
|
||||
url_hash TEXT NOT NULL UNIQUE,
|
||||
title_hash TEXT,
|
||||
duplicate_of INTEGER REFERENCES articles(id) ON DELETE SET NULL,
|
||||
FOREIGN KEY (source_id) REFERENCES sources(id)
|
||||
);
|
||||
|
||||
@@ -62,6 +63,14 @@ CREATE TABLE IF NOT EXISTS article_scores (
|
||||
scored_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS article_embeddings (
|
||||
article_id INTEGER PRIMARY KEY REFERENCES articles(id) ON DELETE CASCADE,
|
||||
vector BLOB NOT NULL,
|
||||
dim INTEGER NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL,
|
||||
@@ -114,7 +123,15 @@ def _migrate(conn: sqlite3.Connection) -> None:
|
||||
CREATE TABLE IF NOT EXISTS never alters an existing table, so new columns
|
||||
need an explicit, idempotent ALTER guarded by the current column set.
|
||||
"""
|
||||
cols = {row["name"] for row in conn.execute("PRAGMA table_info(article_scores)")}
|
||||
score_cols = {row["name"] for row in conn.execute("PRAGMA table_info(article_scores)")}
|
||||
for column in ("topic", "flavor"):
|
||||
if column not in cols:
|
||||
if column not in score_cols:
|
||||
conn.execute(f"ALTER TABLE article_scores ADD COLUMN {column} TEXT")
|
||||
|
||||
article_cols = {row["name"] for row in conn.execute("PRAGMA table_info(articles)")}
|
||||
if "duplicate_of" not in article_cols:
|
||||
conn.execute(
|
||||
"ALTER TABLE articles ADD COLUMN duplicate_of INTEGER REFERENCES articles(id)"
|
||||
)
|
||||
# Created here (not in SCHEMA) so it runs after the column exists on upgrades.
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_articles_duplicate_of ON articles(duplicate_of)")
|
||||
|
||||
Reference in New Issue
Block a user