From e7c3f26582da2940a682afd333ac3d433b83fd03 Mon Sep 17 00:00:00 2001
From: Ahmed Khaled <125749397+ahmedk20@users.noreply.github.com>
Date: Sat, 28 Mar 2026 16:51:21 +0200
Subject: [PATCH] fix: use URL-based    hash as Pinecone vector ID to prevent
 silent data loss
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

   Articles published on the same date shared the same date-integer ID
   (from sorting.ordering_date). Upserting to Pinecone with duplicate IDs
   silently overwrote previous records — only the last article per date
   survived in the index.

   Replace the date-integer with an MD5 hash of the article URL. URLs are
   already deduplicated before this point (seen_urls set), so each hash is
   guaranteed unique per run. Repeated runs correctly overwrite the same
   record rather than creating duplicates.

   Fixes #175
---
 db_update/Update.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/db_update/Update.py b/db_update/Update.py
index 4109397..8f0d771 100644
--- a/db_update/Update.py
+++ b/db_update/Update.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import hashlib
 from dotenv import dotenv_values
 from pinecone import Pinecone, ServerlessSpec
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
@@ -100,9 +101,13 @@ def update_database(overwrite=(len(sys.argv) > 1 and sys.argv[1] == '--overwrite
                 values = [float(emb_array[i]) for i in indices]
                 sparse_vector = {"indices": indices, "values": values}
 
+                # Use a URL-based hash as the Pinecone vector ID.
+                # guaranteed unique per article since URLs are already deduplicated.
+                vector_id = hashlib.md5(url.encode()).hexdigest()
+
                 # Construct record exactly as per Pinecone documentation
                 record = {
-                    "id": str(article.get("id", "")),
+                    "id": vector_id,
                     "values": dense_vector,
                     "sparse_values": sparse_vector,
                     "metadata": {