From 3d1acfdc044cec6c1e6af1e0068bbe39a1cb1984 Mon Sep 17 00:00:00 2001
From: enki <enki@sovbit.dev>
Date: Mon, 18 May 2026 22:43:14 -0700
Subject: [PATCH] avoid re-ingesting already-indexed events on restart

Track the last seen event timestamp per relay in the DB. On startup,
subscribe to kind 2003 per relay using that timestamp as since instead
of always recomputing from backfill_days. Fresh relays with no stored
timestamp fall back to the full backfill window.
---
 src/db/queries.rs   | 11 +++++++++++
 src/nostr/reader.rs | 39 ++++++++++++++++++++++++++++++++-------
 2 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/src/db/queries.rs b/src/db/queries.rs
index 817ab99..6b74775 100644
--- a/src/db/queries.rs
+++ b/src/db/queries.rs
@@ -452,6 +452,17 @@ pub async fn update_relay_last_event(
     Ok(())
 }
 
+/// Returns a map of relay URL → last seen event timestamp for all known relays.
+pub async fn get_all_relay_timestamps(pool: &SqlitePool) -> std::collections::HashMap<String, i64> {
+    let rows: Vec<(String, i64)> = sqlx::query_as(
+        "SELECT url, last_event FROM relays WHERE last_event IS NOT NULL",
+    )
+    .fetch_all(pool)
+    .await
+    .unwrap_or_default();
+    rows.into_iter().collect()
+}
+
 pub async fn update_relay_sync(pool: &SqlitePool, url: &str, ts: i64) -> anyhow::Result<()> {
     sqlx::query("UPDATE relays SET last_sync = ? WHERE url = ?")
         .bind(ts)
diff --git a/src/nostr/reader.rs b/src/nostr/reader.rs
index 30fb1a0..7aa266b 100644
--- a/src/nostr/reader.rs
+++ b/src/nostr/reader.rs
@@ -65,23 +65,48 @@ async fn run_reader(cfg: Arc<Config>, pool: SqlitePool, connected: Arc<AtomicI32
         .unwrap_or_default()
         .as_secs()
         .saturating_sub(cfg.backfill_days as u64 * 86400);
-    let since = Timestamp::from(backfill_since);
 
-    // Subscribe to kind 2003 (torrents)
-    if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(2003)).since(since), None).await {
-        error!("subscribe(2003) failed: {e}");
+    // Load per-relay last-seen timestamps so we only request events we haven't
+    // indexed yet. Fresh relays fall back to the full backfill window.
+    let relay_timestamps = db::get_all_relay_timestamps(&pool).await;
+
+    // Subscribe to kind 2003 (torrents) per relay with its own since value.
+    let mut any_subscribed = false;
+    for relay_url in &cfg.relays {
+        let relay_since = relay_timestamps
+            .get(relay_url.as_str())
+            .copied()
+            .map(|ts| (ts as u64).max(backfill_since))
+            .unwrap_or(backfill_since);
+        let since_ts = Timestamp::from(relay_since);
+
+        match client.subscribe_to(
+            [relay_url.as_str()],
+            Filter::new().kind(Kind::Custom(2003)).since(since_ts),
+            None,
+        ).await {
+            Ok(_) => {
+                debug!(url = relay_url, since = relay_since, "subscribed kind 2003");
+                any_subscribed = true;
+            }
+            Err(e) => warn!(url = relay_url, "subscribe(2003) failed: {e}"),
+        }
+    }
+    if !any_subscribed {
+        error!("all relay subscriptions failed");
         connected.store(0, Ordering::Relaxed);
         return;
     }
-    info!("subscribed to kind 2003 since unix={backfill_since}");
+    info!("subscribed to kind 2003 on {} relays (per-relay since)", cfg.relays.len());
 
     // Subscribe to kind 30004 (curation sets) from configured pubkeys
+    let global_since = Timestamp::from(backfill_since);
     let curation_pks: Vec<PublicKey> = cfg.curation.curation_sets.iter()
         .filter_map(|s| s.parse().ok())
         .collect();
     if !curation_pks.is_empty() {
         if let Err(e) = client.subscribe(
-            Filter::new().kind(Kind::Custom(30004)).authors(curation_pks).since(since),
+            Filter::new().kind(Kind::Custom(30004)).authors(curation_pks).since(global_since),
             None,
         ).await {
             warn!("subscribe(30004) failed: {e}");
@@ -91,7 +116,7 @@ async fn run_reader(cfg: Arc<Config>, pool: SqlitePool, connected: Arc<AtomicI32
     }
 
     // Subscribe to kind 1984 (reports)
-    if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(1984)).since(since), None).await {
+    if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(1984)).since(global_since), None).await {
         warn!("subscribe(1984) failed: {e}");
     }