avoid re-ingesting already-indexed events on restart

Track the last seen event timestamp per relay in the DB. On startup,
subscribe to kind 2003 per relay using that timestamp as since instead
of always recomputing from backfill_days. Fresh relays with no stored
timestamp fall back to the full backfill window.
This commit is contained in:
2026-05-18 22:43:14 -07:00
parent 619ab32421
commit 3d1acfdc04
2 changed files with 43 additions and 7 deletions
+11
View File
@@ -452,6 +452,17 @@ pub async fn update_relay_last_event(
Ok(())
}
/// Returns a map of relay URL → last seen event timestamp for all known relays.
pub async fn get_all_relay_timestamps(pool: &SqlitePool) -> std::collections::HashMap<String, i64> {
let rows: Vec<(String, i64)> = sqlx::query_as(
"SELECT url, last_event FROM relays WHERE last_event IS NOT NULL",
)
.fetch_all(pool)
.await
.unwrap_or_default();
rows.into_iter().collect()
}
pub async fn update_relay_sync(pool: &SqlitePool, url: &str, ts: i64) -> anyhow::Result<()> {
sqlx::query("UPDATE relays SET last_sync = ? WHERE url = ?")
.bind(ts)
+32 -7
View File
@@ -65,23 +65,48 @@ async fn run_reader(cfg: Arc<Config>, pool: SqlitePool, connected: Arc<AtomicI32
.unwrap_or_default()
.as_secs()
.saturating_sub(cfg.backfill_days as u64 * 86400);
let since = Timestamp::from(backfill_since);
// Subscribe to kind 2003 (torrents)
if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(2003)).since(since), None).await {
error!("subscribe(2003) failed: {e}");
// Load per-relay last-seen timestamps so we only request events we haven't
// indexed yet. Fresh relays fall back to the full backfill window.
let relay_timestamps = db::get_all_relay_timestamps(&pool).await;
// Subscribe to kind 2003 (torrents) per relay with its own since value.
let mut any_subscribed = false;
for relay_url in &cfg.relays {
let relay_since = relay_timestamps
.get(relay_url.as_str())
.copied()
.map(|ts| (ts as u64).max(backfill_since))
.unwrap_or(backfill_since);
let since_ts = Timestamp::from(relay_since);
match client.subscribe_to(
[relay_url.as_str()],
Filter::new().kind(Kind::Custom(2003)).since(since_ts),
None,
).await {
Ok(_) => {
debug!(url = relay_url, since = relay_since, "subscribed kind 2003");
any_subscribed = true;
}
Err(e) => warn!(url = relay_url, "subscribe(2003) failed: {e}"),
}
}
if !any_subscribed {
error!("all relay subscriptions failed");
connected.store(0, Ordering::Relaxed);
return;
}
info!("subscribed to kind 2003 since unix={backfill_since}");
info!("subscribed to kind 2003 on {} relays (per-relay since)", cfg.relays.len());
// Subscribe to kind 30004 (curation sets) from configured pubkeys
let global_since = Timestamp::from(backfill_since);
let curation_pks: Vec<PublicKey> = cfg.curation.curation_sets.iter()
.filter_map(|s| s.parse().ok())
.collect();
if !curation_pks.is_empty() {
if let Err(e) = client.subscribe(
Filter::new().kind(Kind::Custom(30004)).authors(curation_pks).since(since),
Filter::new().kind(Kind::Custom(30004)).authors(curation_pks).since(global_since),
None,
).await {
warn!("subscribe(30004) failed: {e}");
@@ -91,7 +116,7 @@ async fn run_reader(cfg: Arc<Config>, pool: SqlitePool, connected: Arc<AtomicI32
}
// Subscribe to kind 1984 (reports)
if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(1984)).since(since), None).await {
if let Err(e) = client.subscribe(Filter::new().kind(Kind::Custom(1984)).since(global_since), None).await {
warn!("subscribe(1984) failed: {e}");
}