indexer: massive perf boost

Cargo.lock

@@ -712,6 +712,7 @@ dependencies = [
  "color-eyre",
  "fjall",
  "log",
+ "mimalloc",
  "rayon",
  "rustc-hash",
  "vecdb",
@@ -1246,7 +1247,7 @@ version = "0.0.111"
 dependencies = [
  "bitcoin",
  "brk_error",
- "byteview 0.6.1",
+ "byteview 0.9.1",
  "derive_deref",
  "itoa",
  "jiff",
@@ -2886,6 +2887,16 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "libproc"
 version = "0.14.11"
@@ -3024,6 +3035,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "mimalloc"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"

Cargo.toml

@@ -57,8 +57,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" }
 brk_types = { version = "0.0.111", path = "crates/brk_types" }
 brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["pco", "derive"] }
 brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
-byteview = "=0.6.1"
-# byteview = "0.9.1"
+# byteview = "=0.6.1"
+byteview = "0.9.1"
 color-eyre = "0.6.5"
 derive_deref = "1.1.1"
 fjall2 = { version = "2.11.8", package = "brk_fjall" }

crates/brk_indexer/Cargo.toml

@@ -29,3 +29,4 @@ vecdb = { workspace = true }
 
 [dev-dependencies]
 color-eyre = { workspace = true }
+mimalloc = { version = "0.1.48", features = ["v3"] }

crates/brk_indexer/examples/indexer_bench.rs

@@ -1,6 +1,7 @@
 use std::{
     env, fs,
     path::Path,
+    thread::sleep,
     time::{Duration, Instant},
 };
 
@@ -11,8 +12,12 @@ use brk_iterator::Blocks;
 use brk_reader::Reader;
 use brk_rpc::{Auth, Client};
 use log::{debug, info};
+use mimalloc::MiMalloc;
 use vecdb::Exit;
 
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
 fn main() -> Result<()> {
     brk_logger::init(None)?;
 
@@ -55,7 +60,7 @@ fn main() -> Result<()> {
     // We want to benchmark the drop too
     drop(indexer);
 
-    std::thread::sleep(Duration::from_secs(10));
+    sleep(Duration::from_secs(10));
 
     Ok(())
 }

crates/brk_indexer/examples/indexer_bench2.rs

@@ -0,0 +1,63 @@
+use std::{
+    env, fs,
+    path::Path,
+    thread::sleep,
+    time::{Duration, Instant},
+};
+
+use brk_bencher::Bencher;
+use brk_error::Result;
+use brk_indexer::Indexer;
+use brk_iterator::Blocks;
+use brk_reader::Reader;
+use brk_rpc::{Auth, Client};
+use log::{debug, info};
+use mimalloc::MiMalloc;
+use vecdb::Exit;
+
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
+fn main() -> Result<()> {
+    brk_logger::init(None)?;
+
+    let bitcoin_dir = Client::default_bitcoin_path();
+    // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
+
+    let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk/benches");
+    fs::create_dir_all(&outputs_dir)?;
+    // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
+
+    let client = Client::new(
+        Client::default_url(),
+        Auth::CookieFile(bitcoin_dir.join(".cookie")),
+    )?;
+
+    let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
+
+    let blocks = Blocks::new(&client, &reader);
+
+    fs::create_dir_all(&outputs_dir)?;
+
+    let mut indexer = Indexer::forced_import(&outputs_dir)?;
+
+    let mut bencher =
+        Bencher::from_cargo_env(env!("CARGO_PKG_NAME"), &outputs_dir.join("indexed"))?;
+    bencher.start()?;
+
+    let exit = Exit::new();
+    exit.set_ctrlc_handler();
+    let bencher_clone = bencher.clone();
+    exit.register_cleanup(move || {
+        let _ = bencher_clone.stop();
+        debug!("Bench stopped.");
+    });
+
+    loop {
+        let i = Instant::now();
+        indexer.checked_index(&blocks, &client, &exit)?;
+        info!("Done in {:?}", i.elapsed());
+
+        sleep(Duration::from_secs(60));
+    }
+}

crates/brk_indexer/src/lib.rs

@@ -12,16 +12,16 @@ mod constants;
 mod indexes;
 mod processor;
 mod readers;
-mod stores_v2;
-// mod stores_v3;
+// mod stores_v2;
+mod stores_v3;
 mod vecs;
 
 use constants::*;
 pub use indexes::*;
 pub use processor::*;
 pub use readers::*;
-pub use stores_v2::*;
-// pub use stores_v3::*;
+// pub use stores_v2::*;
+pub use stores_v3::*;
 pub use vecs::*;
 
 #[derive(Clone)]

crates/brk_indexer/src/stores_v3.rs

@@ -1,9 +1,3 @@
-//! Experimental stores implementation using fjall3.
-//!
-//! This module is currently commented out in lib.rs and not in use.
-//! It exists as a work-in-progress upgrade path from fjall2 (stores_v2) to fjall3.
-//! Do not delete - intended for future activation once fjall3 is stable and tested.
-
 use std::{fs, path::Path, time::Instant};
 
 use brk_error::Result;
@@ -18,7 +12,7 @@ use log::info;
 use rayon::prelude::*;
 use vecdb::{AnyVec, TypedVecIterator, VecIndex, VecIterator};
 
-use crate::Indexes;
+use crate::{Indexes, constants::DUPLICATE_TXID_PREFIXES};
 
 use super::Vecs;
 
@@ -60,7 +54,7 @@ impl Stores {
                 version,
                 Mode3::PushOnly,
                 Kind3::Random,
-                21,
+                10,
             )
         };
 
@@ -125,7 +119,7 @@ impl Stores {
                 version,
                 Mode3::PushOnly,
                 Kind3::Random,
-                21,
+                10,
             )?,
         })
     }
@@ -187,11 +181,6 @@ impl Stores {
         self.db.persist(PersistMode::SyncData)?;
         info!("Stores persisted in {:?}", i.elapsed());
 
-        info!(
-            "self.db.config.cache.size = {}",
-            self.db.config.cache.size()
-        );
-
         Ok(())
     }
 
@@ -265,7 +254,7 @@ impl Stores {
                     let txidprefix = TxidPrefix::from(&txid);
 
                     let is_known_dup =
-                        crate::DUPLICATE_TXID_PREFIXES
+                        DUPLICATE_TXID_PREFIXES
                             .iter()
                             .any(|(dup_prefix, dup_txindex)| {
                                 txindex == *dup_txindex && txidprefix == *dup_prefix

crates/brk_store/src/fjall_v3/mod.rs

@@ -5,7 +5,9 @@ use brk_types::{Height, Version};
 use byteview_f3::ByteView;
 use fjall3::{
     Database, Keyspace, KeyspaceCreateOptions,
-    config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
+    config::{
+        BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PartitioningPolicy, PinningPolicy,
+    },
 };
 
 mod meta;
@@ -29,7 +31,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3);
 
 pub fn open_fjall3_database(path: &Path) -> fjall3::Result<Database> {
     Database::builder(path.join("fjall"))
-        .cache_size(1024 * 1024 * 1024)
+        .cache_size(2 * 1024 * 1024 * 1024)
         .open()
 }
 
@@ -84,7 +86,11 @@ where
         _mode: Mode3,
         kind: Kind3,
     ) -> Result<Keyspace> {
-        let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true);
+        let mut options = KeyspaceCreateOptions::default()
+            .manual_journal_persist(true)
+            .expect_point_read_hits(true)
+            .filter_block_partitioning_policy(PartitioningPolicy::new([false, false, true]))
+            .index_block_partitioning_policy(PartitioningPolicy::new([false, false, true]));
 
         if kind.is_not_vec() {
             options = options.filter_policy(FilterPolicy::new([
@@ -101,6 +107,8 @@ where
 
         if kind.is_sequential() {
             options = options
+                .filter_block_partitioning_policy(PartitioningPolicy::all(true))
+                .index_block_partitioning_policy(PartitioningPolicy::all(true))
                 .filter_block_pinning_policy(PinningPolicy::all(false))
                 .index_block_pinning_policy(PinningPolicy::all(false));
         }