diff --git a/Cargo.lock b/Cargo.lock index 88d9038c6..5389d6789 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -712,6 +712,7 @@ dependencies = [ "color-eyre", "fjall", "log", + "mimalloc", "rayon", "rustc-hash", "vecdb", @@ -1246,7 +1247,7 @@ version = "0.0.111" dependencies = [ "bitcoin", "brk_error", - "byteview 0.6.1", + "byteview 0.9.1", "derive_deref", "itoa", "jiff", @@ -2886,6 +2887,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libproc" version = "0.14.11" @@ -3024,6 +3035,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index d850abd42..11e460309 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,8 +57,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_types = { version = "0.0.111", path = "crates/brk_types" } brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["pco", "derive"] } brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" } -byteview = "=0.6.1" -# byteview = "0.9.1" +# byteview = "=0.6.1" +byteview = "0.9.1" color-eyre = "0.6.5" derive_deref = "1.1.1" fjall2 = { version = "2.11.8", package = "brk_fjall" } diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 538953d7f..6f8dda0c9 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -29,3 +29,4 @@ vecdb = { workspace = true } [dev-dependencies] color-eyre = { workspace = true } +mimalloc = { version = "0.1.48", features = ["v3"] } diff --git a/crates/brk_indexer/examples/indexer_bench.rs b/crates/brk_indexer/examples/indexer_bench.rs index 8ad491614..c7d0b2cff 100644 --- a/crates/brk_indexer/examples/indexer_bench.rs +++ b/crates/brk_indexer/examples/indexer_bench.rs @@ -1,6 +1,7 @@ use std::{ env, fs, path::Path, + thread::sleep, time::{Duration, Instant}, }; @@ -11,8 +12,12 @@ use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; +use mimalloc::MiMalloc; use vecdb::Exit; +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + fn main() -> Result<()> { brk_logger::init(None)?; @@ -55,7 +60,7 @@ fn main() -> Result<()> { // We want to benchmark the drop too drop(indexer); - std::thread::sleep(Duration::from_secs(10)); + sleep(Duration::from_secs(10)); Ok(()) } diff --git a/crates/brk_indexer/examples/indexer_bench2.rs b/crates/brk_indexer/examples/indexer_bench2.rs new file mode 100644 index 000000000..46bc3cb7e --- /dev/null +++ b/crates/brk_indexer/examples/indexer_bench2.rs @@ -0,0 +1,63 @@ +use std::{ + env, fs, + path::Path, + thread::sleep, + time::{Duration, Instant}, +}; + +use brk_bencher::Bencher; +use brk_error::Result; +use brk_indexer::Indexer; +use brk_iterator::Blocks; +use brk_reader::Reader; +use brk_rpc::{Auth, Client}; +use log::{debug, info}; +use mimalloc::MiMalloc; +use vecdb::Exit; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +fn main() -> Result<()> { + brk_logger::init(None)?; + + let bitcoin_dir = Client::default_bitcoin_path(); + // let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin"); + + let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk/benches"); + fs::create_dir_all(&outputs_dir)?; + // let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk"); + + let client = Client::new( + Client::default_url(), + Auth::CookieFile(bitcoin_dir.join(".cookie")), + )?; + + let reader = Reader::new(bitcoin_dir.join("blocks"), &client); + + let blocks = Blocks::new(&client, &reader); + + fs::create_dir_all(&outputs_dir)?; + + let mut indexer = Indexer::forced_import(&outputs_dir)?; + + let mut bencher = + Bencher::from_cargo_env(env!("CARGO_PKG_NAME"), &outputs_dir.join("indexed"))?; + bencher.start()?; + + let exit = Exit::new(); + exit.set_ctrlc_handler(); + let bencher_clone = bencher.clone(); + exit.register_cleanup(move || { + let _ = bencher_clone.stop(); + debug!("Bench stopped."); + }); + + loop { + let i = Instant::now(); + indexer.checked_index(&blocks, &client, &exit)?; + info!("Done in {:?}", i.elapsed()); + + sleep(Duration::from_secs(60)); + } +} diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index bbc39201e..58458c1d5 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -12,16 +12,16 @@ mod constants; mod indexes; mod processor; mod readers; -mod stores_v2; -// mod stores_v3; +// mod stores_v2; +mod stores_v3; mod vecs; use constants::*; pub use indexes::*; pub use processor::*; pub use readers::*; -pub use stores_v2::*; -// pub use stores_v3::*; +// pub use stores_v2::*; +pub use stores_v3::*; pub use vecs::*; #[derive(Clone)] diff --git a/crates/brk_indexer/src/stores_v3.rs b/crates/brk_indexer/src/stores_v3.rs index e42937843..24722ca4e 100644 --- a/crates/brk_indexer/src/stores_v3.rs +++ b/crates/brk_indexer/src/stores_v3.rs @@ -1,9 +1,3 @@ -//! Experimental stores implementation using fjall3. -//! -//! This module is currently commented out in lib.rs and not in use. -//! It exists as a work-in-progress upgrade path from fjall2 (stores_v2) to fjall3. -//! Do not delete - intended for future activation once fjall3 is stable and tested. - use std::{fs, path::Path, time::Instant}; use brk_error::Result; @@ -18,7 +12,7 @@ use log::info; use rayon::prelude::*; use vecdb::{AnyVec, TypedVecIterator, VecIndex, VecIterator}; -use crate::Indexes; +use crate::{Indexes, constants::DUPLICATE_TXID_PREFIXES}; use super::Vecs; @@ -60,7 +54,7 @@ impl Stores { version, Mode3::PushOnly, Kind3::Random, - 21, + 10, ) }; @@ -125,7 +119,7 @@ impl Stores { version, Mode3::PushOnly, Kind3::Random, - 21, + 10, )?, }) } @@ -187,11 +181,6 @@ impl Stores { self.db.persist(PersistMode::SyncData)?; info!("Stores persisted in {:?}", i.elapsed()); - info!( - "self.db.config.cache.size = {}", - self.db.config.cache.size() - ); - Ok(()) } @@ -265,7 +254,7 @@ impl Stores { let txidprefix = TxidPrefix::from(&txid); let is_known_dup = - crate::DUPLICATE_TXID_PREFIXES + DUPLICATE_TXID_PREFIXES .iter() .any(|(dup_prefix, dup_txindex)| { txindex == *dup_txindex && txidprefix == *dup_prefix diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index cc48c57ca..4add6186d 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -5,7 +5,9 @@ use brk_types::{Height, Version}; use byteview_f3::ByteView; use fjall3::{ Database, Keyspace, KeyspaceCreateOptions, - config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy}, + config::{ + BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PartitioningPolicy, PinningPolicy, + }, }; mod meta; @@ -29,7 +31,7 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3); pub fn open_fjall3_database(path: &Path) -> fjall3::Result { Database::builder(path.join("fjall")) - .cache_size(1024 * 1024 * 1024) + .cache_size(2 * 1024 * 1024 * 1024) .open() } @@ -84,7 +86,11 @@ where _mode: Mode3, kind: Kind3, ) -> Result { - let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true); + let mut options = KeyspaceCreateOptions::default() + .manual_journal_persist(true) + .expect_point_read_hits(true) + .filter_block_partitioning_policy(PartitioningPolicy::new([false, false, true])) + .index_block_partitioning_policy(PartitioningPolicy::new([false, false, true])); if kind.is_not_vec() { options = options.filter_policy(FilterPolicy::new([ @@ -101,6 +107,8 @@ where if kind.is_sequential() { options = options + .filter_block_partitioning_policy(PartitioningPolicy::all(true)) + .index_block_partitioning_policy(PartitioningPolicy::all(true)) .filter_block_pinning_policy(PinningPolicy::all(false)) .index_block_pinning_policy(PinningPolicy::all(false)); }