diff --git a/Cargo.lock b/Cargo.lock index eb00cb3b9..22bd9dbf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,8 +675,8 @@ version = "0.0.111" dependencies = [ "bitcoin", "bitcoincore-rpc", - "brk_fjall 2.11.5", - "brk_fjall 3.0.0-pre.4", + "brk_fjall", + "fjall", "jiff", "minreq", "sonic-rs", @@ -713,20 +713,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "brk_fjall" -version = "3.0.0-pre.4" -dependencies = [ - "byteorder-lite", - "byteview 0.8.0", - "dashmap", - "log", - "lsm-tree 3.0.0-pre.4", - "std-semaphore", - "tempfile", - "xxhash-rust", -] - [[package]] name = "brk_grouper" version = "0.0.111" @@ -744,8 +730,7 @@ version = "0.0.111" dependencies = [ "bitcoin", "brk_error", - "brk_fjall 2.11.5", - "brk_fjall 3.0.0-pre.4", + "brk_fjall", "brk_grouper", "brk_iterator", "brk_logger", @@ -754,6 +739,7 @@ dependencies = [ "brk_store", "brk_traversable", "brk_types", + "fjall", "log", "rand 0.9.2", "rayon", @@ -1286,12 +1272,12 @@ name = "brk_store" version = "0.0.111" dependencies = [ "brk_error", - "brk_fjall 2.11.5", - "brk_fjall 3.0.0-pre.4", + "brk_fjall", "brk_types", "byteview 0.6.1", "byteview 0.8.0", "candystore", + "fjall", "log", "parking_lot 0.12.5", "rustc-hash", @@ -1335,7 +1321,7 @@ dependencies = [ "allocative", "bitcoin", "brk_error", - "byteview 0.6.1", + "byteview 0.8.0", "derive_deref", "itoa", "jiff", @@ -2116,6 +2102,22 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "fjall" +version = "3.0.0-pre.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf4746eb86124d4cd91564bc01c247b8c2c3ddf2d9fb3701ca16c86fdb5b538" +dependencies = [ + "byteorder-lite", + "byteview 0.8.0", + "dashmap", + "log", + "lsm-tree 3.0.0-pre.4", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "flate2" version = "1.1.5" @@ -4108,9 +4110,9 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c06cc46cc4b6da16c9981293906062230f822879c2c88367a40673db5e44414" +checksum = "d07ae2ce76ce46d26972046ae67d1f65ddbf5bc1290d3abaee84b7ab159564c4" dependencies = [ "libc", "memmap2", @@ -5321,9 +5323,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0cff5037d7e5d66344dff249c366a527b530ef3c723e04f4a90fa11d5f6a9fc" +checksum = "d96acc025cc0039bf828cbf4efdcb485eb0eda88da7ec805fb6ec9159a340b73" dependencies = [ "allocative", "ctrlc", @@ -5340,9 +5342,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f73046e065e63abb2ad8d1616a08ca8acf7d543f04ffb0222f6fc44e0d3eeec" +checksum = "dae24e54786d92d4aa4b46098eed2594cd6ed51d2a47a36ff8ae3adfa232d958" dependencies = [ "quote", "syn 2.0.109", diff --git a/Cargo.toml b/Cargo.toml index 4a21a4fa4..122c68d72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,14 +52,14 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_types = { version = "0.0.111", path = "crates/brk_types" } brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] } brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" } -byteview = "=0.6.1" -# byteview = "~0.8.0" +# byteview = "=0.6.1" +byteview = "~0.8.0" derive_deref = "1.1.1" fjall2 = { version = "2.11.5", package = "brk_fjall" } # fjall2 = { path = "../fjall2", package = "brk_fjall" } # fjall2 = { version = "2.11.2", package = "fjall" } -# fjall3 = { version = "=3.0.0-pre.0", package = "fjall" } -fjall3 = { path = "../fjall3", package = "brk_fjall" } +fjall3 = { version = "=3.0.0-pre.4", package = "fjall" } +# fjall3 = { path = "../fjall3", package = "brk_fjall" } # fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" } jiff = "0.2.15" log = "0.4.28" @@ -75,7 +75,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] } sonic-rs = "0.5.6" tokio = { version = "1.48.0", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] } -vecdb = { version = "0.3.7", features = ["derive"] } +vecdb = { version = "0.3.8", features = ["derive"] } zerocopy = { version = "0.8.27", features = ["derive"] } [workspace.metadata.release] diff --git a/crates/brk_computer/src/blks.rs b/crates/brk_computer/src/blks.rs index 7403a1447..f08651371 100644 --- a/crates/brk_computer/src/blks.rs +++ b/crates/brk_computer/src/blks.rs @@ -94,22 +94,16 @@ impl Vecs { .try_for_each(|block| -> Result<()> { let height = block.height(); - self.height_to_position.forced_push_at( - height, - block.metadata().position(), - exit, - )?; + self.height_to_position + .forced_push(height, block.metadata().position(), exit)?; let txindex = height_to_first_txindex_iter.get_unwrap(height); block.tx_metadata().iter().enumerate().try_for_each( |(index, metadata)| -> Result<()> { let txindex = txindex + index; - self.txindex_to_position.forced_push_at( - txindex, - metadata.position(), - exit, - )?; + self.txindex_to_position + .forced_push(txindex, metadata.position(), exit)?; Ok(()) }, )?; diff --git a/crates/brk_computer/src/fetched.rs b/crates/brk_computer/src/fetched.rs index a04968185..4d8cf30e3 100644 --- a/crates/brk_computer/src/fetched.rs +++ b/crates/brk_computer/src/fetched.rs @@ -133,7 +133,7 @@ impl Vecs { prev.replace(ohlc.clone()); self.dateindex_to_price_ohlc_in_cents - .forced_push_at_(i, ohlc, exit)?; + .forced_push_at(i, ohlc, exit)?; Ok(()) })?; diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index 0ebdce910..81d1170e4 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -14,7 +14,7 @@ use brk_types::{ }; use vecdb::{ AnyCloneableIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE, - StoredIndex, VecIterator, + StoredIndex, VecIteratorExtended, }; const VERSION: Version = Version::ZERO; @@ -117,15 +117,13 @@ impl Vecs { indexer.vecs.txindex_to_first_txoutindex.boxed_clone(), |index: TxInIndex, txinindex_to_outpoint_iter, txindex_to_first_txoutindex_iter| { txinindex_to_outpoint_iter - .next_at(index.to_usize()) - .map(|(_, outpoint)| { + .get_at(index.to_usize()) + .map(|outpoint| { if outpoint.is_coinbase() { return TxOutIndex::COINBASE; } txindex_to_first_txoutindex_iter - .next_at(outpoint.txindex().to_usize()) - .unwrap() - .1 + .get_unwrap_at(outpoint.txindex().to_usize()) + outpoint.vout() }) }, @@ -160,12 +158,12 @@ impl Vecs { |index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| { let txindex = index.to_usize(); txindex_to_first_txinindex_iter - .next_at(txindex) - .map(|(_, start)| { + .get_at(txindex) + .map(|start| { let start = usize::from(start); let end = txindex_to_first_txinindex_iter - .next_at(txindex + 1) - .map(|(_, v)| usize::from(v)) + .get_at(txindex + 1) + .map(|v| usize::from(v)) .unwrap_or_else(|| txinindex_to_txoutindex_iter.len()); StoredU64::from((start..end).count()) }) @@ -180,12 +178,12 @@ impl Vecs { |index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| { let txindex = index.to_usize(); txindex_to_first_txoutindex_iter - .next_at(txindex) - .map(|(_, start)| { + .get_at(txindex) + .map(|start| { let start = usize::from(start); let end = txindex_to_first_txoutindex_iter - .next_at(txindex + 1) - .map(|(_, v)| usize::from(v)) + .get_at(txindex + 1) + .map(|v| usize::from(v)) .unwrap_or_else(|| txoutindex_to_value_iter.len()); StoredU64::from((start..end).count()) }) @@ -558,7 +556,7 @@ impl Vecs { prev_timestamp_fixed.replace( height_to_timestamp_fixed_iter .into_iter() - .unsafe_get(prev_h), + .get_unwrap(prev_h), ); } let timestamp_fixed = @@ -585,7 +583,7 @@ impl Vecs { let starting_dateindex = self .height_to_dateindex .into_iter() - .get_inner(decremented_starting_height) + .get(decremented_starting_height) .unwrap_or_default(); self.height_to_dateindex.compute_transform( @@ -598,7 +596,7 @@ impl Vecs { let starting_dateindex = if let Some(dateindex) = self .height_to_dateindex .into_iter() - .get_inner(decremented_starting_height) + .get(decremented_starting_height) { starting_dateindex.min(dateindex) } else { @@ -637,7 +635,7 @@ impl Vecs { let starting_weekindex = self .dateindex_to_weekindex .into_iter() - .get_inner(starting_dateindex) + .get(starting_dateindex) .unwrap_or_default(); self.dateindex_to_weekindex.compute_range( @@ -674,7 +672,7 @@ impl Vecs { let starting_difficultyepoch = self .height_to_difficultyepoch .into_iter() - .get_inner(decremented_starting_height) + .get(decremented_starting_height) .unwrap_or_default(); self.height_to_difficultyepoch.compute_from_index( @@ -710,7 +708,7 @@ impl Vecs { let starting_monthindex = self .dateindex_to_monthindex .into_iter() - .get_inner(starting_dateindex) + .get(starting_dateindex) .unwrap_or_default(); self.dateindex_to_monthindex.compute_range( @@ -747,7 +745,7 @@ impl Vecs { let starting_quarterindex = self .monthindex_to_quarterindex .into_iter() - .get_inner(starting_monthindex) + .get(starting_monthindex) .unwrap_or_default(); self.monthindex_to_quarterindex.compute_from_index( @@ -785,7 +783,7 @@ impl Vecs { let starting_semesterindex = self .monthindex_to_semesterindex .into_iter() - .get_inner(starting_monthindex) + .get(starting_monthindex) .unwrap_or_default(); self.monthindex_to_semesterindex.compute_from_index( @@ -823,7 +821,7 @@ impl Vecs { let starting_yearindex = self .monthindex_to_yearindex .into_iter() - .get_inner(starting_monthindex) + .get(starting_monthindex) .unwrap_or_default(); self.monthindex_to_yearindex.compute_from_index( @@ -858,7 +856,7 @@ impl Vecs { let starting_halvingepoch = self .height_to_halvingepoch .into_iter() - .get_inner(decremented_starting_height) + .get(decremented_starting_height) .unwrap_or_default(); self.height_to_halvingepoch.compute_from_index( @@ -886,7 +884,7 @@ impl Vecs { let starting_decadeindex = self .yearindex_to_decadeindex .into_iter() - .get_inner(starting_yearindex) + .get(starting_yearindex) .unwrap_or_default(); self.yearindex_to_decadeindex.compute_from_index( @@ -947,7 +945,7 @@ impl Indexes { pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) { self.indexes.height = height; self.dateindex = - DateIndex::try_from(indexes.height_to_date_fixed.into_iter().unsafe_get(height)) + DateIndex::try_from(indexes.height_to_date_fixed.into_iter().get_unwrap(height)) .unwrap(); self.weekindex = WeekIndex::from(self.dateindex); self.monthindex = MonthIndex::from(self.dateindex); diff --git a/crates/brk_computer/src/traits.rs b/crates/brk_computer/src/traits.rs index 619897db3..384adfc3f 100644 --- a/crates/brk_computer/src/traits.rs +++ b/crates/brk_computer/src/traits.rs @@ -197,7 +197,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec { stacks .iter() - .skip(index) + .skip(index.to_usize()) .enumerate() .try_for_each(|(i, stack)| { let mut avg_price = Dollars::from(f64::NAN); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 71e2ad6db..3b33e4b9c 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -18,20 +18,20 @@ use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended}; mod indexes; -mod stores_v2; -// mod stores_v3; +// mod stores_v2; +mod stores_v3; mod vecs; pub use indexes::*; -pub use stores_v2::*; -// pub use stores_v3::*; +// pub use stores_v2::*; +pub use stores_v3::*; pub use vecs::*; // One version for all data sources // Increment on **change _OR_ addition** const VERSION: Version = Version::new(23); const SNAPSHOT_BLOCK_RANGE: usize = 1_000; -const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000); +const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0); #[derive(Clone)] pub struct Indexer { diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index d327b94b4..652af75ff 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { }); Builder::from_env(Env::default().default_filter_or( - "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", - // "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + // "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", )) .format(move |buf, record| { let date_time = Timestamp::now() diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index db1930631..eeca9cbb2 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -4,7 +4,10 @@ use brk_error::Result; use brk_types::{Height, Version}; use byteview8::ByteView; use fjall3::{ - Database, Keyspace, KeyspaceCreateOptions, ValueType, + Database, + Keyspace, + KeyspaceCreateOptions, + // ValueType, config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy}, }; @@ -198,13 +201,28 @@ where ) .collect::>(); items.sort_unstable(); - batch.ingest( - items - .into_iter() - .map(|i| i.fjalled(&self.keyspace)) - .collect::>(), - ); - batch.commit_keyspace(&self.keyspace)?; + items.into_iter().for_each(|item| match item { + Item::Value { key, value } => { + batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value)) + } + Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)), + }); + batch.commit()?; + + // batch.ingest( + // items + // .into_iter() + // .map(|i| i.fjalled(&self.keyspace)) + // .collect::>(), + // ); + // batch.commit_keyspace(&self.keyspace)?; + // batch.ingest( + // items + // .into_iter() + // .map(|i| i.fjalled(&self.keyspace)) + // .collect::>(), + // ); + // batch.commit_keyspace(&self.keyspace)?; // batch.commit_keyspace(self.keyspace.inner())?; Ok(()) @@ -263,26 +281,26 @@ impl Item { } } - pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item - where - K: Into, - V: Into, - { - let keyspace_id = keyspace.id; - // let keyspace_id = keyspace.inner().id; - match self { - Item::Value { key, value } => fjall3::Item { - keyspace_id, - key: key.into().into(), - value: value.into().into(), - value_type: ValueType::Value, - }, - Item::Tomb(key) => fjall3::Item { - keyspace_id, - key: key.into().into(), - value: [].into(), - value_type: ValueType::WeakTombstone, - }, - } - } + // pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item + // where + // K: Into, + // V: Into, + // { + // let keyspace_id = keyspace.id; + // // let keyspace_id = keyspace.inner().id; + // match self { + // Item::Value { key, value } => fjall3::Item { + // keyspace_id, + // key: key.into().into(), + // value: value.into().into(), + // value_type: ValueType::Value, + // }, + // Item::Tomb(key) => fjall3::Item { + // keyspace_id, + // key: key.into().into(), + // value: [].into(), + // value_type: ValueType::WeakTombstone, + // }, + // } + // } }