From 3d3787a8d9971ab63affa55f866ecc756ed8c314 Mon Sep 17 00:00:00 2001 From: nym21 Date: Fri, 7 Nov 2025 15:13:01 +0100 Subject: [PATCH] indexer: snapshot --- Cargo.lock | 43 ++++----- Cargo.toml | 8 +- crates/brk_indexer/src/lib.rs | 8 +- crates/brk_indexer/src/stores_v2.rs | 42 ++++++-- crates/brk_indexer/src/stores_v3.rs | 82 ++++++++-------- crates/brk_logger/src/lib.rs | 4 +- crates/brk_store/src/fjall_v2/meta.rs | 8 +- crates/brk_store/src/fjall_v2/mod.rs | 80 ++++++++++----- crates/brk_store/src/fjall_v3/mod.rs | 118 ++++++++++++++++------- crates/brk_types/src/addressbyteshash.rs | 6 +- crates/brk_types/src/blockhashprefix.rs | 16 +-- crates/brk_types/src/txidprefix.rs | 16 +-- 12 files changed, 262 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22bd9dbf7..ed2f22100 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1321,7 +1321,7 @@ dependencies = [ "allocative", "bitcoin", "brk_error", - "byteview 0.8.0", + "byteview 0.6.1", "derive_deref", "itoa", "jiff", @@ -1448,9 +1448,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.44" +version = "1.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" dependencies = [ "find-msvc-tools", "jobserver", @@ -2670,24 +2670,24 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" dependencies = [ "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", - "serde", - "windows-sys 0.59.0", + "serde_core", + "windows-sys 0.61.2", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", @@ -3982,9 +3982,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -4110,9 +4110,9 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07ae2ce76ce46d26972046ae67d1f65ddbf5bc1290d3abaee84b7ab159564c4" +checksum = "f79d5eea2a18c2cf7318e327bb3ead546514ea3bbea5f42c4e0cc0a89df2b46d" dependencies = [ "libc", "memmap2", @@ -5323,9 +5323,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96acc025cc0039bf828cbf4efdcb485eb0eda88da7ec805fb6ec9159a340b73" +checksum = "7393aaf472d72417bb7b3366addc225ae4b65f64ceef6ec975dd2891e18b08cf" dependencies = [ "allocative", "ctrlc", @@ -5342,9 +5342,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae24e54786d92d4aa4b46098eed2594cd6ed51d2a47a36ff8ae3adfa232d958" +checksum = "323a1fc669d513d1ad83a2fb1d1a589dfb502c9c626993a3313a834361bc64b5" dependencies = [ "quote", "syn 2.0.109", @@ -5594,15 +5594,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/Cargo.toml b/Cargo.toml index 122c68d72..26b3b6b06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,8 +52,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_types = { version = "0.0.111", path = "crates/brk_types" } brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] } brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" } -# byteview = "=0.6.1" -byteview = "~0.8.0" +byteview = "=0.6.1" +# byteview = "~0.8.0" derive_deref = "1.1.1" fjall2 = { version = "2.11.5", package = "brk_fjall" } # fjall2 = { path = "../fjall2", package = "brk_fjall" } @@ -61,7 +61,7 @@ fjall2 = { version = "2.11.5", package = "brk_fjall" } fjall3 = { version = "=3.0.0-pre.4", package = "fjall" } # fjall3 = { path = "../fjall3", package = "brk_fjall" } # fjall3 = { git = "https://github.com/fjall-rs/fjall.git", rev = "bb15057500dce3115d7644d268b9deeaa895b431", package = "fjall" } -jiff = "0.2.15" +jiff = "0.2.16" log = "0.4.28" minreq = { version = "2.14.1", features = ["https", "serde_json"] } parking_lot = "0.12.5" @@ -75,7 +75,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] } sonic-rs = "0.5.6" tokio = { version = "1.48.0", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] } -vecdb = { version = "0.3.8", features = ["derive"] } +vecdb = { version = "0.3.11", features = ["derive"] } zerocopy = { version = "0.8.27", features = ["derive"] } [workspace.metadata.release] diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 3b33e4b9c..1e3d71514 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -18,13 +18,13 @@ use rayon::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended}; mod indexes; -// mod stores_v2; -mod stores_v3; +mod stores_v2; +// mod stores_v3; mod vecs; pub use indexes::*; -// pub use stores_v2::*; -pub use stores_v3::*; +pub use stores_v2::*; +// pub use stores_v3::*; pub use vecs::*; // One version for all data sources diff --git a/crates/brk_indexer/src/stores_v2.rs b/crates/brk_indexer/src/stores_v2.rs index 6b03e32d7..199897c31 100644 --- a/crates/brk_indexer/src/stores_v2.rs +++ b/crates/brk_indexer/src/stores_v2.rs @@ -1,13 +1,13 @@ use std::{fs, path::Path}; use brk_error::Result; -use brk_store::{AnyStore, StoreFjallV2 as Store}; +use brk_store::{AnyStore, Mode, StoreFjallV2 as Store, Type}; use brk_types::{ AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint, AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout, }; -use fjall2::{PersistMode, TransactionalKeyspace}; +use fjall2::{Keyspace, PersistMode}; use rayon::prelude::*; use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended}; @@ -17,7 +17,7 @@ use super::Vecs; #[derive(Clone)] pub struct Stores { - pub keyspace: TransactionalKeyspace, + pub keyspace: Keyspace, pub addressbyteshash_to_typeindex: Store, pub blockhashprefix_to_height: Store, @@ -48,23 +48,47 @@ impl Stores { Ok(Self { keyspace: keyspace.clone(), - height_to_coinbase_tag: Store::import(keyspace_ref, path, "h2c", version, true)?, - addressbyteshash_to_typeindex: Store::import(keyspace_ref, path, "a2t", version, true)?, - blockhashprefix_to_height: Store::import(keyspace_ref, path, "b2h", version, true)?, - txidprefix_to_txindex: Store::import(keyspace_ref, path, "t2t", version, true)?, + height_to_coinbase_tag: Store::import( + keyspace_ref, + path, + "h2c", + version, + Mode::UniquePushOnly(Type::Sequential), + )?, + addressbyteshash_to_typeindex: Store::import( + keyspace_ref, + path, + "a2t", + version, + Mode::UniquePushOnly(Type::Random), + )?, + blockhashprefix_to_height: Store::import( + keyspace_ref, + path, + "b2h", + version, + Mode::UniquePushOnly(Type::Random), + )?, + txidprefix_to_txindex: Store::import( + keyspace_ref, + path, + "t2t", + version, + Mode::UniquePushOnly(Type::Random), + )?, addresstype_to_addressindex_and_txindex: Store::import( keyspace_ref, path, "aat", version, - false, + Mode::VecLike, )?, addresstype_to_addressindex_and_unspentoutpoint: Store::import( keyspace_ref, path, "aau", version, - false, + Mode::VecLike, )?, }) } diff --git a/crates/brk_indexer/src/stores_v3.rs b/crates/brk_indexer/src/stores_v3.rs index 6ed119f07..c8aa60343 100644 --- a/crates/brk_indexer/src/stores_v3.rs +++ b/crates/brk_indexer/src/stores_v3.rs @@ -1,7 +1,7 @@ use std::{fs, path::Path}; use brk_error::Result; -use brk_store::{AnyStore, StoreFjallV3 as Store}; +use brk_store::{AnyStore, Kind3, Mode3, StoreFjallV3 as Store}; use brk_types::{ AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint, AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex, @@ -24,8 +24,8 @@ pub struct Stores { pub height_to_coinbase_tag: Store, pub txidprefix_to_txindex: Store, pub addresstype_to_addressindex_and_txindex: Store, - pub addresstype_to_addressindex_and_unspentoutpoint: - Store, + // pub addresstype_to_addressindex_and_unspentoutpoint: + // Store, } impl Stores { @@ -53,49 +53,49 @@ impl Stores { path, "height_to_coinbase_tag", version, - true, - true, + Mode3::PushOnly, + Kind3::Sequential, )?, addressbyteshash_to_typeindex: Store::import( database_ref, path, "addressbyteshash_to_typeindex", version, - true, - false, + Mode3::PushOnly, + Kind3::Random, )?, blockhashprefix_to_height: Store::import( database_ref, path, "blockhashprefix_to_height", version, - true, - false, + Mode3::PushOnly, + Kind3::Random, )?, txidprefix_to_txindex: Store::import( database_ref, path, "txidprefix_to_txindex", version, - true, - false, + Mode3::PushOnly, + Kind3::Random, )?, addresstype_to_addressindex_and_txindex: Store::import( database_ref, path, "addresstype_to_addressindex_and_txindex", version, - false, - false, - )?, - addresstype_to_addressindex_and_unspentoutpoint: Store::import( - database_ref, - path, - "addresstype_to_addressindex_and_unspentoutpoint", - version, - false, - false, + Mode3::PushOnly, + Kind3::Vec, )?, + // addresstype_to_addressindex_and_unspentoutpoint: Store::import( + // database_ref, + // path, + // "addresstype_to_addressindex_and_unspentoutpoint", + // version, + // Mode3::Any, + // Kind3::Vec, + // )?, }) } @@ -117,7 +117,7 @@ impl Stores { &mut self.height_to_coinbase_tag, &mut self.txidprefix_to_txindex, &mut self.addresstype_to_addressindex_and_txindex, - &mut self.addresstype_to_addressindex_and_unspentoutpoint, + // &mut self.addresstype_to_addressindex_and_unspentoutpoint, ] .into_par_iter() // Changed from par_iter_mut() .try_for_each(|store| store.commit(height))?; @@ -134,7 +134,7 @@ impl Stores { &self.height_to_coinbase_tag, &self.txidprefix_to_txindex, &self.addresstype_to_addressindex_and_txindex, - &self.addresstype_to_addressindex_and_unspentoutpoint, + // &self.addresstype_to_addressindex_and_unspentoutpoint, ] .into_iter() } @@ -149,9 +149,9 @@ impl Stores { && self.txidprefix_to_txindex.is_empty()? && self.height_to_coinbase_tag.is_empty()? && self.addresstype_to_addressindex_and_txindex.is_empty()? - && self - .addresstype_to_addressindex_and_unspentoutpoint - .is_empty()? + // && self + // .addresstype_to_addressindex_and_unspentoutpoint + // .is_empty()? { return Ok(()); } @@ -351,13 +351,13 @@ impl Stores { ); let outpoint = OutPoint::new(txindex, vout); - self.addresstype_to_addressindex_and_unspentoutpoint.remove( - AddressTypeAddressIndexOutPoint::from(( - addresstype, - addressindex, - outpoint, - )), - ); + // self.addresstype_to_addressindex_and_unspentoutpoint.remove( + // AddressTypeAddressIndexOutPoint::from(( + // addresstype, + // addressindex, + // outpoint, + // )), + // ); }); // Add back outputs that were spent after the rollback point @@ -394,14 +394,14 @@ impl Stores { )), ); - self.addresstype_to_addressindex_and_unspentoutpoint.insert( - AddressTypeAddressIndexOutPoint::from(( - addresstype, - addressindex, - outpoint, - )), - Unit, - ); + // self.addresstype_to_addressindex_and_unspentoutpoint.insert( + // AddressTypeAddressIndexOutPoint::from(( + // addresstype, + // addressindex, + // outpoint, + // )), + // Unit, + // ); } } }); diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index 652af75ff..d327b94b4 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> { }); Builder::from_env(Env::default().default_filter_or( - // "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", - "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", + // "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off", )) .format(move |buf, record| { let date_time = Timestamp::now() diff --git a/crates/brk_store/src/fjall_v2/meta.rs b/crates/brk_store/src/fjall_v2/meta.rs index 8e5e91071..8bcf063eb 100644 --- a/crates/brk_store/src/fjall_v2/meta.rs +++ b/crates/brk_store/src/fjall_v2/meta.rs @@ -5,7 +5,7 @@ use std::{ use brk_error::Result; use brk_types::Version; -use fjall2::{PersistMode, TransactionalKeyspace, TransactionalPartitionHandle}; +use fjall2::{Keyspace, PartitionHandle, PersistMode}; use super::Height; @@ -18,13 +18,13 @@ pub struct StoreMeta { impl StoreMeta { pub fn checked_open( - keyspace: &TransactionalKeyspace, + keyspace: &Keyspace, path: &Path, version: Version, open_partition_handle: F, - ) -> Result<(Self, TransactionalPartitionHandle)> + ) -> Result<(Self, PartitionHandle)> where - F: Fn() -> Result, + F: Fn() -> Result, { fs::create_dir_all(path)?; diff --git a/crates/brk_store/src/fjall_v2/mod.rs b/crates/brk_store/src/fjall_v2/mod.rs index 7bab382af..4bc097a72 100644 --- a/crates/brk_store/src/fjall_v2/mod.rs +++ b/crates/brk_store/src/fjall_v2/mod.rs @@ -3,10 +3,7 @@ use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path}; use brk_error::Result; use brk_types::{Height, Version}; use byteview6::ByteView; -use fjall2::{ - InnerItem, PartitionCreateOptions, TransactionalKeyspace, TransactionalPartitionHandle, - ValueType, -}; +use fjall2::{InnerItem, Keyspace, PartitionCreateOptions, PartitionHandle, ValueType}; use rustc_hash::{FxHashMap, FxHashSet}; use crate::any::AnyStore; @@ -19,16 +16,17 @@ use meta::*; pub struct StoreFjallV2 { meta: StoreMeta, name: &'static str, - keyspace: TransactionalKeyspace, - partition: TransactionalPartitionHandle, + keyspace: Keyspace, + partition: PartitionHandle, puts: FxHashMap, dels: FxHashSet, + mode: Mode, } const MAJOR_FJALL_VERSION: Version = Version::TWO; -pub fn open_keyspace(path: &Path) -> fjall2::Result { - fjall2::Config::new(path.join("fjall")).open_transactional() +pub fn open_keyspace(path: &Path) -> fjall2::Result { + fjall2::Config::new(path.join("fjall")).open() } impl StoreFjallV2 @@ -38,14 +36,14 @@ where ByteView: From + From, { fn open_partition_handle( - keyspace: &TransactionalKeyspace, + keyspace: &Keyspace, name: &str, - bloom_filters: bool, - ) -> Result { + mode: Mode, + ) -> Result { let mut options = PartitionCreateOptions::default().manual_journal_persist(true); - if bloom_filters { - options = options.bloom_filter_bits(Some(5)); + if mode.is_unique_push_only() { + options = options.bloom_filter_bits(Some(7)); } else { options = options.bloom_filter_bits(None); } @@ -54,11 +52,11 @@ where } pub fn import( - keyspace: &TransactionalKeyspace, + keyspace: &Keyspace, path: &Path, name: &str, version: Version, - bloom_filters: bool, + mode: Mode, ) -> Result { fs::create_dir_all(path)?; @@ -67,7 +65,7 @@ where &path.join(format!("meta/{name}")), MAJOR_FJALL_VERSION + version, || { - Self::open_partition_handle(keyspace, name, bloom_filters).inspect_err(|e| { + Self::open_partition_handle(keyspace, name, mode).inspect_err(|e| { eprintln!("{e}"); eprintln!("Delete {path:?} and try again"); }) @@ -81,6 +79,7 @@ where partition, puts: FxHashMap::default(), dels: FxHashSet::default(), + mode, }) } @@ -99,16 +98,12 @@ where } pub fn is_empty(&self) -> Result { - self.keyspace - .read_tx() - .is_empty(&self.partition) - .map_err(|e| e.into()) + self.partition.is_empty().map_err(|e| e.into()) } pub fn iter(&self) -> impl Iterator { - self.keyspace - .read_tx() - .iter(&self.partition) + self.partition + .iter() .map(|res| res.unwrap()) .map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v)))) } @@ -177,6 +172,18 @@ where return Ok(()); } + // if self.mode.is_unique_push_only() { + // if !self.dels.is_empty() { + // unreachable!(); + // } + // let mut puts = mem::take(&mut self.puts).into_iter().collect::>(); + // puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); + // dbg!(&puts); + // self.partition.ingest( + // puts.into_iter() + // .map(|(k, v)| (ByteView::from(k), ByteView::from(v))), + // )?; + // } else { let mut items = mem::take(&mut self.puts) .into_iter() .map(|(key, value)| Item::Value { key, value }) @@ -188,10 +195,11 @@ where .collect::>(); items.sort_unstable(); - self.keyspace.inner().batch().commit_partition( - self.partition.inner(), + self.keyspace.batch().commit_partition( + &self.partition, items.into_iter().map(InnerItem::from).collect::>(), )?; + // } Ok(()) } @@ -271,3 +279,25 @@ where } } } + +#[derive(Debug, Clone, Copy)] +pub enum Mode { + VecLike, + UniquePushOnly(Type), +} + +#[derive(Debug, Clone, Copy)] +pub enum Type { + Random, + Sequential, +} + +impl Mode { + pub fn is_vec_like(&self) -> bool { + matches!(*self, Self::VecLike) + } + + pub fn is_unique_push_only(&self) -> bool { + matches!(*self, Self::UniquePushOnly(_)) + } +} diff --git a/crates/brk_store/src/fjall_v3/mod.rs b/crates/brk_store/src/fjall_v3/mod.rs index eeca9cbb2..8c15066c0 100644 --- a/crates/brk_store/src/fjall_v3/mod.rs +++ b/crates/brk_store/src/fjall_v3/mod.rs @@ -26,6 +26,8 @@ pub struct StoreFjallV3 { keyspace: Keyspace, puts: FxHashMap, dels: FxHashSet, + mode: Mode3, + kind: Kind3, } const MAJOR_FJALL_VERSION: Version = Version::new(3); @@ -47,8 +49,8 @@ where path: &Path, name: &str, version: Version, - bloom_filters: bool, - sequential: bool, + mode: Mode3, + kind: Kind3, ) -> Result { fs::create_dir_all(path)?; @@ -57,7 +59,7 @@ where &path.join(format!("meta/{name}")), MAJOR_FJALL_VERSION + version, || { - Self::open_keyspace(database, name, bloom_filters, sequential).inspect_err(|e| { + Self::open_keyspace(database, name, mode, kind).inspect_err(|e| { eprintln!("{e}"); eprintln!("Delete {path:?} and try again"); }) @@ -71,29 +73,31 @@ where keyspace, puts: FxHashMap::default(), dels: FxHashSet::default(), + mode, + kind, }) } fn open_keyspace( database: &Database, name: &str, - bloom_filters: bool, - sequential: bool, + _mode: Mode3, + kind: Kind3, ) -> Result { let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true); - if bloom_filters { + if kind.is_not_vec() { options = options.filter_policy(FilterPolicy::new(&[ FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)), - FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.5)), - FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(5.0)), - FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(2.5)), + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)), + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(8.0)), + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.0)), ])); } else { options = options.filter_policy(FilterPolicy::disabled()); } - if sequential { + if kind.is_sequential() { options = options .filter_block_pinning_policy(PinningPolicy::all(false)) .index_block_pinning_policy(PinningPolicy::all(false)); @@ -109,11 +113,6 @@ where { if let Some(v) = self.puts.get(key) { Ok(Some(Cow::Borrowed(v))) - // } else if let Some(slice) = self - // .database - // .read_tx() - // .get(&self.keyspace, ByteView::from(key))? - // { } else if let Some(slice) = self.keyspace.get(ByteView::from(key))? { Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) } else { @@ -123,9 +122,6 @@ where #[inline] pub fn is_empty(&self) -> Result { - // self.database - // .read_tx() - // .is_empty(&self.keyspace) self.keyspace.is_empty().map_err(|e| e.into()) } @@ -189,25 +185,38 @@ where return Ok(()); } - let mut batch = self.database.batch(); - // let mut batch = self.database.inner().batch(); - let mut items = mem::take(&mut self.puts) - .into_iter() - .map(|(key, value)| Item::Value { key, value }) - .chain( - mem::take(&mut self.dels) - .into_iter() - .map(|key| Item::Tomb(key)), - ) - .collect::>(); - items.sort_unstable(); - items.into_iter().for_each(|item| match item { - Item::Value { key, value } => { - batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value)) + if self.mode.is_push_only() { + if !self.dels.is_empty() { + unreachable!(); } - Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)), - }); - batch.commit()?; + let mut puts = mem::take(&mut self.puts).into_iter().collect::>(); + puts.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); + // dbg!(&puts); + self.keyspace.ingest( + puts.into_iter() + .map(|(k, v)| (ByteView::from(k), ByteView::from(v))), + )?; + } else { + let mut batch = self.database.batch(); + // let mut batch = self.database.inner().batch(); + let mut items = mem::take(&mut self.puts) + .into_iter() + .map(|(key, value)| Item::Value { key, value }) + .chain( + mem::take(&mut self.dels) + .into_iter() + .map(|key| Item::Tomb(key)), + ) + .collect::>(); + items.sort_unstable(); + items.into_iter().for_each(|item| match item { + Item::Value { key, value } => { + batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value)) + } + Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)), + }); + batch.commit()?; + } // batch.ingest( // items @@ -304,3 +313,40 @@ impl Item { // } // } } + +#[derive(Debug, Clone, Copy)] +pub enum Mode3 { + Any, + PushOnly, +} + +impl Mode3 { + pub fn is_any(&self) -> bool { + matches!(*self, Self::Any) + } + + pub fn is_push_only(&self) -> bool { + matches!(*self, Self::PushOnly) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Kind3 { + Random, + Sequential, + Vec, +} + +impl Kind3 { + pub fn is_sequential(&self) -> bool { + matches!(*self, Self::Sequential) + } + + pub fn is_random(&self) -> bool { + matches!(*self, Self::Random) + } + + pub fn is_not_vec(&self) -> bool { + !matches!(*self, Self::Vec) + } +} diff --git a/crates/brk_types/src/addressbyteshash.rs b/crates/brk_types/src/addressbyteshash.rs index 6363f4327..ac62876cf 100644 --- a/crates/brk_types/src/addressbyteshash.rs +++ b/crates/brk_types/src/addressbyteshash.rs @@ -2,6 +2,8 @@ use byteview::ByteView; use derive_deref::Deref; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::copy_first_8bytes; + use super::AddressBytes; #[derive( @@ -31,7 +33,7 @@ impl From<&AddressBytes> for AddressBytesHash { impl From for AddressBytesHash { #[inline] fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() + Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap())) } } @@ -45,6 +47,6 @@ impl From for ByteView { impl From<&AddressBytesHash> for ByteView { #[inline] fn from(value: &AddressBytesHash) -> Self { - Self::new(value.as_bytes()) + Self::new(&value.0.to_be_bytes()) } } diff --git a/crates/brk_types/src/blockhashprefix.rs b/crates/brk_types/src/blockhashprefix.rs index 583640785..e5a2115b9 100644 --- a/crates/brk_types/src/blockhashprefix.rs +++ b/crates/brk_types/src/blockhashprefix.rs @@ -40,14 +40,7 @@ impl From<&BlockHash> for BlockHashPrefix { impl From for BlockHashPrefix { #[inline] fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() - } -} - -impl From<&BlockHashPrefix> for ByteView { - #[inline] - fn from(value: &BlockHashPrefix) -> Self { - Self::new(value.as_bytes()) + Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap())) } } @@ -57,3 +50,10 @@ impl From for ByteView { Self::from(&value) } } + +impl From<&BlockHashPrefix> for ByteView { + #[inline] + fn from(value: &BlockHashPrefix) -> Self { + Self::from(value.to_be_bytes()) + } +} diff --git a/crates/brk_types/src/txidprefix.rs b/crates/brk_types/src/txidprefix.rs index 4029539d0..e6e644575 100644 --- a/crates/brk_types/src/txidprefix.rs +++ b/crates/brk_types/src/txidprefix.rs @@ -40,14 +40,7 @@ impl From<&Txid> for TxidPrefix { impl From for TxidPrefix { #[inline] fn from(value: ByteView) -> Self { - Self::read_from_bytes(&value).unwrap() - } -} - -impl From<&TxidPrefix> for ByteView { - #[inline] - fn from(value: &TxidPrefix) -> Self { - Self::new(value.as_bytes()) + Self(u64::from_be_bytes(copy_first_8bytes(&value).unwrap())) } } @@ -58,6 +51,13 @@ impl From for ByteView { } } +impl From<&TxidPrefix> for ByteView { + #[inline] + fn from(value: &TxidPrefix) -> Self { + Self::from(value.to_be_bytes()) + } +} + impl From<[u8; 8]> for TxidPrefix { #[inline] fn from(value: [u8; 8]) -> Self {