diff --git a/Cargo.lock b/Cargo.lock index 534b8a9e7..4913a9beb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,7 +75,9 @@ dependencies = [ "color-eyre", "derive_deref", "exit", + "fjall", "jiff", + "rapidhash", "rayon", "snkrj", "storable_vec", @@ -317,6 +319,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -333,6 +345,20 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core 0.9.10", +] + [[package]] name = "derive_deref" version = "1.1.1" @@ -355,12 +381,46 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "exit" version = "0.1.0" @@ -378,6 +438,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fjall" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80910a26e4fb5e5393ff64d293602ac1ade56cf4d14d244c02a7d4ddcd5f10bc" +dependencies = [ + "byteorder", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "fs2" version = "0.4.3" @@ -405,6 +487,18 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "guardian" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "493913a18c0d7bebb75127a26a432162c59edbe06f6cf712001e3e769345e8b5" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hex-conservative" version = "0.2.1" @@ -503,6 +597,12 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "lock_api" version = "0.4.12" @@ -519,6 +619,35 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +[[package]] +name = "lsm-tree" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d858efa63a32a286a06aa3968f3ed09811d556697e9f9804aa72ea679ed3b83" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + [[package]] name = "memchr" version = "2.7.4" @@ -534,6 +663,12 @@ dependencies = [ "libc", ] +[[package]] +name = "min-max-heap" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -595,7 +730,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.6", ] [[package]] @@ -607,11 +742,42 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.8", + "smallvec", + "windows-targets", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -651,6 +817,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick_cache" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d7c94f8935a9df96bb6380e8592c70edf497a643f94bd23b2f76b399385dbf4" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "quote" version = "1.0.38" @@ -690,6 +866,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rapidhash" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3cf30fb8a0540827f3e22514836533403dece9ac517631160a1fb36d0dd5d6" + [[package]] name = "rayon" version = "1.10.0" @@ -719,12 +901,40 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.8.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "ryu" version = "1.0.18" @@ -779,6 +989,12 @@ dependencies = [ "cc", ] +[[package]] +name = "self_cell" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe" + [[package]] name = "serde" version = "1.0.217" @@ -839,6 +1055,12 @@ dependencies = [ "sanakirja", ] +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + [[package]] name = "storable_vec" version = "0.1.2" @@ -890,6 +1112,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -973,6 +1209,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-log" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f90495556d09c3026f7f3897f8a7db59c8c701082e32dcf58c2319062ae1eb0" +dependencies = [ + "byteorder", + "log", + "min-max-heap", + "path-absolutize", + "quick_cache", + "rustc-hash", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1074,6 +1332,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/bindex/Cargo.toml b/bindex/Cargo.toml index c3957119a..f716175b6 100644 --- a/bindex/Cargo.toml +++ b/bindex/Cargo.toml @@ -9,6 +9,7 @@ biter = { path = "../biter" } color-eyre = "0.6.3" derive_deref = "1.1.1" exit = { path = "../exit" } +fjall = "2.5.0" jiff = "0.1.24" rapidhash = "1.3.0" rayon = "1.10.0" diff --git a/bindex/src/lib.rs b/bindex/src/lib.rs index 17584650a..f04a6cdcd 100644 --- a/bindex/src/lib.rs +++ b/bindex/src/lib.rs @@ -3,7 +3,8 @@ use std::{ io::{Read, Write}, path::Path, str::FromStr, - thread::{self}, + thread::{self, sleep}, + time::Duration, }; use biter::{ @@ -12,12 +13,13 @@ use biter::{ }; use color_eyre::eyre::{eyre, ContextCompat}; use exit::Exit; +use fjall::{PersistMode, ReadTransaction, TransactionalKeyspace}; use rayon::prelude::*; mod storage; mod structs; -use storage::{Stores, Vecs}; +use storage::{Partitions, Vecs}; use structs::{ Addressbytes, AddressbytesPrefix, Addressindex, Addresstype, Amount, BlockHashPrefix, Height, Timestamp, TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, @@ -35,17 +37,17 @@ impl Indexer { let mut vecs = Vecs::import(&indexes_dir.join("vecs"))?; - let open_stores = || Stores::open(&indexes_dir.join("stores")); - let stores = open_stores()?; + let keyspace = fjall::Config::new(indexes_dir.join("fjall")).open_transactional()?; + let mut parts = Partitions::import(&keyspace, &exit)?; + let rtx = keyspace.read_tx(); let mut height = vecs .min_height() .unwrap_or_default() - .min(stores.min_height()) + .min(parts.min_height()) .and_then(|h| h.checked_sub(UNSAFE_BLOCKS)) .map(Height::from) .unwrap_or_default(); - // let mut height = Height::default(); let mut txindex_global = vecs.height_to_first_txindex.get_or_default(height)?; let mut txinindex_global = vecs.height_to_first_txinindex.get_or_default(height)?; @@ -64,31 +66,37 @@ impl Indexer { let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?; let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?; - let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> { + let export = |keyspace: &TransactionalKeyspace, + rtx: ReadTransaction, + parts: &mut Partitions, + vecs: &mut Vecs, + height: Height| + -> color_eyre::Result<()> { println!("Exporting..."); - if height >= Height::from(400_000_u32) { - pause(); - // println!("Flushing vecs..."); - } + drop(rtx); exit.block(); thread::scope(|scope| -> color_eyre::Result<()> { let vecs_handle = scope.spawn(|| vecs.flush(height)); - let stores_handle = scope.spawn(|| stores.export(height)); + + parts.write(keyspace, height)?; + keyspace.persist(PersistMode::SyncAll)?; + vecs_handle.join().unwrap()?; - stores_handle.join().unwrap()?; Ok(()) })?; exit.unblock(); + Ok(()) }; - let mut stores_opt = Some(stores); + // let mut stores_opt = Some(stores); + let mut rtx_opt = Some(rtx); - biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc) + biter::new(bitcoin_dir, Some(height.into()), None, rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { println!("Processing block {_height}..."); @@ -96,27 +104,28 @@ impl Indexer { height = Height::from(_height); let timestamp = Timestamp::try_from(block.header.time)?; - let mut stores = stores_opt.take().context("option should have wtx")?; + // let mut stores = stores_opt.take().context("option should have store")?; + let rtx = rtx_opt.take().context("option should have rtx")?; if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? { if &blockhash != saved_blockhash.as_ref() { todo!("Rollback not implemented"); - // parts.rollback_from(&mut wtx, height, &exit)?; + // parts.rollback_from(&mut rtx, height, &exit)?; } } let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?; - if stores + if parts .blockhash_prefix_to_height - .get(&blockhash_prefix)? - .is_some_and(|prev_height| *prev_height != height) + .get(&rtx, &blockhash_prefix)? + .is_some_and(|prev_height| prev_height != height) { dbg!(blockhash); return Err(eyre!("Collision, expect prefix to need be set yet")); } - stores + parts .blockhash_prefix_to_height .insert_if_needed(blockhash_prefix, height, height); @@ -194,9 +203,9 @@ impl Indexer { let txid_prefix = TxidPrefix::try_from(&txid)?; let prev_txindex_slice_opt = - if check_collisions && stores.txid_prefix_to_txindex.needs(height) { + if check_collisions && parts.txid_prefix_to_txindex.needs(height) { // Should only find collisions for two txids (duplicates), see below - stores.txid_prefix_to_txindex.get(&txid_prefix)?.cloned() + parts.txid_prefix_to_txindex.get(&rtx, &txid_prefix)? } else { None }; @@ -235,14 +244,14 @@ impl Indexer { return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin)))); } - let prev_txindex = if let Some(txindex) = stores + let prev_txindex = if let Some(txindex) = parts .txid_prefix_to_txindex - .get(&TxidPrefix::try_from(&outpoint.txid)?)? + .get(&rtx, &TxidPrefix::try_from(&outpoint.txid)?)? .and_then(|txindex| { // Checking if not finding txindex from the future - (txindex < &txindex_global).then_some(txindex) + (txindex < txindex_global).then_some(txindex) }) { - *txindex + txindex } else { // dbg!(txindex_global + block_txindex, txindex, txin, vin); return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin)))); @@ -312,11 +321,10 @@ impl Indexer { }); let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| { - stores + parts .addressbytes_prefix_to_addressindex - .get(&AddressbytesPrefix::from((addressbytes, addresstype))) + .get(&rtx, &AddressbytesPrefix::from((addressbytes, addresstype))) .unwrap() - .cloned() // Checking if not in the future .and_then(|addressindex_local| { (addressindex_local < addressindex_global).then_some(addressindex_local) @@ -346,7 +354,7 @@ impl Indexer { if (vecs.addressindex_to_addresstype.hasnt(addressindex) && addresstype != prev_addresstype) - || (stores.addressbytes_prefix_to_addressindex.needs(height) + || (parts.addressbytes_prefix_to_addressindex.needs(height) && prev_addressbytes != addressbytes) { let txid = tx.compute_txid(); @@ -494,9 +502,9 @@ impl Indexer { let addressbytes_prefix = addressbytes_prefix.unwrap(); already_added_addressbytes_prefix - .insert(addressbytes_prefix.clone(), addressindex); + .insert(addressbytes_prefix, addressindex); - stores.addressbytes_prefix_to_addressindex.insert_if_needed( + parts.addressbytes_prefix_to_addressindex.insert_if_needed( addressbytes_prefix, addressindex, height, @@ -580,7 +588,7 @@ impl Indexer { match prev_txindex_opt { None => { - stores + parts .txid_prefix_to_txindex .insert_if_needed(txid_prefix, txindex, height); } @@ -646,16 +654,24 @@ impl Indexer { let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active(); if should_snapshot { - export(stores, &mut vecs, height)?; - stores_opt.replace(open_stores()?); + export(&keyspace, rtx, &mut parts, &mut vecs, height)?; + rtx_opt.replace(keyspace.read_tx()); } else { - stores_opt.replace(stores); + rtx_opt.replace(rtx); } Ok(()) })?; - export(stores_opt.take().context("option should have wtx")?, &mut vecs, height)?; + export( + &keyspace, + rtx_opt.take().context("option should have wtx")?, + &mut parts, + &mut vecs, + height, + )?; + + sleep(Duration::from_millis(100)); Ok(()) } diff --git a/bindex/src/storage/canopy/database.rs b/bindex/src/storage/canopy/database.rs new file mode 100644 index 000000000..fd871d742 --- /dev/null +++ b/bindex/src/storage/canopy/database.rs @@ -0,0 +1,50 @@ +use std::{ + ops::{Deref, DerefMut}, + time::Duration, +}; + +use canopydb::{Database as CanopyDatabase, DbOptions, Error, WriteTransaction}; + +use super::Environment; + +#[derive(Debug)] +pub struct Database { + db: CanopyDatabase, + // pub wtx: WriteTransaction, +} +impl Deref for Database { + type Target = CanopyDatabase; + fn deref(&self) -> &Self::Target { + &self.db + } +} +impl DerefMut for Database { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.db + } +} + +impl Database { + pub fn new(environment: &Environment, name: &str) -> color_eyre::Result { + let mut options = DbOptions::default(); + options.use_wal = false; + options.checkpoint_interval = Duration::from_secs(u64::MAX); + options.checkpoint_target_size = usize::MAX; + options.throttle_memory_limit = usize::MAX; + options.stall_memory_limit = usize::MAX; + options.write_txn_memory_limit = usize::MAX; + + let db = environment.get_or_create_database_with(name, options)?; + + Ok(Self { + // wtx: db.begin_write()?, + db, + }) + } + + pub fn flush(&mut self) -> Result<(), Error> { + // drop(blockhash_prefix_to_height_tree); + // blockhash_prefix_to_height_tx_opt.take().map(|tx| tx.commit()); + self.checkpoint() + } +} diff --git a/bindex/src/storage/canopy/environment.rs b/bindex/src/storage/canopy/environment.rs new file mode 100644 index 000000000..562e159d9 --- /dev/null +++ b/bindex/src/storage/canopy/environment.rs @@ -0,0 +1,20 @@ +use std::path::Path; + +use canopydb::{EnvOptions, Environment as CanopyEnvironment}; +use derive_deref::{Deref, DerefMut}; + +#[derive(Debug, Deref, DerefMut)] +pub struct Environment(CanopyEnvironment); + +impl Environment { + pub fn new(path: &Path) -> color_eyre::Result { + let mut options = EnvOptions::new(path); + // options.use_mmap = true; + options.disable_fsync = true; + options.wal_new_file_on_checkpoint = false; + options.wal_background_sync_interval = None; + options.wal_write_batch_memory_limit = usize::MAX; + + Ok(Self(CanopyEnvironment::with_options(options)?)) + } +} diff --git a/bindex/src/storage/canopy/mod.rs b/bindex/src/storage/canopy/mod.rs new file mode 100644 index 000000000..46cb95a00 --- /dev/null +++ b/bindex/src/storage/canopy/mod.rs @@ -0,0 +1,9 @@ +mod database; +mod environment; +// mod transaction; +mod tree; + +pub use database::*; +pub use environment::*; +// pub use transaction::*; +pub use tree::*; diff --git a/bindex/src/storage/canopy/transaction.rs b/bindex/src/storage/canopy/transaction.rs new file mode 100644 index 000000000..d1a6ca13c --- /dev/null +++ b/bindex/src/storage/canopy/transaction.rs @@ -0,0 +1,19 @@ +use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction}; + +use super::{Database, Tree}; + +#[derive(Debug)] +pub struct Transaction<'a, K, V> { + tx: WriteTransaction, + tree: Tree<'a, K, V>, +} + +impl<'a, K, V> Transaction<'a, K, V> { + pub fn new(db: &Database) -> color_eyre::Result { + let tx = db.begin_write()?; + + let tree = Tree::new(&tx)?; + + Ok(Self { tx, tree }) + } +} diff --git a/bindex/src/storage/canopy/tree.rs b/bindex/src/storage/canopy/tree.rs new file mode 100644 index 000000000..7809835ea --- /dev/null +++ b/bindex/src/storage/canopy/tree.rs @@ -0,0 +1,84 @@ +use std::{ + fmt::Debug, + marker::PhantomData, + ops::{Deref, DerefMut}, +}; + +use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction}; +use color_eyre::eyre::eyre; + +#[derive(Debug)] +pub struct Tree<'a, K, V> { + tree: CanopyTree<'a>, + k: PhantomData, + v: PhantomData, +} +impl<'a, K, V> Deref for Tree<'a, K, V> { + type Target = CanopyTree<'a>; + fn deref(&self) -> &Self::Target { + &self.tree + } +} +impl<'a, K, V> DerefMut for Tree<'a, K, V> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.tree + } +} + +impl<'a, K, V> Tree<'a, K, V> +where + K: Debug + Sized, + V: Debug + Sized + Clone + Copy, +{ + const SIZE_OF_K: usize = size_of::(); + const SIZE_OF_V: usize = size_of::(); + + pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result { + let mut options = TreeOptions::new(); + options.compress_overflow_values = None; + options.fixed_key_len = size_of::() as i8; + options.fixed_value_len = size_of::() as i8; + + Ok(Self { + tree: tx.get_or_create_tree_with(b"tree", options)?, + k: PhantomData, + v: PhantomData, + }) + } + + pub fn get(&self, key: &K) -> color_eyre::Result> { + let slice = self.tree.get(Self::key_as_slice(key))?; + + if slice.is_none() { + return Ok(None); + } + + let slice = slice.unwrap(); + + let (prefix, shorts, suffix) = unsafe { slice.align_to::() }; + + if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() { + dbg!(&key, &prefix, &shorts, &suffix); + return Err(eyre!("align_to issue")); + } + + Ok(Some(shorts[0])) + } + + pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> { + self.tree + .insert(Self::key_as_slice(key), Self::value_as_slice(value)) + } + + fn key_as_slice(key: &K) -> &[u8] { + let data: *const K = key; + let data: *const u8 = data as *const u8; + unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) } + } + + fn value_as_slice(value: &V) -> &[u8] { + let data: *const V = value; + let data: *const u8 = data as *const u8; + unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) } + } +} diff --git a/bindex/src/storage/fjall/base.rs b/bindex/src/storage/fjall/base.rs new file mode 100644 index 000000000..9ed674187 --- /dev/null +++ b/bindex/src/storage/fjall/base.rs @@ -0,0 +1,145 @@ +use std::{collections::BTreeMap, mem}; + +use exit::Exit; +use fjall::{ + PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, + TransactionalPartitionHandle, TxKeyspace, WriteTransaction, +}; + +use crate::structs::{Height, Version}; + +pub struct Partition { + version: Version, + data: TransactionalPartitionHandle, + meta: TransactionalPartitionHandle, + height: Option, + puts: BTreeMap, +} + +impl Partition +where + Key: Into + Ord, + Value: Into + TryFrom + Clone, +{ + pub const VERSION: &str = "version"; + pub const HEIGHT: &str = "height"; + + pub fn import( + keyspace: &TransactionalKeyspace, + name: &str, + version: Version, + exit: &Exit, + ) -> color_eyre::Result { + let data = Self::open_data(keyspace, name)?; + let meta = Self::open_meta(keyspace, name)?; + + let height = if let Some(slice) = meta.get(Self::HEIGHT)? { + Some(Height::try_from(slice)?) + } else { + None + }; + + let mut this = Self { + version, + height, + data, + meta, + puts: BTreeMap::new(), + }; + + if let Some(slice) = this.meta.get(Self::VERSION)? { + if version != Version::try_from(slice)? { + this = this.reset(keyspace, name, exit)?; + } + } + + Ok(this) + } + + fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result { + keyspace.open_partition(&format!("{name}-data"), Self::create_options()) + } + + fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result { + keyspace.open_partition(&format!("{name}-meta"), Self::create_options()) + } + + fn create_options() -> PartitionCreateOptions { + PartitionCreateOptions::default().manual_journal_persist(true) + } + + pub fn has(&self, height: Height) -> bool { + self.height.is_some_and(|self_height| self_height >= height) + } + pub fn needs(&self, height: Height) -> bool { + !self.has(height) + } + + pub fn get<'a>(&self, rtx: &ReadTransaction, key: &'a Key) -> color_eyre::Result> + where + fjall::Slice: std::convert::From<&'a Key>, + >::Error: std::error::Error + Send + Sync, + >::Error: 'static, + { + if let Some(v) = self.puts.get(key) { + return Ok(Some(v.clone())); + } + + if let Some(slice) = rtx.get(&self.data, Slice::from(key))? { + let v_res = Value::try_from(slice); + let v = v_res?; + Ok(Some(v)) + } else { + Ok(None) + } + } + + pub fn insert_if_needed(&mut self, key: Key, value: Value, height: Height) { + if self.needs(height) { + self.puts.insert(key, value); + } + } + + fn update_meta(&self, wtx: &mut WriteTransaction, height: Height) { + wtx.insert(&self.meta, Self::VERSION, self.version()); + wtx.insert(&self.meta, Self::HEIGHT, height); + } + + pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> Result<()> { + if self.has(height) && self.puts.is_empty() { + return Ok(()); + } + + let mut wtx = keyspace.write_tx(); + mem::take(&mut self.puts) + .into_iter() + .for_each(|(key, value)| wtx.insert(&self.data, key, value)); + self.update_meta(&mut wtx, height); + wtx.commit() + } + + pub fn version(&self) -> Version { + self.version + } + + fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result { + exit.block(); + + keyspace.delete_partition(self.data)?; + keyspace.delete_partition(self.meta)?; + + keyspace.persist(PersistMode::SyncAll)?; + + self.data = Self::open_data(keyspace, name)?; + self.meta = Self::open_meta(keyspace, name)?; + self.height = None; + + exit.unblock(); + + Ok(self) + } + + pub fn height(&self) -> Option<&Height> { + self.height.as_ref() + } +} diff --git a/bindex/src/storage/fjall/mod.rs b/bindex/src/storage/fjall/mod.rs new file mode 100644 index 000000000..e458eacc6 --- /dev/null +++ b/bindex/src/storage/fjall/mod.rs @@ -0,0 +1,200 @@ +use std::thread; + +use crate::{structs::Version, AddressbytesPrefix, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex}; + +mod base; +use base::*; +use exit::Exit; +use fjall::{TransactionalKeyspace, TxKeyspace}; + +pub struct Partitions { + pub addressbytes_prefix_to_addressindex: Partition, + pub blockhash_prefix_to_height: Partition, + pub txid_prefix_to_txindex: Partition, +} + +impl Partitions { + pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result { + Ok(Self { + addressbytes_prefix_to_addressindex: Partition::import( + keyspace, + "addressbytes_prefix_to_addressindex", + Version::from(1), + exit, + )?, + blockhash_prefix_to_height: Partition::import( + keyspace, + "blockhash_prefix_to_height", + Version::from(1), + exit, + )?, + txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?, + }) + } + + // pub fn rollback_from( + // &mut self, + // _wtx: &mut WriteTransaction, + // _height: Height, + // _exit: &Exit, + // ) -> color_eyre::Result<()> { + // panic!(); + // let mut txindex = None; + + // wtx.range(self.height_to_blockhash.data(), Slice::from(height)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (height_slice, slice_blockhash) = slice?; + // let blockhash = BlockHash::from_slice(&slice_blockhash)?; + + // wtx.remove(self.height_to_blockhash.data(), height_slice); + + // wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix()); + + // if txindex.is_none() { + // txindex.replace( + // wtx.get(self.height_to_first_txindex.data(), height_slice)? + // .context("for height to have first txindex")?, + // ); + // } + // wtx.remove(self.height_to_first_txindex.data(), height_slice); + // wtx.remove(self.height_to_last_txindex.data(), height_slice); + + // Ok(()) + // })?; + + // let txindex = txindex.context("txindex to not be none by now")?; + + // wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (slice_txindex, slice_txid) = slice?; + // let txindex = Txindex::from(slice_txindex); + // let txid = Txid::from_slice(&slice_txid)?; + + // wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex)); + // wtx.remove(self.txindex_to_height.data(), Slice::from(txindex)); + // wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix()); + + // Ok(()) + // })?; + + // let txoutindex = Txoutindex::from(txindex); + + // let mut addressindexes = BTreeSet::new(); + + // wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..) + // .try_for_each(|slice| -> color_eyre::Result<()> { + // let (txoutindex_slice, _) = slice?; + + // wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice); + + // if let Some(addressindex_slice) = + // wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)? + // { + // wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice); + + // let addressindex = Addressindex::from(addressindex_slice); + // addressindexes.insert(addressindex); + + // let txoutindex = Txoutindex::from(txoutindex_slice); + // let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex)); + + // wtx.remove( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(addresstxoutindex), + // ); + // } + + // Ok(()) + // })?; + + // addressindexes + // .into_iter() + // .filter(|addressindex| { + // let is_empty = wtx + // .prefix( + // self.addressindex_to_txoutindexes.data(), + // Slice::from(*addressindex), + // ) + // .next() + // .is_none(); + // is_empty + // }) + // .try_for_each(|addressindex| -> color_eyre::Result<()> { + // let addressindex_slice = Slice::from(addressindex); + + // let addressbytes = Addressbytes::from( + // wtx.get( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // )? + // .context("addressindex_to_address to have value")?, + // ); + // wtx.remove( + // self.addressbytes_prefix_to_addressindex.data(), + // addressbytes.prefix(), + // ); + // wtx.remove( + // self.addressindex_to_addressbytes.data(), + // &addressindex_slice, + // ); + // wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice); + + // Ok(()) + // })?; + // + + // todo!("clear addresstxoutindexes_out") + // todo!("clear addresstxoutindexes_in") + // todo!("clear zero_txoutindexes") + // todo!("clear txindexvout_to_txoutindex") + + // Ok(()) + // } + + pub fn min_height(&self) -> Option { + [ + self.addressbytes_prefix_to_addressindex.height(), + self.blockhash_prefix_to_height.height(), + self.txid_prefix_to_txindex.height(), + ] + .into_iter() + .min() + .flatten() + .cloned() + } + + pub fn write(&mut self, keyspace: &TxKeyspace, height: Height) -> fjall::Result<()> { + thread::scope(|scope| { + let addressbytes_prefix_to_addressindex_write_handle = + scope.spawn(|| self.addressbytes_prefix_to_addressindex.write(keyspace, height)); + let blockhash_prefix_to_height_write_handle = + scope.spawn(|| self.blockhash_prefix_to_height.write(keyspace, height)); + let txid_prefix_to_txindex_write_handle = + scope.spawn(|| self.txid_prefix_to_txindex.write(keyspace, height)); + + addressbytes_prefix_to_addressindex_write_handle.join().unwrap()?; + blockhash_prefix_to_height_write_handle.join().unwrap()?; + txid_prefix_to_txindex_write_handle.join().unwrap()?; + + Ok(()) + }) + } + + // pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) { + // self.addressbytes_prefix_to_addressindex.update_meta(wtx, height); + // self.blockhash_prefix_to_height.update_meta(wtx, height); + // self.txid_prefix_to_txindex.update_meta(wtx, height); + // } + + // pub fn export(self, height: Height) -> Result<(), snkrj::Error> { + // thread::scope(|scope| { + // vec![ + // scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)), + // scope.spawn(|| self.blockhash_prefix_to_height.export(height)), + // scope.spawn(|| self.txid_prefix_to_txindex.export(height)), + // ] + // .into_iter() + // .try_for_each(|handle| -> Result<(), snkrj::Error> { handle.join().unwrap() }) + // }) + // } +} diff --git a/bindex/src/storage/mod.rs b/bindex/src/storage/mod.rs index 96983bd40..c920d9e38 100644 --- a/bindex/src/storage/mod.rs +++ b/bindex/src/storage/mod.rs @@ -1,5 +1,9 @@ -mod stores; -mod vecs; +// mod canopy; +mod fjall; +// mod sanakirja; +mod storable_vec; -pub use stores::*; -pub use vecs::*; +// pub use canopy::*; +pub use fjall::*; +// pub use sanakirja::*; +pub use storable_vec::*; diff --git a/bindex/src/storage/partitions/mod.rs b/bindex/src/storage/partitions/mod.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/bindex/src/storage/stores/meta.rs b/bindex/src/storage/sanakirja/meta.rs similarity index 100% rename from bindex/src/storage/stores/meta.rs rename to bindex/src/storage/sanakirja/meta.rs diff --git a/bindex/src/storage/stores/mod.rs b/bindex/src/storage/sanakirja/mod.rs similarity index 100% rename from bindex/src/storage/stores/mod.rs rename to bindex/src/storage/sanakirja/mod.rs diff --git a/bindex/src/storage/stores/multi.rs b/bindex/src/storage/sanakirja/multi.rs similarity index 100% rename from bindex/src/storage/stores/multi.rs rename to bindex/src/storage/sanakirja/multi.rs diff --git a/bindex/src/storage/stores/unique.rs b/bindex/src/storage/sanakirja/unique.rs similarity index 100% rename from bindex/src/storage/stores/unique.rs rename to bindex/src/storage/sanakirja/unique.rs diff --git a/bindex/src/storage/vecs/base.rs b/bindex/src/storage/storable_vec/base.rs similarity index 100% rename from bindex/src/storage/vecs/base.rs rename to bindex/src/storage/storable_vec/base.rs diff --git a/bindex/src/storage/vecs/mod.rs b/bindex/src/storage/storable_vec/mod.rs similarity index 100% rename from bindex/src/storage/vecs/mod.rs rename to bindex/src/storage/storable_vec/mod.rs diff --git a/bindex/src/structs/addressindex.rs b/bindex/src/structs/addressindex.rs index 7da1b22c0..eab1f7c97 100644 --- a/bindex/src/structs/addressindex.rs +++ b/bindex/src/structs/addressindex.rs @@ -1,5 +1,6 @@ use derive_deref::{Deref, DerefMut}; use snkrj::{direct_repr, Storable, UnsizedStorable}; +use storable_vec::UnsafeSizedSerDe; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Addressindex(u32); @@ -48,3 +49,15 @@ impl From for usize { value.0 as usize } } + +impl TryFrom for Addressindex { + type Error = storable_vec::Error; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From for fjall::Slice { + fn from(value: Addressindex) -> Self { + Self::new(value.unsafe_as_slice()) + } +} diff --git a/bindex/src/structs/height.rs b/bindex/src/structs/height.rs index 1fdb478c3..717a1cde5 100644 --- a/bindex/src/structs/height.rs +++ b/bindex/src/structs/height.rs @@ -119,3 +119,15 @@ impl TryFrom<&rpc::Client> for Height { Ok((value.get_blockchain_info()?.blocks as usize - 1).into()) } } + +impl TryFrom for Height { + type Error = storable_vec::Error; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From for fjall::Slice { + fn from(value: Height) -> Self { + Self::new(value.unsafe_as_slice()) + } +} diff --git a/bindex/src/structs/prefix.rs b/bindex/src/structs/prefix.rs index 6ae4e0306..cc2e2e15e 100644 --- a/bindex/src/structs/prefix.rs +++ b/bindex/src/structs/prefix.rs @@ -3,10 +3,11 @@ use std::hash::Hasher; use biter::bitcoin::{BlockHash, Txid}; use derive_deref::Deref; use snkrj::{direct_repr, Storable, UnsizedStorable}; +use storable_vec::UnsafeSizedSerDe; use super::{Addressbytes, Addresstype, SliceExtended}; -#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct AddressbytesPrefix([u8; 8]); direct_repr!(AddressbytesPrefix); impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix { @@ -23,8 +24,24 @@ impl From<[u8; 8]> for AddressbytesPrefix { Self(value) } } +impl TryFrom for AddressbytesPrefix { + type Error = color_eyre::Report; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From<&AddressbytesPrefix> for fjall::Slice { + fn from(value: &AddressbytesPrefix) -> Self { + Self::new(value.unsafe_as_slice()) + } +} +impl From for fjall::Slice { + fn from(value: AddressbytesPrefix) -> Self { + Self::from(&value) + } +} -#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockHashPrefix([u8; 8]); direct_repr!(BlockHashPrefix); impl TryFrom<&BlockHash> for BlockHashPrefix { @@ -33,8 +50,24 @@ impl TryFrom<&BlockHash> for BlockHashPrefix { Ok(Self((&value[..]).read_8x_u8()?)) } } +impl TryFrom for BlockHashPrefix { + type Error = color_eyre::Report; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From<&BlockHashPrefix> for fjall::Slice { + fn from(value: &BlockHashPrefix) -> Self { + Self::new(value.unsafe_as_slice()) + } +} +impl From for fjall::Slice { + fn from(value: BlockHashPrefix) -> Self { + Self::from(&value) + } +} -#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct TxidPrefix([u8; 8]); direct_repr!(TxidPrefix); impl TryFrom<&Txid> for TxidPrefix { @@ -43,3 +76,19 @@ impl TryFrom<&Txid> for TxidPrefix { Ok(Self((&value[..]).read_8x_u8()?)) } } +impl TryFrom for TxidPrefix { + type Error = color_eyre::Report; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From<&TxidPrefix> for fjall::Slice { + fn from(value: &TxidPrefix) -> Self { + Self::new(value.unsafe_as_slice()) + } +} +impl From for fjall::Slice { + fn from(value: TxidPrefix) -> Self { + Self::from(&value) + } +} diff --git a/bindex/src/structs/txindex.rs b/bindex/src/structs/txindex.rs index 70a3735e3..4b2b781e6 100644 --- a/bindex/src/structs/txindex.rs +++ b/bindex/src/structs/txindex.rs @@ -2,6 +2,7 @@ use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; use snkrj::{direct_repr, Storable, UnsizedStorable}; +use storable_vec::UnsafeSizedSerDe; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txindex(u32); @@ -57,3 +58,15 @@ impl From for usize { value.0 as usize } } + +impl TryFrom for Txindex { + type Error = storable_vec::Error; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From for fjall::Slice { + fn from(value: Txindex) -> Self { + Self::new(value.unsafe_as_slice()) + } +} diff --git a/bindex/src/structs/version.rs b/bindex/src/structs/version.rs index e5c35b83a..4faa4ec0d 100644 --- a/bindex/src/structs/version.rs +++ b/bindex/src/structs/version.rs @@ -23,3 +23,15 @@ impl TryFrom<&Path> for Version { Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned()) } } + +impl TryFrom for Version { + type Error = color_eyre::Report; + fn try_from(value: fjall::Slice) -> Result { + Ok(*Self::unsafe_try_from_slice(&value)?) + } +} +impl From for fjall::Slice { + fn from(value: Version) -> Self { + Self::new(value.unsafe_as_slice()) + } +} diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index 69839dbd7..ad5b5de24 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -27,7 +27,7 @@ use memmap2::{Mmap, MmapOptions}; pub struct StorableVec { pathbuf: PathBuf, file: File, - cache: Vec>>, // Boxed to reduce the size of the lock (24 > 16) + cache: Vec>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16) disk_len: usize, pushed: Vec, // updated: BTreeMap, @@ -40,6 +40,7 @@ pub struct StorableVec { const MAX_PAGE_SIZE: usize = 4 * 4096; const ONE_MB: usize = 1000 * 1024; const MAX_CACHE_SIZE: usize = 100 * ONE_MB; +// const MAX_CACHE_SIZE: usize = 100 * ONE_MB; impl StorableVec where @@ -50,8 +51,7 @@ where pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; /// In bytes pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T; - pub const CACHE_LENGTH: usize = usize::MAX; - // pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; + pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; pub fn import(path: &Path) -> Result { let file = Self::open_file(path)?; @@ -78,13 +78,18 @@ where // self.cache.clear(); // self.cache.resize_with(len, Default::default); + // par_iter_mut ? self.cache.iter_mut().for_each(|lock| { lock.take(); }); + let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize; - self.cache - .resize_with(Self::CACHE_LENGTH.min(len), Default::default); - self.cache.shrink_to_fit(); + let len = Self::CACHE_LENGTH.min(len); + + if self.cache.len() != len { + self.cache.resize_with(len, Default::default); + self.cache.shrink_to_fit(); + } } fn open_file(path: &Path) -> Result { @@ -222,7 +227,7 @@ where let len = self.len(); match len.cmp(&index) { Ordering::Greater => { - // dbg!(len, index); + // dbg!(len, index, &self.pathbuf); // panic!(); Ok(()) } @@ -230,7 +235,10 @@ where self.push(value); Ok(()) } - Ordering::Less => Err(Error::IndexTooHigh), + Ordering::Less => { + dbg!(index, value); + Err(Error::IndexTooHigh) + } } }