diff --git a/Cargo.lock b/Cargo.lock index af827c497..544e428e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,8 +50,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f" dependencies = [ - "bitcoin-internals 0.3.0", - "bitcoin_hashes 0.14.0", + "bitcoin-internals", + "bitcoin_hashes", ] [[package]] @@ -71,7 +71,6 @@ name = "bindex" version = "0.1.0" dependencies = [ "bitcoin", - "bitcoin_hashes 0.16.0", "biter", "color-eyre", "derive_deref", @@ -92,11 +91,11 @@ checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026" dependencies = [ "base58ck", "bech32", - "bitcoin-internals 0.3.0", - "bitcoin-io 0.1.3", + "bitcoin-internals", + "bitcoin-io", "bitcoin-units", - "bitcoin_hashes 0.14.0", - "hex-conservative 0.2.1", + "bitcoin_hashes", + "hex-conservative", "hex_lit", "secp256k1", "serde", @@ -111,34 +110,19 @@ dependencies = [ "serde", ] -[[package]] -name = "bitcoin-internals" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b854212e29b96c8f0fe04cab11d57586c8f3257de0d146c76cb3b42b3eb9118" - [[package]] name = "bitcoin-io" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" -[[package]] -name = "bitcoin-io" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26792cd2bf245069a1c5acb06aa7ad7abe1de69b507c90b490bca81e0665d0ee" -dependencies = [ - "bitcoin-internals 0.4.0", -] - [[package]] name = "bitcoin-units" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" dependencies = [ - "bitcoin-internals 0.3.0", + "bitcoin-internals", "serde", ] @@ -148,21 +132,11 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" dependencies = [ - "bitcoin-io 0.1.3", - "hex-conservative 0.2.1", + "bitcoin-io", + "hex-conservative", "serde", ] -[[package]] -name = "bitcoin_hashes" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5d09f16329cd545d7e6008b2c6b2af3a90bc678cf41ac3d2f6755943301b16" -dependencies = [ - "bitcoin-io 0.2.0", - "hex-conservative 0.3.0", -] - [[package]] name = "bitcoincore-rpc" version = "0.19.0" @@ -217,7 +191,6 @@ dependencies = [ "exit", "fjall", "jiff", - "rayon", "storable_vec", "unsafe_slice_serde", ] @@ -494,15 +467,6 @@ dependencies = [ "arrayvec", ] -[[package]] -name = "hex-conservative" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afe881d0527571892c4034822e59bb10c6c991cce6abe8199b6f5cf10766f55" -dependencies = [ - "arrayvec", -] - [[package]] name = "hex_lit" version = "0.1.1" @@ -896,7 +860,7 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ - "bitcoin_hashes 0.14.0", + "bitcoin_hashes", "rand", "secp256k1-sys", "serde", @@ -996,9 +960,7 @@ dependencies = [ name = "struct_iterable_derive" version = "0.1.0" dependencies = [ - "proc-macro2", "quote", - "struct_iterable_internal", "syn 2.0.96", ] diff --git a/computer/Cargo.toml b/computer/Cargo.toml index 4ea89fad8..b1ac43a13 100644 --- a/computer/Cargo.toml +++ b/computer/Cargo.toml @@ -11,6 +11,5 @@ derive_deref = { workspace = true } exit = { workspace = true } fjall = { workspace = true } jiff = { workspace = true } -rayon = { workspace = true } storable_vec = { workspace = true } unsafe_slice_serde = { workspace = true } diff --git a/computer/src/lib.rs b/computer/src/lib.rs index c59dbed44..124ed8369 100644 --- a/computer/src/lib.rs +++ b/computer/src/lib.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use bindex::{Height, Indexer}; +use bindex::Indexer; use biter::rpc; use exit::Exit; @@ -36,34 +36,63 @@ impl Computer { indexer.index(bitcoin_dir, rpc, exit)?; } + let height_count = indexer.vecs().height_to_size.len(); + let txindexes_count = indexer.vecs().txindex_to_txid.len(); + let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len(); + let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len(); + // TODO: Remove all outdated - // Compute txindex to X + self.vecs + .txindex_to_last_txinindex + .compute_last_index_from_first(&indexer.vecs().txindex_to_first_txinindex, txinindexes_count)?; + + self.vecs.txindex_to_inputcount.compute_count_from_indexes( + &indexer.vecs().txindex_to_first_txinindex, + &self.vecs.txindex_to_last_txinindex, + )?; + + self.vecs + .txindex_to_last_txoutindex + .compute_last_index_from_first(&indexer.vecs().txindex_to_first_txoutindex, txoutindexes_count)?; + + self.vecs.txindex_to_outputcount.compute_count_from_indexes( + &indexer.vecs().txindex_to_first_txoutindex, + &self.vecs.txindex_to_last_txoutindex, + )?; - // Compute height to X - indexer - .vecs() - .height_to_timestamp - .read_from_(self.vecs.height_to_date.len(), |(_height, timestamp)| { - self.vecs - .height_to_date - .push_if_needed(Height::from(_height), Date::from(timestamp)) - })?; self.vecs .height_to_date - .read_from_(self.vecs.date_to_first_height.len(), |(_height, date)| { - self.vecs - .date_to_first_height - .push_if_needed(*date, Height::from(_height)) - })?; + .compute_transform(&indexer.vecs().height_to_timestamp, |timestamp| Date::from(timestamp))?; - // Compute date to X + self.vecs + .height_to_last_txindex + .compute_last_index_from_first(&indexer.vecs().height_to_first_txindex, height_count)?; + + self.vecs.txindex_to_height.compute_inverse_less_to_more( + &indexer.vecs().height_to_first_txindex, + &self.vecs.height_to_last_txindex, + )?; + + let date_count = self.vecs.height_to_date.len(); + + self.vecs + .date_to_first_height + .compute_inverse_more_to_less(&self.vecs.height_to_date)?; + + // --- + // Date to X + // --- // ... - // Compute month to X + // --- + // Month to X + // --- // ... - // Compute year to X + // --- + // Year to X + // --- // ... Ok(()) diff --git a/computer/src/storage/fjalls.rs b/computer/src/storage/fjalls.rs index d4a5c6d34..aba352bc6 100644 --- a/computer/src/storage/fjalls.rs +++ b/computer/src/storage/fjalls.rs @@ -5,18 +5,18 @@ use bindex::{Store, Version}; use crate::structs::{AddressindexTxoutindex, Unit}; pub struct Fjalls { - pub address_txoutindex_in: Store, - pub address_txoutindex_out: Store, + pub address_to_utxos_received: Store, + pub address_to_utxos_spent: Store, } impl Fjalls { pub fn import(path: &Path) -> color_eyre::Result { - let address_txoutindex_in = Store::import(&path.join("address_txoutindex_in"), Version::from(1))?; - let address_txoutindex_out = Store::import(&path.join("address_txoutindex_out"), Version::from(1))?; + let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?; + let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?; Ok(Self { - address_txoutindex_in, - address_txoutindex_out, + address_to_utxos_received, + address_to_utxos_spent, }) } } diff --git a/computer/src/storage/storable_vecs/base.rs b/computer/src/storage/storable_vecs/base.rs index 04e86f108..b19e8196a 100644 --- a/computer/src/storage/storable_vecs/base.rs +++ b/computer/src/storage/storable_vecs/base.rs @@ -1,74 +1,144 @@ use std::{ + error, fmt::Debug, io, - ops::{Deref, DerefMut}, + ops::{Add, Sub}, path::Path, }; -use bindex::{Indexer, Version}; +use derive_deref::{Deref, DerefMut}; +use storable_vec::{StorableVecIndex, StorableVecType, Version}; -use crate::Computer; +#[derive(Debug, Deref, DerefMut)] +pub struct StorableVec(storable_vec::StorableVec); -pub struct StorableVec { - vec: bindex::StorableVec, - f: Box storable_vec::Result>>, -} +const FLUSH_EVERY: usize = 10_000; impl StorableVec where - I: TryInto, - T: Sized + Debug + Clone, + I: StorableVecIndex, + T: StorableVecType, { - pub fn import(path: &Path, version: Version, f: F) -> io::Result + pub fn import(path: &Path, version: Version) -> io::Result { + Ok(Self(storable_vec::StorableVec::import(path, version)?)) + } + + fn flush_vec_if_needed(&mut self) -> io::Result<()> { + if self.pushed_len() == FLUSH_EVERY { + self.flush() + } else { + Ok(()) + } + } + + pub fn compute_inverse_more_to_less(&mut self, other: &storable_vec::StorableVec) -> storable_vec::Result<()> where - F: Fn(&Indexer, &Computer) -> storable_vec::Result> + 'static, + I: StorableVecType, + T: StorableVecIndex, { - let vec = bindex::StorableVec::import(path, version)?; - - Ok(Self { vec, f: Box::new(f) }) + other.iter_from(self.last()?.map(|v| *v).unwrap_or_default(), |(v, i)| { + self.push_if_needed(*i, v) + }) } - pub fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> { - (self.f)(indexer, computer)? - .into_iter() - .try_for_each(|(i, v)| self.push_if_needed(i, v)) + pub fn compute_inverse_less_to_more( + &mut self, + first_indexes: &storable_vec::StorableVec, + last_indexes: &storable_vec::StorableVec, + ) -> color_eyre::Result<()> + where + I: StorableVecType, + T: StorableVecIndex, + { + let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?; + first_indexes.iter_from(T::from(self.len()), |(value, first_index)| { + let first_index: usize = (*first_index) + .try_into() + .map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?; + let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?; + let last_index: usize = (*last_index) + .try_into() + .map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?; + (first_index..last_index).try_for_each(|index| self.push_if_needed(I::from(index), value))?; + Ok(()) + })?; + self.flush()?; + Ok(()) } - // pub fn fill(&mut self) { - // self - // .vecs() - // .height_to_timestamp - // .read_iter(move |(_height, timestamp)| { - // let height = Height::from(_height); - // let date = Date::from(timestamp); - // self.vecs.date_to_first_height.push_if_needed(date, height)?; - // Ok(()) - // })?; - // } -} + pub fn compute_transform(&mut self, other: &storable_vec::StorableVec, t: F) -> storable_vec::Result<()> + where + A: StorableVecType, + F: Fn(&A) -> T, + { + other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a))) + } -impl Deref for StorableVec { - type Target = bindex::StorableVec; - fn deref(&self) -> &Self::Target { - &self.vec - } -} -impl DerefMut for StorableVec { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.vec - } -} - -pub trait AnyComputedStorableVec { - fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()>; -} - -impl AnyComputedStorableVec for StorableVec -where - I: TryInto, - T: Sized + Debug + Clone, -{ - fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> { - self.compute(indexer, computer) + pub fn compute_is_first_ordered( + &mut self, + self_to_other: &storable_vec::StorableVec, + other_to_self: &storable_vec::StorableVec, + ) -> storable_vec::Result<()> + where + A: StorableVecIndex + StorableVecType, + { + // let mut prev_a_opt = None; + // self_to_other.iter_from(I::from(self.len()), |(i, a)| { + // if prev_a_opt.is_none() { + // prev_a_opt.replace(a); + // self.push_if_needed(i, other_to_self.read_at(a) == i); + // } else { + // let prev_a = prev_a_opt.unwrap(); + // if a != prev_a + // } + // other_to_self.seek_read(a); + // self.push_if_needed(i, t(a)); + // Ok(()) + // }) + Ok(()) + } + + pub fn compute_last_index_from_first( + &mut self, + first_index_vec: &storable_vec::StorableVec, + final_len: usize, + ) -> color_eyre::Result<()> + where + T: Copy + From + Sub + StorableVecIndex, + { + let mut prev_index: Option = None; + first_index_vec.iter_from(I::from(self.len()), |(i, v)| { + if let Some(prev_index) = prev_index { + self.push_if_needed(prev_index, *v - T::from(1))?; + } + prev_index.replace(i); + self.flush_vec_if_needed().map_err(storable_vec::Error::IO) + })?; + if let Some(prev_index) = prev_index { + self.push_if_needed(prev_index, T::from(final_len) - T::from(1))?; + } + self.flush()?; + Ok(()) + } + + pub fn compute_count_from_indexes( + &mut self, + first_indexes: &storable_vec::StorableVec, + last_indexes: &storable_vec::StorableVec, + ) -> color_eyre::Result<()> + where + T: From, + T2: StorableVecType + Copy + Add + Sub + TryInto, + >::Error: error::Error + Send + Sync + 'static, + { + let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?; + first_indexes.iter_from(I::from(self.len()), |(i, first_index)| { + let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?; + let count = *last_index + 1_usize - *first_index; + self.push_if_needed(i, count.into())?; + self.flush_vec_if_needed().map_err(storable_vec::Error::IO) + })?; + self.flush()?; + Ok(()) } } diff --git a/computer/src/storage/storable_vecs/mod.rs b/computer/src/storage/storable_vecs/mod.rs index 27cf5a856..c2becc932 100644 --- a/computer/src/storage/storable_vecs/mod.rs +++ b/computer/src/storage/storable_vecs/mod.rs @@ -1,12 +1,13 @@ use std::{fs, path::Path}; -use bindex::{Addressindex, Amount, Height, StorableVec, Timestamp, Txindex, Txinindex, Txoutindex, Version}; +use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex}; +use storable_vec::Version; use crate::structs::{Date, Feerate}; -// mod base; +mod base; -// use base::*; +use base::*; pub struct StorableVecs { pub date_to_first_height: StorableVec, @@ -15,7 +16,7 @@ pub struct StorableVecs { // pub height_to_fee: StorableVec, // pub height_to_inputcount: StorableVec, // pub height_to_last_addressindex: StorableVec, - // pub height_to_last_txindex: StorableVec, + pub height_to_last_txindex: StorableVec, // pub height_to_last_txoutindex: StorableVec, // pub height_to_maxfeerate: StorableVec, // pub height_to_medianfeerate: StorableVec, @@ -25,6 +26,8 @@ pub struct StorableVecs { // pub height_to_totalfees: StorableVec, // pub height_to_txcount: StorableVec, pub txindex_to_fee: StorableVec, + pub txindex_to_height: StorableVec, + pub txindex_to_is_coinbase: StorableVec, // pub txindex_to_feerate: StorableVec, pub txindex_to_inputcount: StorableVec, pub txindex_to_last_txinindex: StorableVec, @@ -46,7 +49,7 @@ impl StorableVecs { // &path.join("height_to_last_addressindex"), // Version::from(1), // )?, - // height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?, + height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?, // height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?, // height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?, // height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?, @@ -56,6 +59,8 @@ impl StorableVecs { // height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?, // height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?, txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?, + txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?, + txindex_to_is_coinbase: StorableVec::import(&path.join("txindex_to_is_coinbase"), Version::from(1))?, // txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?, txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?, txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?, diff --git a/computer/src/structs/date.rs b/computer/src/structs/date.rs index dda76cdea..fd6937b0f 100644 --- a/computer/src/structs/date.rs +++ b/computer/src/structs/date.rs @@ -1,9 +1,11 @@ +use std::ops::Add; + use bindex::Timestamp; use color_eyre::eyre::eyre; use derive_deref::Deref; -use jiff::{civil::Date as _Date, tz::TimeZone}; +use jiff::{civil::Date as _Date, tz::TimeZone, Span}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)] pub struct Date(_Date); impl Date { @@ -39,3 +41,16 @@ impl TryFrom for usize { } } } + +impl From for Date { + fn from(value: usize) -> Self { + Self(Self::INDEX_ZERO.checked_add(Span::new().days(value as i64)).unwrap()) + } +} + +impl Add for Date { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0.checked_add(Span::new().days(rhs as i64)).unwrap()) + } +} diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 4a07f4015..2ef5fe3a3 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" [dependencies] bitcoin = { workspace = true } -bitcoin_hashes = "0.16.0" biter = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 6ff495281..2256ebf1b 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -104,7 +104,7 @@ impl Indexer { height = Height::from(_height); - if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? { + if let Some(saved_blockhash) = vecs.height_to_blockhash.cached_get(height)? { if &blockhash != saved_blockhash.as_ref() { todo!("Rollback not implemented"); // trees.rollback_from(&mut rtx, height, &exit)?; @@ -260,7 +260,7 @@ impl Indexer { let txoutindex = *vecs .txindex_to_first_txoutindex - .get(prev_txindex)? + .cached_get(prev_txindex)? .context("Expect txoutindex to not be none") .inspect_err(|_| { dbg!(outpoint.txid, prev_txindex, vout); @@ -336,12 +336,12 @@ impl Indexer { let prev_addresstype = *vecs .addressindex_to_addresstype - .get(addressindex)? + .cached_get(addressindex)? .context("Expect to have address type")?; let addresstypeindex = *vecs .addressindex_to_addresstypeindex - .get(addressindex)? + .cached_get(addressindex)? .context("Expect to have address type index")?; // Good first time // Wrong after rerun @@ -605,7 +605,7 @@ impl Indexer { // Ok if `get` is not par as should happen only twice let prev_txid = vecs .txindex_to_txid - .get(prev_txindex)? + .cached_get(prev_txindex)? .context("To have txid for txindex") .inspect_err(|_| { dbg!(txindex, txid, len); @@ -630,7 +630,7 @@ impl Indexer { if !is_dup { let prev_height = - vecs.txindex_to_height.get(prev_txindex)?.expect("To have height"); + vecs.txindex_to_height.cached_get(prev_txindex)?.expect("To have height"); dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex); return Err(eyre!("Expect none")); } diff --git a/indexer/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs index 41707602c..8d1030ab7 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/indexer/src/storage/storable_vecs/base.rs @@ -1,79 +1,44 @@ use std::{ fmt::Debug, - fs, io, + io, ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; -use super::{Height, Version}; +use storable_vec::{StorableVecIndex, StorableVecType, Version}; + +use super::Height; #[derive(Debug)] pub struct StorableVec { height: Option, - pathbuf: PathBuf, - version: Version, vec: storable_vec::StorableVec, } impl StorableVec where - I: TryInto, - T: Sized + Debug + Clone, + I: StorableVecIndex, + T: StorableVecType, { pub fn import(path: &Path, version: Version) -> io::Result { - fs::create_dir_all(path)?; - - let pathbuf = path.to_owned(); - let path_vec = Self::path_vec_(path); - let path_version = Self::path_version_(path); - - let is_same_version = - Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version); - if !is_same_version { - let _ = fs::remove_file(&path_vec); - let _ = fs::remove_file(&path_version); - let _ = fs::remove_file(Self::path_height_(path)); - } - - let this = Self { + Ok(Self { height: Height::try_from(Self::path_height_(path).as_path()).ok(), - pathbuf, - version, - vec: storable_vec::StorableVec::import(&path_vec)?, - }; - - this.version.write(&this.path_version())?; - - Ok(this) + vec: storable_vec::StorableVec::import(path, version)?, + }) } pub fn flush(&mut self, height: Height) -> io::Result<()> { if self.needs(height) { height.write(&self.path_height())?; } - self.vec.flush() } - // fn path_vec(&self) -> PathBuf { - // Self::_path_vec(&self.path) - // } - fn path_vec_(path: &Path) -> PathBuf { - path.join("vec") - } - - fn path_version(&self) -> PathBuf { - Self::path_version_(&self.pathbuf) - } - fn path_version_(path: &Path) -> PathBuf { - path.join("version") - } - pub fn height(&self) -> color_eyre::Result { Height::try_from(self.path_height().as_path()) } fn path_height(&self) -> PathBuf { - Self::path_height_(&self.pathbuf) + Self::path_height_(self.vec.path()) } fn path_height_(path: &Path) -> PathBuf { path.join("height") @@ -103,17 +68,22 @@ impl DerefMut for StorableVec { pub trait AnyStorableVec { fn height(&self) -> color_eyre::Result; fn flush(&mut self, height: Height) -> io::Result<()>; + fn reset_cache(&mut self); } impl AnyStorableVec for StorableVec where - I: Into, - T: Sized + Debug + Clone, + I: StorableVecIndex, + T: StorableVecType, { fn height(&self) -> color_eyre::Result { self.height() } + fn reset_cache(&mut self) { + self.vec.reset_cache() + } + fn flush(&mut self, height: Height) -> io::Result<()> { self.flush(height) } diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index 7d251df42..15aa0c749 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -3,11 +3,12 @@ use std::{fs, io, path::Path}; use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight}; use exit::Exit; use rayon::prelude::*; +use storable_vec::Version; use crate::structs::{ Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex, - Txinindex, Txoutindex, Version, + Txinindex, Txoutindex, }; mod base; @@ -203,37 +204,37 @@ impl StorableVecs { Ok(match addresstype { Addresstype::P2PK65 => self .p2pk65index_to_p2pk65addressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2PK33 => self .p2pk33index_to_p2pk33addressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2PKH => self .p2pkhindex_to_p2pkhaddressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2SH => self .p2shindex_to_p2shaddressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2WPKH => self .p2wpkhindex_to_p2wpkhaddressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2WSH => self .p2wshindex_to_p2wshaddressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), Addresstype::P2TR => self .p2trindex_to_p2traddressbytes - .get(addresstypeindex)? + .cached_get(addresstypeindex)? // .map(|v| Addressbytes::from(v.clone())), .map(|v| Addressbytes::from(v.into_inner())), _ => unreachable!(), @@ -354,6 +355,10 @@ impl StorableVecs { // Ok(()) } + pub fn reset_cache(&mut self) { + self.as_mut_slice().into_par_iter().for_each(|vec| vec.reset_cache()) + } + pub fn flush(&mut self, height: Height) -> io::Result<()> { self.as_mut_slice() .into_par_iter() diff --git a/indexer/src/structs/addressindex.rs b/indexer/src/structs/addressindex.rs index 02578722b..0a5ea91c9 100644 --- a/indexer/src/structs/addressindex.rs +++ b/indexer/src/structs/addressindex.rs @@ -1,3 +1,5 @@ +use std::ops::Add; + use derive_deref::{Deref, DerefMut}; use fjall::Slice; use unsafe_slice_serde::UnsafeSliceSerde; @@ -60,3 +62,17 @@ impl From for Slice { Self::new(value.unsafe_as_slice()) } } + +impl Add for Addressindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u32) + } +} + +impl Add for Addressindex { + type Output = Self; + fn add(self, rhs: Addressindex) -> Self::Output { + Self(self.0 + rhs.0) + } +} diff --git a/indexer/src/structs/addresstypeindex.rs b/indexer/src/structs/addresstypeindex.rs index 971be4455..d94938e97 100644 --- a/indexer/src/structs/addresstypeindex.rs +++ b/indexer/src/structs/addresstypeindex.rs @@ -1,3 +1,5 @@ +use std::ops::Add; + use derive_deref::{Deref, DerefMut}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] @@ -50,3 +52,17 @@ impl From for usize { value.0 as usize } } + +impl Add for Addresstypeindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u32) + } +} + +impl Add for Addresstypeindex { + type Output = Self; + fn add(self, rhs: Addresstypeindex) -> Self::Output { + Self(self.0 + rhs.0) + } +} diff --git a/indexer/src/structs/height.rs b/indexer/src/structs/height.rs index ee829a992..b9ff4b92f 100644 --- a/indexer/src/structs/height.rs +++ b/indexer/src/structs/height.rs @@ -24,11 +24,19 @@ impl PartialEq for Height { } } +impl Add for Height { + type Output = Height; + + fn add(self, rhs: Height) -> Self::Output { + Self::from(self.0 + rhs.0) + } +} + impl Add for Height { type Output = Height; fn add(self, rhs: u32) -> Self::Output { - Self::from(*self + rhs) + Self::from(self.0 + rhs) } } @@ -42,7 +50,6 @@ impl Add for Height { impl Sub for Height { type Output = Height; - fn sub(self, rhs: Height) -> Self::Output { Self::from(*self - *rhs) } diff --git a/indexer/src/structs/txindex.rs b/indexer/src/structs/txindex.rs index 2ec5b4ac6..906e605d8 100644 --- a/indexer/src/structs/txindex.rs +++ b/indexer/src/structs/txindex.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, AddAssign}; +use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; use fjall::Slice; @@ -24,12 +24,26 @@ impl Add for Txindex { } } +impl Add for Txindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u32) + } +} + impl AddAssign for Txindex { fn add_assign(&mut self, rhs: Txindex) { self.0 += rhs.0 } } +impl Sub for Txindex { + type Output = Txindex; + fn sub(self, rhs: Txindex) -> Self::Output { + Self::from(*self - *rhs) + } +} + impl From for Txindex { fn from(value: u32) -> Self { Self(value) diff --git a/indexer/src/structs/txinindex.rs b/indexer/src/structs/txinindex.rs index 2c8fcfb07..9c217961c 100644 --- a/indexer/src/structs/txinindex.rs +++ b/indexer/src/structs/txinindex.rs @@ -1,8 +1,8 @@ -use std::ops::{Add, AddAssign}; +use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; -use super::Vout; +use super::Vin; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] pub struct Txinindex(u64); @@ -24,19 +24,42 @@ impl Add for Txinindex { } } -impl Add for Txinindex { +impl Add for Txinindex { type Output = Self; - fn add(self, rhs: Vout) -> Self::Output { + fn add(self, rhs: Vin) -> Self::Output { Self(self.0 + u64::from(rhs)) } } +impl Add for Txinindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u64) + } +} + impl AddAssign for Txinindex { fn add_assign(&mut self, rhs: Txinindex) { self.0 += rhs.0 } } +impl Sub for Txinindex { + type Output = Self; + fn sub(self, rhs: Txinindex) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl From for u32 { + fn from(value: Txinindex) -> Self { + if value.0 > u32::MAX as u64 { + panic!() + } + value.0 as u32 + } +} + impl From for Txinindex { fn from(value: u64) -> Self { Self(value) diff --git a/indexer/src/structs/txoutindex.rs b/indexer/src/structs/txoutindex.rs index bf7517f35..541ae40ab 100644 --- a/indexer/src/structs/txoutindex.rs +++ b/indexer/src/structs/txoutindex.rs @@ -1,4 +1,4 @@ -use std::ops::{Add, AddAssign}; +use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; @@ -37,12 +37,35 @@ impl Add for Txoutindex { } } +impl Add for Txoutindex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs as u64) + } +} + impl AddAssign for Txoutindex { fn add_assign(&mut self, rhs: Txoutindex) { self.0 += rhs.0 } } +impl Sub for Txoutindex { + type Output = Self; + fn sub(self, rhs: Txoutindex) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl From for u32 { + fn from(value: Txoutindex) -> Self { + if value.0 > u32::MAX as u64 { + panic!() + } + value.0 as u32 + } +} + impl From for Txoutindex { fn from(value: u64) -> Self { Self(value) diff --git a/indexer/src/structs/vin.rs b/indexer/src/structs/vin.rs index 65524d489..a26e0cfa1 100644 --- a/indexer/src/structs/vin.rs +++ b/indexer/src/structs/vin.rs @@ -4,7 +4,8 @@ use derive_deref::Deref; pub struct Vin(u32); impl Vin { - const ZERO: Self = Vin(0_u32); + pub const ZERO: Self = Vin(0); + pub const ONE: Self = Vin(1); pub fn is_zero(&self) -> bool { *self == Self::ZERO diff --git a/storable_vec/src/any.rs b/storable_vec/src/any.rs new file mode 100644 index 000000000..ec13a397a --- /dev/null +++ b/storable_vec/src/any.rs @@ -0,0 +1,27 @@ +use std::io; + +use crate::{StorableVec, StorableVecIndex, StorableVecType}; + +pub trait AnyStorableVec { + fn len(&self) -> usize; + fn is_empty(&self) -> bool; + fn unsafe_flush(&mut self) -> io::Result<()>; +} + +impl AnyStorableVec for StorableVec +where + I: StorableVecIndex, + T: StorableVecType, +{ + fn len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn unsafe_flush(&mut self) -> io::Result<()> { + self.flush() + } +} diff --git a/storable_vec/src/bytes.rs b/storable_vec/src/bytes.rs new file mode 100644 index 000000000..569d9dd32 --- /dev/null +++ b/storable_vec/src/bytes.rs @@ -0,0 +1,11 @@ +use std::sync::Arc; + +use crate::Result; + +pub trait Bytes: Sized { + const LEN: usize = size_of::(); + fn to_bytes(&self) -> Arc<[u8]>; + fn try_from_bytes(bytes: &[u8]) -> Result; +} + +pub trait UnsafeBytes {} diff --git a/storable_vec/src/error.rs b/storable_vec/src/error.rs new file mode 100644 index 000000000..63f503b52 --- /dev/null +++ b/storable_vec/src/error.rs @@ -0,0 +1,34 @@ +use std::{ + fmt::{self, Debug}, + io, +}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum Error { + MmapsVecIsTooSmall, + IO(io::Error), + UnsafeSliceSerde(unsafe_slice_serde::Error), + IndexTooHigh, + ExpectFileToHaveIndex, + ExpectVecToHaveIndex, + FailedKeyTryIntoUsize, +} + +impl fmt::Display for Error { + // This trait requires `fmt` with this exact signature. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), + Error::IO(error) => Debug::fmt(&error, f), + Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), + Error::IndexTooHigh => write!(f, "Index too high"), + Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), + Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), + Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"), + } + } +} + +impl std::error::Error for Error {} diff --git a/storable_vec/src/index.rs b/storable_vec/src/index.rs new file mode 100644 index 000000000..fff90ada1 --- /dev/null +++ b/storable_vec/src/index.rs @@ -0,0 +1,11 @@ +use std::{fmt::Debug, ops::Add}; + +pub trait StorableVecIndex +where + Self: Debug + Default + Copy + Clone + TryInto + From + Add, +{ +} +impl StorableVecIndex for I where + I: Debug + Default + Copy + Clone + TryInto + From + Add +{ +} diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index df2297f25..905c2d952 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -1,21 +1,34 @@ use std::{ cmp::Ordering, - fmt::{self, Debug}, - fs::{File, OpenOptions}, + fmt::Debug, + fs::{self, File, OpenOptions}, io::{self, Read, Seek, SeekFrom, Write}, marker::PhantomData, mem, - ops::{Deref, Range}, + ops::Range, path::{Path, PathBuf}, - sync::{ - // atomic::{AtomicUsize, Ordering as AtomicOrdering}, - OnceLock, - }, + sync::OnceLock, }; use memmap2::{Mmap, MmapOptions}; use unsafe_slice_serde::UnsafeSliceSerde; +mod any; +// mod bytes; +mod error; +mod index; +mod type_; +mod value; +mod version; + +pub use any::*; +// pub use bytes::*; +pub use error::*; +pub use index::*; +pub use type_::*; +pub use value::*; +pub use version::*; + /// /// A very small, fast, efficient and simple storable Vec /// @@ -30,7 +43,7 @@ use unsafe_slice_serde::UnsafeSliceSerde; #[derive(Debug)] pub struct StorableVec { pathbuf: PathBuf, - file: File, + unsafe_file: File, cache: Vec>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16) disk_len: usize, pushed: Vec, @@ -50,8 +63,8 @@ const MAX_CACHE_SIZE: usize = 100 * ONE_MB; impl StorableVec where - I: TryInto, - T: Sized + Debug + Clone, + I: StorableVecIndex, + T: StorableVecType, { pub const SIZE_OF_T: usize = size_of::(); pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; @@ -59,13 +72,23 @@ where pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T; pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE; - pub fn import(path: &Path) -> Result { - let file = Self::open_file_(path)?; + pub fn import(path: &Path, version: Version) -> Result { + fs::create_dir_all(path)?; + + let path_version = Self::path_version_(path); + let is_same_version = + Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version); + if !is_same_version { + fs::remove_dir_all(path)?; + } + version.write(&path_version)?; + + let unsafe_file = Self::open_file_(&Self::path_vec_(path))?; let mut this = Self { pathbuf: path.to_owned(), - disk_len: Self::byte_index_to_index(Self::file_len(&file)?), - file, + disk_len: Self::disk_len(&unsafe_file)?, + unsafe_file, cache: vec![], pushed: vec![], // updated: BTreeMap::new(), @@ -76,20 +99,17 @@ where // opened_mmaps: AtomicUsize::new(0), }; + // TODO: Only if write mode this.reset_cache(); Ok(this) } - fn file_len(file: &File) -> io::Result { - Ok(file.metadata()?.len() as usize) + pub fn disk_len(file: &File) -> io::Result { + Ok(Self::byte_index_to_index(file.metadata()?.len() as usize)) } - fn reset_cache(&mut self) { - // let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize; - // self.cache.clear(); - // self.cache.resize_with(len, Default::default); - + pub fn reset_cache(&mut self) { // par_iter_mut ? self.cache.iter_mut().for_each(|lock| { lock.take(); @@ -105,7 +125,7 @@ where } fn open_file(&self) -> Result { - Self::open_file_(&self.pathbuf).map_err(Error::IO) + Self::open_file_(&self.path_vec()).map_err(Error::IO) } fn open_file_(path: &Path) -> Result { OpenOptions::new() @@ -146,10 +166,10 @@ where } #[inline] - pub fn get(&self, index: I) -> Result>> { - self.get_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?) + pub fn cached_get(&self, index: I) -> Result>> { + self.cached_get_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?) } - pub fn get_(&self, index: usize) -> Result>> { + fn cached_get_(&self, index: usize) -> Result>> { match self.index_to_pushed_index(index) { Ok(index) => { if let Some(index) = index { @@ -166,7 +186,6 @@ where // } // } - let byte_index = Self::index_to_byte_index(index); let page_index = index / Self::PER_PAGE; let last_index = self.disk_len - 1; let max_page_index = last_index / Self::PER_PAGE; @@ -188,7 +207,7 @@ where MmapOptions::new() .len(Self::PAGE_SIZE) .offset((page_index * Self::PAGE_SIZE) as u64) - .map(&self.file) + .map(&self.unsafe_file) .unwrap() }) }); @@ -201,49 +220,78 @@ where T::unsafe_try_from_slice(slice).map_err(Error::UnsafeSliceSerde)?, ))) } else { - let mut file = self.open_file()?; - let mut buf = vec![0; Self::SIZE_OF_T]; - - file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?; - file.read_exact(&mut buf).map_err(Error::IO)?; - let value = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; - + let (mut file, mut buf) = self.prepare_to_read()?; + Self::seek_(&mut file, index)?; + let value = self.read_exact(&mut file, &mut buf)?; Ok(Some(Value::Owned(value.to_owned()))) } } + pub fn get_or_default(&self, index: I) -> Result where T: Default + Clone, { - Ok(self.get(index)?.map(|v| (*v).clone()).unwrap_or(Default::default())) + Ok(self + .cached_get(index)? + .map(|v| (*v).clone()) + .unwrap_or(Default::default())) } - pub fn read_iter(&self, f: F) -> Result<()> - where - F: FnMut((usize, &T)) -> Result<()>, - { - self.read_from_(0, f) + pub fn seek(file: &mut File, index: I) -> Result<()> { + Self::seek_(file, index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?) } - pub fn read_from_(&self, index: usize, mut f: F) -> Result<()> - where - F: FnMut((usize, &T)) -> Result<()>, - { - let mut file = self.open_file()?; - let mut buf = vec![0; Self::SIZE_OF_T]; - + pub fn seek_(file: &mut File, index: usize) -> Result<()> { let byte_index = Self::index_to_byte_index(index); - file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?; + Ok(()) + } - let mut i = 0; + pub fn iter(&self, f: F) -> Result<()> + where + F: FnMut((I, &T)) -> Result<()>, + { + self.iter_from(I::from(0_usize), f) + } - loop { - if file.read_exact(&mut buf).map_err(Error::IO).is_err() { - break; - } - let v = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; - f((i, v))?; + pub fn prepare_to_read(&self) -> Result<(File, Vec)> { + let file = self.open_file()?; + let buf = vec![0; Self::SIZE_OF_T]; + Ok((file, buf)) + } + + pub fn prepare_to_read_at(&self, index: I) -> Result<(File, Vec)> { + self.prepare_to_read_at_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?) + } + pub fn prepare_to_read_at_(&self, index: usize) -> Result<(File, Vec)> { + let (mut file, buf) = self.prepare_to_read()?; + Self::seek_(&mut file, index)?; + Ok((file, buf)) + } + + pub fn read_exact<'a>(&self, file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> { + file.read_exact(buf).map_err(Error::IO)?; + let v = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?; + Ok(v) + } + + pub fn iter_from(&self, index: I, mut f: F) -> Result<()> + where + F: FnMut((I, &T)) -> Result<()>, + { + let (mut file, mut buf) = self.prepare_to_read()?; + let disk_len = Self::disk_len(&file).map_err(Error::IO)?; + Self::seek(&mut file, index)?; + + let mut i: usize = index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?; + while i < disk_len { + let v = self.read_exact(&mut file, &mut buf)?; + f((I::from(i), v))?; + i += 1; + } + i = 0; + while i < self.pushed_len() { + f((I::from(i + disk_len), self.pushed.get(i).as_ref().unwrap()))?; i += 1; } @@ -252,7 +300,7 @@ where #[allow(unused)] pub fn first(&self) -> Result>> { - self.get_(0) + self.cached_get_(0) } #[allow(unused)] @@ -261,7 +309,7 @@ where if len == 0 { return Ok(None); } - self.get_(len - 1) + self.cached_get_(len - 1) } pub fn push(&mut self, value: T) { @@ -271,7 +319,7 @@ where pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { self.push_if_needed_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?, value) } - pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> { + fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> { let len = self.len(); match len.cmp(&index) { Ordering::Greater => { @@ -331,7 +379,11 @@ where // } pub fn len(&self) -> usize { - self.disk_len + self.pushed.len() + self.disk_len + self.pushed_len() + } + + pub fn pushed_len(&self) -> usize { + self.pushed.len() } pub fn is_empty(&self) -> bool { @@ -341,18 +393,30 @@ where pub fn has(&self, index: I) -> Result { Ok(self.has_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)) } - pub fn has_(&self, index: usize) -> bool { + fn has_(&self, index: usize) -> bool { index < self.len() } pub fn hasnt(&self, index: I) -> Result { Ok(self.hasnt_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)) } - pub fn hasnt_(&self, index: usize) -> bool { + fn hasnt_(&self, index: usize) -> bool { !self.has_(index) } + // pub fn flush(&mut self) -> io::Result<()> + // where + // T: Bytes, + // { + // self.flush_(|bytes, v| bytes.extend_from_slice(&v.to_bytes())) + // } pub fn flush(&mut self) -> io::Result<()> { + // self.flush_(|bytes, v| bytes.extend_from_slice(v.unsafe_as_slice())) + // } + // fn flush_(&mut self, mut extend: F) -> io::Result<()> + // where + // F: FnMut(&mut Vec, T), + // { self.reset_cache(); if self.pushed.is_empty() { @@ -366,99 +430,25 @@ where mem::take(&mut self.pushed) .into_iter() .for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice())); + // .for_each(|v| extend(&mut bytes, v)); - self.file.write_all(&bytes)?; + self.unsafe_file.write_all(&bytes)?; Ok(()) } -} -pub trait AnyStorableVec { - fn len(&self) -> usize; - fn is_empty(&self) -> bool; - fn flush(&mut self) -> io::Result<()>; -} - -impl AnyStorableVec for StorableVec -where - I: Into, - T: Sized + Debug + Clone, -{ - fn len(&self) -> usize { - self.len() + pub fn path(&self) -> &Path { + &self.pathbuf } - fn is_empty(&self) -> bool { - self.is_empty() + fn path_vec(&self) -> PathBuf { + Self::path_vec_(&self.pathbuf) + } + fn path_vec_(path: &Path) -> PathBuf { + path.join("vec") } - fn flush(&mut self) -> io::Result<()> { - self.flush() + fn path_version_(path: &Path) -> PathBuf { + path.join("version") } } - -#[derive(Debug, Clone)] -pub enum Value<'a, T> { - Ref(&'a T), - Owned(T), -} - -impl Value<'_, T> -where - T: Sized + Debug + Clone, -{ - pub fn into_inner(self) -> T { - match self { - Self::Ref(t) => t.to_owned(), - Self::Owned(t) => t, - } - } -} -impl Deref for Value<'_, T> { - type Target = T; - fn deref(&self) -> &Self::Target { - match self { - Self::Ref(t) => t, - Self::Owned(t) => t, - } - } -} -impl AsRef for Value<'_, T> -where - T: Sized + Debug + Clone, -{ - fn as_ref(&self) -> &T { - match self { - Self::Ref(t) => t, - Self::Owned(t) => t, - } - } -} - -pub type Result = std::result::Result; - -#[derive(Debug)] -pub enum Error { - MmapsVecIsTooSmall, - IO(io::Error), - UnsafeSliceSerde(unsafe_slice_serde::Error), - IndexTooHigh, - ExpectFileToHaveIndex, - ExpectVecToHaveIndex, - FailedKeyTryIntoUsize, -} -impl fmt::Display for Error { - // This trait requires `fmt` with this exact signature. - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), - Error::IO(error) => Debug::fmt(&error, f), - Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), - Error::IndexTooHigh => write!(f, "Index too high"), - Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), - Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"), - Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"), - } - } -} -impl std::error::Error for Error {} diff --git a/storable_vec/src/main.rs b/storable_vec/src/main.rs index a71155def..8a32b4ed2 100644 --- a/storable_vec/src/main.rs +++ b/storable_vec/src/main.rs @@ -1,24 +1,24 @@ use std::path::Path; -use storable_vec::StorableVec; +use storable_vec::{StorableVec, Version}; fn main() -> Result<(), Box> { { - let mut vec: StorableVec = StorableVec::import(Path::new("./v"))?; + let mut vec: StorableVec = StorableVec::import(Path::new("./v"), Version::from(1))?; vec.push(0); vec.push(1); vec.push(2); - dbg!(vec.get(0)?); // Some(0) - dbg!(vec.get(21)?); // None + dbg!(vec.cached_get(0)?); // Some(0) + dbg!(vec.cached_get(21)?); // None vec.flush()?; } { - let vec: StorableVec = StorableVec::import(Path::new("./v"))?; + let vec: StorableVec = StorableVec::import(Path::new("./v"), Version::from(1))?; - dbg!(vec.get(0)?); // 0 + dbg!(vec.cached_get(0)?); // 0 } Ok(()) diff --git a/storable_vec/src/type_.rs b/storable_vec/src/type_.rs new file mode 100644 index 000000000..5fdeeeeef --- /dev/null +++ b/storable_vec/src/type_.rs @@ -0,0 +1,8 @@ +use std::fmt::Debug; + +pub trait StorableVecType +where + Self: Sized + Debug + Clone, +{ +} +impl StorableVecType for T where T: Sized + Debug + Clone {} diff --git a/storable_vec/src/value.rs b/storable_vec/src/value.rs new file mode 100644 index 000000000..7adaf7bb2 --- /dev/null +++ b/storable_vec/src/value.rs @@ -0,0 +1,39 @@ +use std::{fmt::Debug, ops::Deref}; + +#[derive(Debug, Clone)] +pub enum Value<'a, T> { + Ref(&'a T), + Owned(T), +} + +impl Value<'_, T> +where + T: Sized + Debug + Clone, +{ + pub fn into_inner(self) -> T { + match self { + Self::Ref(t) => t.to_owned(), + Self::Owned(t) => t, + } + } +} +impl Deref for Value<'_, T> { + type Target = T; + fn deref(&self) -> &Self::Target { + match self { + Self::Ref(t) => t, + Self::Owned(t) => t, + } + } +} +impl AsRef for Value<'_, T> +where + T: Sized + Debug + Clone, +{ + fn as_ref(&self) -> &T { + match self { + Self::Ref(t) => t, + Self::Owned(t) => t, + } + } +} diff --git a/storable_vec/src/version.rs b/storable_vec/src/version.rs new file mode 100644 index 000000000..e3c22c552 --- /dev/null +++ b/storable_vec/src/version.rs @@ -0,0 +1,29 @@ +use std::{ + fs, + io::{self, Read}, + path::Path, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Version(u32); + +impl Version { + pub fn write(&self, path: &Path) -> Result<(), io::Error> { + fs::write(path, self.0.to_le_bytes()) + } +} + +impl From for Version { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl TryFrom<&Path> for Version { + type Error = io::Error; + fn try_from(value: &Path) -> Result { + let mut buf = [0; 4]; + fs::read(value)?.as_slice().read_exact(&mut buf)?; + Ok(Self(u32::from_le_bytes(buf))) + } +} diff --git a/struct_iterable/struct_iterable_derive/Cargo.toml b/struct_iterable/struct_iterable_derive/Cargo.toml index c47efa491..3f591a895 100644 --- a/struct_iterable/struct_iterable_derive/Cargo.toml +++ b/struct_iterable/struct_iterable_derive/Cargo.toml @@ -13,5 +13,3 @@ proc-macro = true [dependencies] syn = "2.0.96" quote = "1.0.38" -proc-macro2 = "1.0.93" -struct_iterable_internal = { path = "../struct_iterable_internal" }