diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs index ac358bbf4..489142d26 100644 --- a/crates/brk_cli/src/query.rs +++ b/crates/brk_cli/src/query.rs @@ -14,7 +14,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> { indexer.import_vecs()?; let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed); - computer.import_vecs()?; + computer.import_vecs(&indexer)?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index d623b3111..c8be9f5f9 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -34,7 +34,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> { let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed); computer.import_stores(&indexer)?; - computer.import_vecs()?; + computer.import_vecs(&indexer)?; tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/brk_computer/examples/iter.rs b/crates/brk_computer/examples/iter.rs deleted file mode 100644 index 98963db28..000000000 --- a/crates/brk_computer/examples/iter.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::path::Path; - -use brk_core::dot_brk_path; -use brk_indexer::Indexer; - -pub fn main() -> color_eyre::Result<()> { - color_eyre::install()?; - - brk_logger::init(Some(Path::new(".log"))); - - let outputs_dir = dot_brk_path().join("outputs"); - // let outputs_dir = Path::new("../../_outputs"); - - let compressed = false; - - let mut indexer = Indexer::new(outputs_dir.as_path(), compressed, true)?; - indexer.import_stores()?; - indexer.import_vecs()?; - - let height_to_timestamp = &indexer.vecs().height_to_timestamp; - - dbg!(height_to_timestamp.len()); - - // height_to_timestamp.iter().for_each(|t| { - // dbg!(t); - // }); - - // let index = max_from.min(A::from(self.len())); - let mut height_to_timestamp_iter = height_to_timestamp.iter(); - // height_to_timestamp.iter().for_each(|t| { - // dbg!(t); - // }); - (0..2).for_each(|i| { - dbg!(height_to_timestamp_iter.get_(i)); - }); - // for_each(|t| { - // dbg!(t); - // }); - // .try_for_each(|(height, timestamp)| -> Result<()> { - // let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| { - // dbg!((height, prev_h)); - // let prev_timestamp = height_to_timestamp_iter - // .nth(prev_h.unwrap_to_usize()) - // .context("To work") - // .inspect_err(|_| { - // dbg!(prev_h); - // }) - // .unwrap() - // .1 - // .into_inner(); - // timestamp - // .into_inner() - // .checked_sub(prev_timestamp) - // .unwrap_or(Timestamp::ZERO) - // // Ok(()) - // }); - - // Ok(()) - // // let (i, v) = t((a, b.into_inner(), self, &mut other_iter)); - // // self.forced_push_at(i, v, exit) - // })?; - - Ok(()) -} diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index 85c1f116c..f7685d3ba 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -34,7 +34,7 @@ pub fn main() -> color_eyre::Result<()> { let mut computer = Computer::new(outputs_dir, Some(fetcher), compressed); computer.import_stores(&indexer)?; - computer.import_vecs()?; + computer.import_vecs(&indexer)?; let starting_indexes = indexer.index(&parser, rpc, &exit)?; diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 4edf1fa00..9170f274b 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -36,9 +36,10 @@ impl Computer { } } - pub fn import_vecs(&mut self) -> color_eyre::Result<()> { + pub fn import_vecs(&mut self, indexer: &Indexer) -> color_eyre::Result<()> { self.vecs = Some(Vecs::import( &self.path.join("vecs/computed"), + indexer, self.fetcher.is_some(), self.compressed, )?); diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 27eae7370..71d9e1050 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -7,7 +7,7 @@ use brk_core::{ use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::bitcoin; -use brk_vec::{Compressed, Version}; +use brk_vec::{Compressed, VecIterator, Version}; use super::{ EagerVec, Indexes, diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index 7744d757e..da68eb39d 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -2,7 +2,9 @@ use std::path::Path; use brk_core::{CheckedSub, StoredUsize}; use brk_exit::Exit; -use brk_vec::{Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, Version}; +use brk_vec::{ + Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, VecIterator, Version, +}; use color_eyre::eyre::ContextCompat; use crate::storage::{ComputedType, EagerVec}; diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 00f59ee93..b99d5e278 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -8,7 +8,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{Compressed, Version}; +use brk_vec::{Compressed, VecIterator, Version}; use super::EagerVec; diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index 931e598ec..ba1dcb447 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -7,7 +7,7 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{Compressed, Version}; +use brk_vec::{Compressed, VecIterator, Version}; use super::{ EagerVec, Indexes, diff --git a/crates/brk_computer/src/storage/vecs/mining.rs b/crates/brk_computer/src/storage/vecs/mining.rs index 2644c18ea..736fe7d61 100644 --- a/crates/brk_computer/src/storage/vecs/mining.rs +++ b/crates/brk_computer/src/storage/vecs/mining.rs @@ -3,7 +3,7 @@ use std::{fs, path::Path}; use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{Compressed, Version}; +use brk_vec::{Compressed, VecIterator, Version}; use super::{ Indexes, diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 7289b0450..6bc24fb10 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -26,14 +26,19 @@ pub struct Vecs { } impl Vecs { - pub fn import(path: &Path, fetch: bool, compressed: Compressed) -> color_eyre::Result { + pub fn import( + path: &Path, + indexer: &Indexer, + fetch: bool, + compressed: Compressed, + ) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { blocks: blocks::Vecs::forced_import(path, compressed)?, indexes: indexes::Vecs::forced_import(path, compressed)?, mining: mining::Vecs::forced_import(path, compressed)?, - transactions: transactions::Vecs::forced_import(path, compressed, fetch)?, + transactions: transactions::Vecs::forced_import(path, indexer, compressed, fetch)?, marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()), }) } diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 62799a44b..767780b57 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -6,10 +6,10 @@ use brk_core::{ use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::bitcoin; -use brk_vec::{Compressed, DynamicVec, StoredIndex, Version}; +use brk_vec::{Compressed, StoredIndex, VecIterator, Version}; use super::{ - EagerVec, Indexes, + EagerVec, Indexes, LazyVec, grouped::{ ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions, @@ -47,7 +47,7 @@ pub struct Vecs { pub indexes_to_tx_vsize: ComputedVecsFromTxindex, pub indexes_to_tx_weight: ComputedVecsFromTxindex, pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, - pub inputindex_to_value: EagerVec, + pub inputindex_to_value: LazyVec, pub indexes_to_input_count: ComputedVecsFromTxindex, pub txindex_to_is_coinbase: EagerVec, pub indexes_to_output_count: ComputedVecsFromTxindex, @@ -58,6 +58,7 @@ pub struct Vecs { impl Vecs { pub fn forced_import( path: &Path, + indexer: &Indexer, compressed: Compressed, compute_dollars: bool, ) -> color_eyre::Result { @@ -109,11 +110,34 @@ impl Vecs { .add_sum() .add_total(), )?, - inputindex_to_value: EagerVec::forced_import( - &path.join("inputindex_to_value"), + inputindex_to_value: LazyVec::init( + // &path.join("inputindex_to_value"), Version::ZERO, - compressed, - )?, + indexer.vecs().outputindex_to_value.vec().clone(), + indexer.vecs().inputindex_to_outputindex.vec().clone(), + |index, outputindex_to_value_iter, inputindex_to_outputindex_iter| { + // outputindex_to_value_iter.get(i) + // inputindex_to_outputindex_iter.get + // let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter(); + // self.inputindex_to_value.compute_transform( + // starting_indexes.inputindex, + // indexer.vecs().inputindex_to_outputindex.vec(), + // |(inputindex, outputindex, ..)| { + // let value = if outputindex == OutputIndex::COINBASE { + // Sats::ZERO + // } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { + // value.into_inner() + // } else { + // dbg!(inputindex, outputindex); + // panic!() + // }; + // (inputindex, value) + // }, + // exit, + // )?; + Some(Sats::ZERO) + }, + ), indexes_to_tx_v1: ComputedVecsFromHeight::forced_import( path, "tx_v1", @@ -517,24 +541,23 @@ impl Vecs { exit, )?; - let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter(); - let inputs_len = indexer.vecs().inputindex_to_outputindex.vec().len(); - self.inputindex_to_value.compute_transform( - starting_indexes.inputindex, - indexer.vecs().inputindex_to_outputindex.vec(), - |(inputindex, outputindex, ..)| { - let value = if outputindex == OutputIndex::COINBASE { - Sats::ZERO - } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { - value.into_inner() - } else { - dbg!(inputindex, outputindex, inputs_len); - panic!() - }; - (inputindex, value) - }, - exit, - )?; + // let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter(); + // self.inputindex_to_value.compute_transform( + // starting_indexes.inputindex, + // indexer.vecs().inputindex_to_outputindex.vec(), + // |(inputindex, outputindex, ..)| { + // let value = if outputindex == OutputIndex::COINBASE { + // Sats::ZERO + // } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { + // value.into_inner() + // } else { + // dbg!(inputindex, outputindex); + // panic!() + // }; + // (inputindex, value) + // }, + // exit, + // )?; self.indexes_to_output_value.compute_all( indexer, @@ -552,21 +575,21 @@ impl Vecs { }, )?; - self.indexes_to_input_value.compute_all( - indexer, - indexes, - starting_indexes, - exit, - |vec, indexer, _, starting_indexes, exit| { - vec.compute_sum_from_indexes( - starting_indexes.txindex, - indexer.vecs().txindex_to_first_inputindex.vec(), - self.indexes_to_input_count.txindex.as_ref().unwrap().vec(), - self.inputindex_to_value.vec(), - exit, - ) - }, - )?; + // self.indexes_to_input_value.compute_all( + // indexer, + // indexes, + // starting_indexes, + // exit, + // |vec, indexer, _, starting_indexes, exit| { + // vec.compute_sum_from_indexes( + // starting_indexes.txindex, + // indexer.vecs().txindex_to_first_inputindex.vec(), + // self.indexes_to_input_count.txindex.as_ref().unwrap().vec(), + // self.inputindex_to_value.vec(), + // exit, + // ) + // }, + // )?; self.indexes_to_fee.compute_all( indexer, @@ -868,7 +891,7 @@ impl Vecs { [ vec![ self.txindex_to_is_coinbase.any_vec(), - self.inputindex_to_value.any_vec(), + // self.inputindex_to_value.any_vec(), self.txindex_to_weight.any_vec(), self.txindex_to_vsize.any_vec(), ], diff --git a/crates/brk_computer/src/storage/vecs/vec/eager.rs b/crates/brk_computer/src/storage/vecs/vec/eager.rs index 8298bc91e..e95bdd77b 100644 --- a/crates/brk_computer/src/storage/vecs/vec/eager.rs +++ b/crates/brk_computer/src/storage/vecs/vec/eager.rs @@ -10,7 +10,7 @@ use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, T use brk_exit::Exit; use brk_vec::{ Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, - StoredVecIterator, Value, Version, + StoredVecIterator, Value, VecIterator, Version, }; use log::info; @@ -211,10 +211,7 @@ where )?; let index = max_from.min( - self.inner - .iter() - .last() - .map_or_else(T::default, |(_, v)| v.into_inner()), + VecIterator::last(self.inner.iter()).map_or_else(T::default, |(_, v)| v.into_inner()), ); let mut prev_i = None; other.iter_at(index).try_for_each(|(v, i)| -> Result<()> { diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy.rs b/crates/brk_computer/src/storage/vecs/vec/lazy.rs index 76ee8e2e7..1517b2412 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy.rs +++ b/crates/brk_computer/src/storage/vecs/vec/lazy.rs @@ -1,43 +1,204 @@ -use std::fmt::Debug; +use std::marker::PhantomData; -use brk_vec::{DynamicVec, GenericVec, StoredIndex, StoredType, StoredVec, Version}; +use brk_vec::{StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version}; -#[derive(Debug)] -pub struct LazyVec +// LazyVec owns SourceVecs +// +// Functions: +// init() +// version() +// iter() +// len() ? +// +// When .iter() called convert SourcesVecs into iterators +// iter owns source iters +// call compute function to convert index and source iters to T + +#[derive(Clone)] +pub struct LazyVec where I: StoredIndex, T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, { - inner: StoredVec, + version: Version, + source1: StoredVec, + source2: StoredVec, + compute: for<'a> fn( + usize, + &mut (dyn Iterator)>), + &mut (dyn Iterator)>), + ) -> Option, + phantom: PhantomData, } -impl LazyVec +impl LazyVec where I: StoredIndex, T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, { - const SIZE_OF: usize = size_of::(); + pub fn init( + version: Version, + source1: StoredVec, + source2: StoredVec, + compute: for<'a> fn( + usize, + &mut (dyn Iterator)>), + &mut (dyn Iterator)>), + ) -> Option, + ) -> Self { + Self { + version, + source1, + source2, + compute, + phantom: PhantomData, + } + } fn version(&self) -> Version { - self.inner.version() + self.version } - pub fn len(&self) -> usize { - self.inner.len() - } - pub fn is_empty(&self) -> bool { - self.inner.is_empty() + fn len(&self) -> usize { + todo!() } } -impl Clone for LazyVec +pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), + lazy: &'a LazyVec, + source1: StoredVecIterator<'a, S1I, S1T>, + source2: StoredVecIterator<'a, S2I, S2T>, + index: usize, +} + +impl LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T> +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ + #[inline] + pub fn set(&mut self, i: I) -> &mut Self { + self.index = i.unwrap_to_usize(); + self + } + + #[inline] + pub fn set_(&mut self, i: usize) { + self.index = i; + } + + #[inline] + pub fn get(&mut self, i: I) -> Option> { + self.set(i).next().map(|(_, v)| v) + } + + #[inline] + pub fn get_(&mut self, i: usize) -> Option> { + self.set_(i); + self.next().map(|(_, v)| v) + } +} + +// impl<'a, I, T, S1I, S1T, S2I, S2T> VecIterator<'a> for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> +// where +// I: StoredIndex, +// T: StoredType, +// S1I: StoredIndex, +// S1T: StoredType, +// S2I: StoredIndex, +// S2T: StoredType, +// { +// type I = I; +// type T = T; + +// #[inline] +// fn mut_index(&mut self) -> &mut usize { +// &mut self.index +// } + +// #[inline] +// fn len(&self) -> usize { +// todo!(); +// // self.vec.len() +// } +// } + +impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ + type Item = (I, Value<'a, T>); + + fn next(&mut self) -> Option { + let opt = (self.lazy.compute)( + self.index, + &mut self.lazy.source1.iter(), + &mut self.lazy.source2.iter(), + ) + .map(|v| (I::from(self.index), Value::Owned(v))); + if opt.is_some() { + self.index += 1; + } + opt + } + + // fn last(mut self) -> Option + // where + // Self: Sized, + // { + // let len = self.vec.len(); + // if len == 0 { + // return None; + // } + // let i = len - 1; + // self.get_(i) + // .map(|v| (I::from(i), Value::Owned(v.into_inner()))) + // } +} + +impl<'a, I, T, S1I, S1T, S2I, S2T> IntoIterator for &'a LazyVec +where + I: StoredIndex, + T: StoredType + 'a, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ + type Item = (I, Value<'a, T>); + type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>; + + fn into_iter(self) -> Self::IntoIter { + LazyVecIterator { + lazy: self, + source1: self.source1.iter(), + source2: self.source2.iter(), + index: 0, } } } diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index e55acab46..9a31151a3 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -5,7 +5,7 @@ use brk_core::{ P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_vec::{StoredIndex, StoredType}; +use brk_vec::{StoredIndex, StoredType, VecIterator}; use color_eyre::eyre::ContextCompat; use crate::{IndexedVec, Stores, Vecs}; diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index b0708e07c..166dbc556 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -18,7 +18,7 @@ pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; -use brk_vec::{Compressed, DynamicVec}; +use brk_vec::{Compressed, DynamicVec, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use fjall::TransactionalKeyspace; use log::{error, info}; diff --git a/crates/brk_indexer/src/stores/mod.rs b/crates/brk_indexer/src/stores/mod.rs index 777c0ba21..5156ea4f4 100644 --- a/crates/brk_indexer/src/stores/mod.rs +++ b/crates/brk_indexer/src/stores/mod.rs @@ -4,7 +4,7 @@ use brk_core::{ AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex, TxidPrefix, }; -use brk_vec::{Value, Version}; +use brk_vec::{Value, VecIterator, Version}; use fjall::{PersistMode, TransactionalKeyspace}; use crate::Indexes; diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index 464897c77..c7f8fe73b 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -6,7 +6,7 @@ use std::{ use brk_vec::{ Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec, - StoredVecIterator, Value, Version, + StoredVecIterator, Value, VecIterator, Version, }; use super::Height; diff --git a/crates/brk_query/examples/main.rs b/crates/brk_query/examples/main.rs index a59fce8b5..21e5e5412 100644 --- a/crates/brk_query/examples/main.rs +++ b/crates/brk_query/examples/main.rs @@ -15,7 +15,7 @@ pub fn main() -> color_eyre::Result<()> { indexer.import_vecs()?; let mut computer = Computer::new(outputs_dir, None, compressed); - computer.import_vecs()?; + computer.import_vecs(&indexer)?; let query = Query::build(&indexer, &computer); diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 0d2ac3426..8c50767e2 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -40,7 +40,7 @@ pub fn main() -> color_eyre::Result<()> { let mut computer = Computer::new(outputs_dir, fetcher, compressed); computer.import_stores(&indexer)?; - computer.import_vecs()?; + computer.import_vecs(&indexer)?; tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index 500a2a6b0..d8ca84fae 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_vec::{Compressed, DynamicVec, GenericVec, StoredVec, Version}; +use brk_vec::{Compressed, DynamicVec, GenericVec, StoredVec, VecIterator, Version}; fn main() -> Result<(), Box> { let _ = fs::remove_dir_all("./vec"); @@ -64,14 +64,14 @@ fn main() -> Result<(), Box> { let mut iter = vec.iter(); - dbg!(iter.get(0)); - dbg!(iter.get(5)); + iter.get(0); + iter.get(5); dbg!(iter.get(20)); dbg!(vec.collect_signed_range(Some(-5), None)?); vec.push(vec.len() as u32); - dbg!(vec.iter().last()); + dbg!(VecIterator::last(vec.iter())); dbg!(vec.into_iter().collect::>()); } diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 3729dd09c..46060a958 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -218,45 +218,30 @@ where Compressed(CompressedVecIterator<'a, I, T>), } -impl StoredVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - pub fn unwrap_get_inner(&mut self, i: I) -> T { - self.get_(i.unwrap_to_usize()).unwrap().into_inner() - } +// impl StoredVecIterator<'_, I, T> +// impl<'a, I, T> VecIterator<'a> for StoredVecIterator<'a, I, T> +// where +// I: StoredIndex, +// T: StoredType, +// { +// type I = I; +// type T = T; - #[inline] - pub fn get_inner(&mut self, i: I) -> Option { - self.get_(i.unwrap_to_usize()).map(|v| v.into_inner()) - } +// #[inline] +// fn mut_index(&mut self) -> &mut usize { +// match self { +// Self::Compressed(iter) => iter.mut_index(), +// Self::Raw(iter) => iter.mut_index(), +// } +// } - #[inline] - pub fn get(&mut self, i: I) -> Option> { - self.get_(i.unwrap_to_usize()) - } - - #[inline] - pub fn get_(&mut self, i: usize) -> Option> { - match self { - Self::Compressed(iter) => iter.get_(i), - Self::Raw(iter) => iter.get_(i), - } - } - - pub fn set(&mut self, i: I) { - match self { - Self::Compressed(iter) => { - iter.set(i); - } - Self::Raw(iter) => { - iter.set(i); - } - }; - } -} +// fn len(&self) -> usize { +// match self { +// Self::Compressed(i) => i.len(), +// Self::Raw(i) => i.len(), +// } +// } +// } impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T> where @@ -270,23 +255,6 @@ where Self::Raw(i) => i.next(), } } - - fn last(self) -> Option - where - Self: Sized, - { - match self { - Self::Compressed(i) => i.last(), - Self::Raw(i) => i.last(), - } - } - - fn skip(self, _: usize) -> std::iter::Skip - where - Self: Sized, - { - todo!("") - } } impl<'a, I, T> IntoIterator for &'a StoredVec diff --git a/crates/brk_vec/src/traits/iterator.rs b/crates/brk_vec/src/traits/iterator.rs new file mode 100644 index 000000000..95d0d96aa --- /dev/null +++ b/crates/brk_vec/src/traits/iterator.rs @@ -0,0 +1,129 @@ +use crate::Value; + +use super::{StoredIndex, StoredType}; + +// pub trait BaseVecIterator: Iterator { +// fn mut_index(&mut self) -> &mut usize; + +// fn len(&self) -> usize; + +// fn is_empty(&self) -> bool { +// self.len() == 0 +// } + +// #[inline] +// fn set_(&mut self, i: usize) -> &mut Self { +// *self.mut_index() = i; +// self +// } + +// fn skip(self, _: usize) -> std::iter::Skip +// where +// Self: Sized, +// { +// todo!("") +// } +// } + +pub trait VecIterator<'a>: Iterator)> + 'a { + type I: StoredIndex; + type T: StoredType; + + fn mut_index(&mut self) -> &mut usize; + + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + fn set_(&mut self, i: usize) -> &mut Self { + *self.mut_index() = i; + self + } + + fn skip(self, _: usize) -> std::iter::Skip + where + Self: Sized, + { + todo!("") + } + + // fn set(&mut self, i: Self::I) -> &mut Self; + + // fn get_(&mut self, i: usize) -> Option>; + + // fn get(&mut self, i: Self::I) -> Option>; + + // fn unwrap_get_inner(&mut self, i: Self::I) -> Self::T; + + // fn get_inner(&mut self, i: Self::I) -> Option; + + // fn last(self) -> Option; + // } + + // impl<'a, I, T, Iter> VecIterator<'a, I, T> for Iter + // where + // I: StoredIndex, + // T: StoredType + 'a, + // Iter: Iterator)> + BaseVecIterator, + // { + #[inline] + fn set(&mut self, i: Self::I) -> &mut Self { + self.set_(i.unwrap_to_usize()) + } + + #[inline] + fn get_(&mut self, i: usize) -> Option> { + self.set_(i); + self.next().map(|(_, v)| v) + } + + #[inline] + fn get(&mut self, i: Self::I) -> Option> { + self.get_(i.unwrap_to_usize()) + } + + #[inline] + fn unwrap_get_inner(&mut self, i: Self::I) -> Self::T { + self.get_(i.unwrap_to_usize()).unwrap().into_inner() + } + + #[inline] + fn get_inner(&mut self, i: Self::I) -> Option { + self.get_(i.unwrap_to_usize()).map(|v| v.into_inner()) + } + + fn last(mut self) -> Option + where + Self: Sized, + { + let len = self.len(); + if len == 0 { + return None; + } + let i = len - 1; + self.set_(i); + self.next().map(|(i, v)| (i, Value::Owned(v.into_inner()))) + } +} + +impl<'a, I, T, Iter> VecIterator<'a> for Iter +where + Iter: Iterator)> + 'a, + I: StoredIndex, + T: StoredType + 'a, +{ + type I = I; + type T = T; + + fn len(&self) -> usize { + todo!() + } + + fn mut_index(&mut self) -> &mut usize { + todo!() + } +} +// pub trait VecIterator<'a>: Iterator)> + 'a { diff --git a/crates/brk_vec/src/traits/mod.rs b/crates/brk_vec/src/traits/mod.rs index 7f210ba99..c8891be5c 100644 --- a/crates/brk_vec/src/traits/mod.rs +++ b/crates/brk_vec/src/traits/mod.rs @@ -1,9 +1,11 @@ mod dynamic; mod generic; +mod iterator; mod stored_index; mod stored_type; pub use dynamic::*; pub use generic::*; +pub use iterator::*; pub use stored_index::*; pub use stored_type::*; diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 07f9c7c0f..fed570dce 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -12,7 +12,7 @@ use zstd::DEFAULT_COMPRESSION_LEVEL; use crate::{ CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error, GenericVec, RawVec, Result, - StoredIndex, StoredType, UnsafeSlice, Value, Version, + StoredIndex, StoredType, UnsafeSlice, Value, VecIterator, Version, }; const ONE_KIB: usize = 1024; @@ -390,36 +390,34 @@ where { const SIZE_OF_T: usize = size_of::(); const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T; - - #[inline] - pub fn set(&mut self, i: I) -> &mut Self { - self.index = i.unwrap_to_usize(); - self - } - - #[inline] - pub fn set_(&mut self, i: usize) { - self.index = i; - } - - #[inline] - pub fn get(&mut self, i: I) -> Option> { - self.set(i).next().map(|(_, v)| v) - } - - #[inline] - pub fn get_(&mut self, i: usize) -> Option> { - self.set_(i); - self.next().map(|(_, v)| v) - } } +// impl<'a, I, T> VecIterator<'a> for CompressedVecIterator<'a, I, T> +// where +// I: StoredIndex, +// T: StoredType, +// { +// type I = I; +// type T = T; + +// #[inline] +// fn mut_index(&mut self) -> &mut usize { +// &mut self.index +// } + +// #[inline] +// fn len(&self) -> usize { +// self.vec.len() +// } +// } + impl<'a, I, T> Iterator for CompressedVecIterator<'a, I, T> where I: StoredIndex, T: StoredType, { type Item = (I, Value<'a, T>); + fn next(&mut self) -> Option { let mmap = &self.guard; let i = self.index; @@ -460,20 +458,6 @@ where result } - - #[inline] - fn last(mut self) -> Option - where - Self: Sized, - { - let len = self.vec.len(); - if len == 0 { - return None; - } - let i = len - 1; - self.get_(i) - .map(|v| (I::from(i), Value::Owned(v.into_inner()))) - } } impl<'a, I, T> IntoIterator for &'a CompressedVec diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 49a270e9c..0743bdd35 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -11,7 +11,8 @@ use memmap2::Mmap; use rayon::prelude::*; use crate::{ - DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version, + DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, + VecIterator, Version, }; #[derive(Debug)] @@ -216,33 +217,24 @@ pub struct RawVecIterator<'a, I, T> { index: usize, } -impl RawVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - pub fn set(&mut self, i: I) -> &mut Self { - self.index = i.unwrap_to_usize(); - self - } +// impl<'a, I, T> VecIterator<'a> for RawVecIterator<'a, I, T> +// where +// I: StoredIndex, +// T: StoredType, +// { +// type I = I; +// type T = T; - #[inline] - pub fn set_(&mut self, i: usize) { - self.index = i; - } +// #[inline] +// fn mut_index(&mut self) -> &mut usize { +// &mut self.index +// } - #[inline] - pub fn get(&mut self, i: I) -> Option> { - self.set(i).next().map(|(_, v)| v) - } - - #[inline] - pub fn get_(&mut self, i: usize) -> Option> { - self.set_(i); - self.next().map(|(_, v)| v) - } -} +// #[inline] +// fn len(&self) -> usize { +// self.vec.len() +// } +// } impl<'a, I, T> Iterator for RawVecIterator<'a, I, T> where @@ -250,6 +242,7 @@ where T: StoredType, { type Item = (I, Value<'a, T>); + fn next(&mut self) -> Option { let mmap = &self.guard; let index = self.index; @@ -266,19 +259,6 @@ where opt } - - fn last(mut self) -> Option - where - Self: Sized, - { - let len = self.vec.len(); - if len == 0 { - return None; - } - let i = len - 1; - self.get_(i) - .map(|v| (I::from(i), Value::Owned(v.into_inner()))) - } } impl<'a, I, T> IntoIterator for &'a RawVec