diff --git a/.gitignore b/.gitignore index 6c8b845bd..ce5f5f453 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,5 @@ paths.d.ts # Outputs _outputs - # Logs .log diff --git a/Cargo.lock b/Cargo.lock index 11a72bd5b..458a418bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,6 +409,7 @@ dependencies = [ "brk_parser", "brk_query", "brk_server", + "brk_vec", "clap", "clap_derive", "color-eyre", @@ -553,6 +554,7 @@ dependencies = [ "brk_logger", "brk_parser", "brk_query", + "brk_vec", "clap", "clap_derive", "color-eyre", @@ -572,6 +574,11 @@ name = "brk_vec" version = "0.0.31" dependencies = [ "arc-swap", + "brk_core", + "brk_exit", + "clap", + "clap_derive", + "log", "memmap2", "rayon", "serde", diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 87b87dd1d..2d00d2904 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -16,6 +16,7 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_query = { workspace = true } brk_server = { workspace = true } +brk_vec = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 16dcd1023..f9ab8b780 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -5,13 +5,14 @@ use std::{ time::Duration, }; -use brk_computer::{Computation, Computer}; +use brk_computer::Computer; use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::rpc::{self, Auth, Client, RpcApi}; use brk_server::{Server, Website, tokio}; +use brk_vec::Computation; use clap_derive::{Parser, ValueEnum}; use color_eyre::eyre::eyre; use log::info; diff --git a/crates/brk_computer/examples/main.rs b/crates/brk_computer/examples/main.rs index 32287f2f8..e0174ece6 100644 --- a/crates/brk_computer/examples/main.rs +++ b/crates/brk_computer/examples/main.rs @@ -1,11 +1,12 @@ use std::path::Path; -use brk_computer::{Computation, Computer}; +use brk_computer::Computer; use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::{Parser, rpc}; +use brk_vec::Computation; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index be1c0e683..19a88d78f 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -12,9 +12,8 @@ pub use brk_parser::rpc; mod storage; -use brk_vec::Compressed; +use brk_vec::{AnyCollectableVec, Compressed, Computation}; use log::info; -pub use storage::Computation; use storage::{Stores, Vecs}; #[derive(Clone)] @@ -82,13 +81,14 @@ impl Computer { Ok(()) } - pub fn vecs(&self) -> &Vecs { - self.vecs.as_ref().unwrap() + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + // pub fn vecs(&self) -> &Vecs { + self.vecs.as_ref().unwrap().vecs() } - pub fn mut_vecs(&mut self) -> &mut Vecs { - self.vecs.as_mut().unwrap() - } + // pub fn mut_vecs(&mut self) -> &mut Vecs { + // self.vecs.as_mut().unwrap() + // } pub fn stores(&self) -> &Stores { self.stores.as_ref().unwrap() diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 8436c83f7..d5b481625 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -7,10 +7,10 @@ use brk_core::{ use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::bitcoin; -use brk_vec::{Compressed, VecIterator, Version}; +use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version}; use super::{ - EagerVec, Indexes, + Indexes, grouped::{ComputedVecsFromDateindex, ComputedVecsFromHeight, StorableVecGeneatorOptions}, indexes, }; @@ -30,7 +30,11 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import( + path: &Path, + computation: Computation, + compressed: Compressed, + ) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { @@ -223,7 +227,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ vec![ self.height_to_interval.any_vec(), @@ -231,12 +235,12 @@ impl Vecs { self.difficultyepoch_to_timestamp.any_vec(), self.halvingepoch_to_timestamp.any_vec(), ], - self.timeindexes_to_timestamp.any_vecs(), - self.indexes_to_block_count.any_vecs(), - self.indexes_to_block_interval.any_vecs(), - self.indexes_to_block_size.any_vecs(), - self.indexes_to_block_vbytes.any_vecs(), - self.indexes_to_block_weight.any_vecs(), + self.timeindexes_to_timestamp.vecs(), + self.indexes_to_block_count.vecs(), + self.indexes_to_block_interval.vecs(), + self.indexes_to_block_size.vecs(), + self.indexes_to_block_vbytes.vecs(), + self.indexes_to_block_weight.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index 4b8233134..1a39cc8d9 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -3,11 +3,12 @@ use std::path::Path; use brk_core::{CheckedSub, StoredUsize}; use brk_exit::Exit; use brk_vec::{ - Compressed, DynamicVec, Result, StoredIndex, StoredType, StoredVec, VecIterator, Version, + AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredIndex, StoredType, VecIterator, + Version, }; use color_eyre::eyre::ContextCompat; -use crate::storage::{ComputedType, EagerVec}; +use super::ComputedType; #[derive(Clone, Debug)] pub struct ComputedVecBuilder @@ -137,7 +138,12 @@ where Ok(s) } - pub fn extend(&mut self, max_from: I, source: &StoredVec, exit: &Exit) -> Result<()> { + pub fn extend( + &mut self, + max_from: I, + source: &impl AnyIterableVec, + exit: &Exit, + ) -> Result<()> { if self.total.is_none() { return Ok(()); }; @@ -162,9 +168,9 @@ where pub fn compute( &mut self, max_from: I, - source: &StoredVec, - first_indexes: &StoredVec, - count_indexes: &StoredVec, + source: &impl AnyIterableVec, + first_indexes: &impl AnyIterableVec, + count_indexes: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -244,12 +250,12 @@ where dbg!( &values, max.path(), - first_indexes.path(), + first_indexes.name(), first_index, - count_indexes.path(), + count_indexes.name(), count_index, source.len(), - source.path() + source.name() ); }) .unwrap() @@ -319,8 +325,8 @@ where &mut self, max_from: I, source: &ComputedVecBuilder, - first_indexes: &StoredVec, - count_indexes: &StoredVec, + first_indexes: &impl AnyIterableVec, + count_indexes: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -482,7 +488,7 @@ where fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.any_vecs().into_iter().map(|v| v.len()).min().unwrap(), + self.vecs().into_iter().map(|v| v.len()).min().unwrap(), )) } @@ -523,7 +529,7 @@ where self.total.as_mut().unwrap() } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { let mut v: Vec<&dyn brk_vec::AnyVec> = vec![]; if let Some(first) = self.first.as_ref() { diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index 4c091e7b8..63309f6ba 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -3,11 +3,11 @@ use std::path::Path; use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, Version}; +use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version}; -use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; +use crate::storage::{Indexes, indexes}; -use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromDateindex @@ -95,60 +95,60 @@ where )?; self.dateindex_extra - .extend(starting_indexes.dateindex, self.dateindex.vec(), exit)?; + .extend(starting_indexes.dateindex, self.dateindex.iter_vec(), exit)?; self.weekindex.compute( starting_indexes.weekindex, - self.dateindex.vec(), - indexes.weekindex_to_first_dateindex.vec(), - indexes.weekindex_to_dateindex_count.vec(), + self.dateindex.iter_vec(), + indexes.weekindex_to_first_dateindex.iter_vec(), + indexes.weekindex_to_dateindex_count.iter_vec(), exit, )?; self.monthindex.compute( starting_indexes.monthindex, - self.dateindex.vec(), - indexes.monthindex_to_first_dateindex.vec(), - indexes.monthindex_to_dateindex_count.vec(), + self.dateindex.iter_vec(), + indexes.monthindex_to_first_dateindex.iter_vec(), + indexes.monthindex_to_dateindex_count.iter_vec(), exit, )?; self.quarterindex.from_aligned( starting_indexes.quarterindex, &self.monthindex, - indexes.quarterindex_to_first_monthindex.vec(), - indexes.quarterindex_to_monthindex_count.vec(), + indexes.quarterindex_to_first_monthindex.iter_vec(), + indexes.quarterindex_to_monthindex_count.iter_vec(), exit, )?; self.yearindex.from_aligned( starting_indexes.yearindex, &self.monthindex, - indexes.yearindex_to_first_monthindex.vec(), - indexes.yearindex_to_monthindex_count.vec(), + indexes.yearindex_to_first_monthindex.iter_vec(), + indexes.yearindex_to_monthindex_count.iter_vec(), exit, )?; self.decadeindex.from_aligned( starting_indexes.decadeindex, &self.yearindex, - indexes.decadeindex_to_first_yearindex.vec(), - indexes.decadeindex_to_yearindex_count.vec(), + indexes.decadeindex_to_first_yearindex.iter_vec(), + indexes.decadeindex_to_yearindex_count.iter_vec(), exit, )?; Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ vec![self.dateindex.any_vec()], - self.dateindex_extra.any_vecs(), - self.weekindex.any_vecs(), - self.monthindex.any_vecs(), - self.quarterindex.any_vecs(), - self.yearindex.any_vecs(), - self.decadeindex.any_vecs(), + self.dateindex_extra.vecs(), + self.weekindex.vecs(), + self.monthindex.vecs(), + self.quarterindex.vecs(), + self.yearindex.vecs(), + self.decadeindex.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index be13052bb..23f740571 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -5,11 +5,11 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version}; -use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; +use crate::storage::{Indexes, indexes}; -use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeight @@ -114,9 +114,9 @@ where indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - height: Option<&StoredVec>, + height: Option<&impl AnyIterableVec>, ) -> color_eyre::Result<()> { - let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().vec()); + let height = height.unwrap_or_else(|| self.height.as_ref().unwrap().iter_vec()); self.height_extra .extend(starting_indexes.height, height, exit)?; @@ -124,74 +124,74 @@ where self.dateindex.compute( starting_indexes.dateindex, height, - indexes.dateindex_to_first_height.vec(), - indexes.dateindex_to_height_count.vec(), + indexes.dateindex_to_first_height.iter_vec(), + indexes.dateindex_to_height_count.iter_vec(), exit, )?; self.weekindex.from_aligned( starting_indexes.weekindex, &self.dateindex, - indexes.weekindex_to_first_dateindex.vec(), - indexes.weekindex_to_dateindex_count.vec(), + indexes.weekindex_to_first_dateindex.iter_vec(), + indexes.weekindex_to_dateindex_count.iter_vec(), exit, )?; self.monthindex.from_aligned( starting_indexes.monthindex, &self.dateindex, - indexes.monthindex_to_first_dateindex.vec(), - indexes.monthindex_to_dateindex_count.vec(), + indexes.monthindex_to_first_dateindex.iter_vec(), + indexes.monthindex_to_dateindex_count.iter_vec(), exit, )?; self.quarterindex.from_aligned( starting_indexes.quarterindex, &self.monthindex, - indexes.quarterindex_to_first_monthindex.vec(), - indexes.quarterindex_to_monthindex_count.vec(), + indexes.quarterindex_to_first_monthindex.iter_vec(), + indexes.quarterindex_to_monthindex_count.iter_vec(), exit, )?; self.yearindex.from_aligned( starting_indexes.yearindex, &self.monthindex, - indexes.yearindex_to_first_monthindex.vec(), - indexes.yearindex_to_monthindex_count.vec(), + indexes.yearindex_to_first_monthindex.iter_vec(), + indexes.yearindex_to_monthindex_count.iter_vec(), exit, )?; self.decadeindex.from_aligned( starting_indexes.decadeindex, &self.yearindex, - indexes.decadeindex_to_first_yearindex.vec(), - indexes.decadeindex_to_yearindex_count.vec(), + indexes.decadeindex_to_first_yearindex.iter_vec(), + indexes.decadeindex_to_yearindex_count.iter_vec(), exit, )?; self.difficultyepoch.compute( starting_indexes.difficultyepoch, height, - indexes.difficultyepoch_to_first_height.vec(), - indexes.difficultyepoch_to_height_count.vec(), + indexes.difficultyepoch_to_first_height.iter_vec(), + indexes.difficultyepoch_to_height_count.iter_vec(), exit, )?; Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]), - self.height_extra.any_vecs(), - self.dateindex.any_vecs(), - self.weekindex.any_vecs(), - self.difficultyepoch.any_vecs(), - self.monthindex.any_vecs(), - self.quarterindex.any_vecs(), - self.yearindex.any_vecs(), - // self.halvingepoch.any_vecs(), - self.decadeindex.any_vecs(), + self.height_extra.vecs(), + self.dateindex.vecs(), + self.weekindex.vecs(), + self.difficultyepoch.vecs(), + self.monthindex.vecs(), + self.quarterindex.vecs(), + self.yearindex.vecs(), + // self.halvingepoch.vecs(), + self.decadeindex.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 9847197b7..e98a4520b 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -3,11 +3,11 @@ use std::path::Path; use brk_core::{DifficultyEpoch, Height}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, Version}; +use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, Version}; -use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; +use crate::storage::{Indexes, indexes}; -use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromHeightStrict @@ -73,25 +73,25 @@ where compute(&mut self.height, indexer, indexes, starting_indexes, exit)?; self.height_extra - .extend(starting_indexes.height, self.height.vec(), exit)?; + .extend(starting_indexes.height, self.height.iter_vec(), exit)?; self.difficultyepoch.compute( starting_indexes.difficultyepoch, - self.height.vec(), - indexes.difficultyepoch_to_first_height.vec(), - indexes.difficultyepoch_to_height_count.vec(), + self.height.iter_vec(), + indexes.difficultyepoch_to_first_height.iter_vec(), + indexes.difficultyepoch_to_height_count.iter_vec(), exit, )?; Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ vec![self.height.any_vec()], - self.height_extra.any_vecs(), - self.difficultyepoch.any_vecs(), - // self.halvingepoch.any_vecs(), + self.height_extra.vecs(), + self.difficultyepoch.vecs(), + // self.halvingepoch.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index ec66f9684..0ed333e48 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -6,11 +6,11 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version}; -use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; +use crate::storage::{Indexes, indexes}; -use super::{ComputedVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct ComputedVecsFromTxindex @@ -119,7 +119,7 @@ where exit: &Exit, txindex: Option<&StoredVec>, ) -> color_eyre::Result<()> { - let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().vec()); + let txindex = txindex.unwrap_or_else(|| self.txindex.as_ref().unwrap().iter_vec()); self.height.compute( starting_indexes.height, @@ -188,18 +188,18 @@ where Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]), - self.height.any_vecs(), - self.dateindex.any_vecs(), - self.weekindex.any_vecs(), - self.difficultyepoch.any_vecs(), - self.monthindex.any_vecs(), - self.quarterindex.any_vecs(), - self.yearindex.any_vecs(), - // self.halvingepoch.any_vecs(), - self.decadeindex.any_vecs(), + self.height.vecs(), + self.dateindex.vecs(), + self.weekindex.vecs(), + self.difficultyepoch.vecs(), + self.monthindex.vecs(), + self.quarterindex.vecs(), + self.yearindex.vecs(), + // self.halvingepoch.vecs(), + self.decadeindex.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/mod.rs b/crates/brk_computer/src/storage/vecs/grouped/mod.rs index 54861f84d..92df66067 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/mod.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/mod.rs @@ -3,6 +3,7 @@ mod from_dateindex; mod from_height; mod from_height_strict; mod from_txindex; +mod r#type; mod value_from_height; mod value_from_txindex; @@ -11,5 +12,6 @@ pub use from_dateindex::*; pub use from_height::*; pub use from_height_strict::*; pub use from_txindex::*; +use r#type::*; pub use value_from_height::*; pub use value_from_txindex::*; diff --git a/crates/brk_computer/src/storage/vecs/vec/_type.rs b/crates/brk_computer/src/storage/vecs/grouped/type.rs similarity index 100% rename from crates/brk_computer/src/storage/vecs/vec/_type.rs rename to crates/brk_computer/src/storage/vecs/grouped/type.rs diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs index 57d89caa6..a7c7794b7 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs @@ -3,10 +3,10 @@ use std::path::Path; use brk_core::{Bitcoin, Dollars, Height, Sats}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyIterableVec, AnyVec, Compressed, EagerVec, Result, StoredVec, Version}; use crate::storage::{ - EagerVec, marketprice, + marketprice, vecs::{Indexes, indexes}, }; @@ -110,7 +110,7 @@ impl ComputedValueVecsFromHeight { .compute_rest(indexes, starting_indexes, exit, None)?; } - let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().vec()); + let height = height.unwrap_or_else(|| self.sats.height.as_ref().unwrap().iter_vec()); self.bitcoin.compute_all( indexer, @@ -122,13 +122,13 @@ impl ComputedValueVecsFromHeight { }, )?; - let txindex = self.bitcoin.height.as_ref().unwrap().vec(); + let txindex = self.bitcoin.height.as_ref().unwrap().iter_vec(); let price = marketprices .as_ref() .unwrap() .chainindexes_to_close .height - .vec(); + .iter_vec(); if let Some(dollars) = self.dollars.as_mut() { dollars.compute_all( @@ -145,11 +145,11 @@ impl ComputedValueVecsFromHeight { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ - self.sats.any_vecs(), - self.bitcoin.any_vecs(), - self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()), + self.sats.vecs(), + self.bitcoin.vecs(), + self.dollars.as_ref().map_or(vec![], |v| v.vecs()), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs index c66fc4bb6..d01490c07 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs @@ -3,10 +3,10 @@ use std::path::Path; use brk_core::{Bitcoin, Dollars, Sats, TxIndex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyVec, Compressed, EagerVec, Result, StoredVec, Version}; use crate::storage::{ - EagerVec, marketprice, + marketprice, vecs::{Indexes, indexes}, }; @@ -147,11 +147,11 @@ impl ComputedValueVecsFromTxindex { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyVec> { [ - self.sats.any_vecs(), - self.bitcoin.any_vecs(), - self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()), + self.sats.vecs(), + self.bitcoin.vecs(), + self.dollars.as_ref().map_or(vec![], |v| v.vecs()), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 6296e148d..71b8c8369 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -8,13 +8,14 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{Compressed, VecIterator, Version}; - -use super::EagerVec; +use brk_vec::{ + AnyCollectableVec, CloneableAnyIterableVec, Compressed, Computation, ComputedVec, + ComputedVecFrom1, EagerVec, VecIterator, Version, +}; #[derive(Clone)] pub struct Vecs { - pub dateindex_to_date: EagerVec, + pub dateindex_to_date: ComputedVecFrom1, pub dateindex_to_dateindex: EagerVec, pub dateindex_to_first_height: EagerVec, pub dateindex_to_height_count: EagerVec, @@ -70,20 +71,36 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import( + path: &Path, + computation: Computation, + compressed: Compressed, + ) -> color_eyre::Result { fs::create_dir_all(path)?; + let dateindex_to_dateindex = EagerVec::forced_import( + &path.join("dateindex_to_dateindex"), + Version::ZERO, + compressed, + )?; + + let dateindex_to_date = ComputedVec::forced_import_or_init_from_1( + computation, + path, + "dateindex_to_date", + Version::ZERO, + compressed, + dateindex_to_dateindex.boxed_clone(), + |index, dateindex_to_dateindex_iter| { + dateindex_to_dateindex_iter + .next_at(index) + .map(|(dateindex, _)| Date::from(dateindex)) + }, + )?; + Ok(Self { - dateindex_to_date: EagerVec::forced_import( - &path.join("dateindex_to_date"), - Version::ZERO, - compressed, - )?, - dateindex_to_dateindex: EagerVec::forced_import( - &path.join("dateindex_to_dateindex"), - Version::ZERO, - compressed, - )?, + dateindex_to_date, + dateindex_to_dateindex, dateindex_to_first_height: EagerVec::forced_import( &path.join("dateindex_to_first_height"), Version::ZERO, @@ -356,84 +373,84 @@ impl Vecs { self.outputindex_to_outputindex.compute_range( starting_indexes.outputindex, - indexer_vecs.outputindex_to_value.vec(), + &indexer_vecs.outputindex_to_value, |i| (i, i), exit, )?; self.p2pk33index_to_p2pk33index.compute_range( starting_indexes.p2pk33index, - indexer_vecs.p2pk33index_to_p2pk33bytes.vec(), + &indexer_vecs.p2pk33index_to_p2pk33bytes, |i| (i, i), exit, )?; self.p2pk65index_to_p2pk65index.compute_range( starting_indexes.p2pk65index, - indexer_vecs.p2pk65index_to_p2pk65bytes.vec(), + &indexer_vecs.p2pk65index_to_p2pk65bytes, |i| (i, i), exit, )?; self.p2pkhindex_to_p2pkhindex.compute_range( starting_indexes.p2pkhindex, - indexer_vecs.p2pkhindex_to_p2pkhbytes.vec(), + &indexer_vecs.p2pkhindex_to_p2pkhbytes, |i| (i, i), exit, )?; self.p2shindex_to_p2shindex.compute_range( starting_indexes.p2shindex, - indexer_vecs.p2shindex_to_p2shbytes.vec(), + &indexer_vecs.p2shindex_to_p2shbytes, |i| (i, i), exit, )?; self.p2trindex_to_p2trindex.compute_range( starting_indexes.p2trindex, - indexer_vecs.p2trindex_to_p2trbytes.vec(), + &indexer_vecs.p2trindex_to_p2trbytes, |i| (i, i), exit, )?; self.p2wpkhindex_to_p2wpkhindex.compute_range( starting_indexes.p2wpkhindex, - indexer_vecs.p2wpkhindex_to_p2wpkhbytes.vec(), + &indexer_vecs.p2wpkhindex_to_p2wpkhbytes, |i| (i, i), exit, )?; self.p2wshindex_to_p2wshindex.compute_range( starting_indexes.p2wshindex, - indexer_vecs.p2wshindex_to_p2wshbytes.vec(), + &indexer_vecs.p2wshindex_to_p2wshbytes, |i| (i, i), exit, )?; self.emptyoutputindex_to_emptyoutputindex.compute_range( starting_indexes.emptyoutputindex, - indexer_vecs.emptyoutputindex_to_txindex.vec(), + &indexer_vecs.emptyoutputindex_to_txindex, |i| (i, i), exit, )?; self.p2msindex_to_p2msindex.compute_range( starting_indexes.p2msindex, - indexer_vecs.p2msindex_to_txindex.vec(), + &indexer_vecs.p2msindex_to_txindex, |i| (i, i), exit, )?; self.opreturnindex_to_opreturnindex.compute_range( starting_indexes.opreturnindex, - indexer_vecs.opreturnindex_to_txindex.vec(), + &indexer_vecs.opreturnindex_to_txindex, |i| (i, i), exit, )?; self.p2aindex_to_p2aindex.compute_range( starting_indexes.p2aindex, - indexer_vecs.p2aindex_to_p2abytes.vec(), + &indexer_vecs.p2aindex_to_p2abytes, |i| (i, i), exit, )?; @@ -441,7 +458,7 @@ impl Vecs { self.unknownoutputindex_to_unknownoutputindex .compute_range( starting_indexes.unknownoutputindex, - indexer_vecs.unknownoutputindex_to_txindex.vec(), + &indexer_vecs.unknownoutputindex_to_txindex, |i| (i, i), exit, )?; @@ -452,7 +469,7 @@ impl Vecs { self.inputindex_to_inputindex.compute_range( starting_indexes.inputindex, - indexer_vecs.inputindex_to_outputindex.vec(), + &indexer_vecs.inputindex_to_outputindex, |i| (i, i), exit, )?; @@ -463,22 +480,22 @@ impl Vecs { self.txindex_to_txindex.compute_range( starting_indexes.txindex, - indexer_vecs.txindex_to_txid.vec(), + &indexer_vecs.txindex_to_txid, |i| (i, i), exit, )?; self.height_to_txindex_count.compute_count_from_indexes( starting_indexes.height, - indexer_vecs.height_to_first_txindex.vec(), - indexer_vecs.txindex_to_txid.vec(), + &indexer_vecs.height_to_first_txindex, + &indexer_vecs.txindex_to_txid, exit, )?; self.txindex_to_height.compute_inverse_less_to_more( starting_indexes.height, - indexer_vecs.height_to_first_txindex.vec(), - self.height_to_txindex_count.vec(), + &indexer_vecs.height_to_first_txindex, + &self.height_to_txindex_count, exit, )?; @@ -488,14 +505,14 @@ impl Vecs { self.height_to_height.compute_range( starting_indexes.height, - indexer_vecs.height_to_timestamp.vec(), + &indexer_vecs.height_to_timestamp, |h| (h, h), exit, )?; self.height_to_date.compute_transform( starting_indexes.height, - indexer_vecs.height_to_timestamp.vec(), + &indexer_vecs.height_to_timestamp, |(h, t, ..)| (h, Date::from(t)), exit, )?; @@ -503,7 +520,7 @@ impl Vecs { let mut prev_timestamp_fixed = None; self.height_to_timestamp_fixed.compute_transform( starting_indexes.height, - indexer_vecs.height_to_timestamp.vec(), + &indexer_vecs.height_to_timestamp, |(h, timestamp, height_to_timestamp_fixed_iter)| { if prev_timestamp_fixed.is_none() { if let Some(prev_h) = h.decremented() { @@ -524,7 +541,7 @@ impl Vecs { self.height_to_date_fixed.compute_transform( starting_indexes.height, - self.height_to_timestamp_fixed.vec(), + &self.height_to_timestamp_fixed, |(h, t, ..)| (h, Date::from(t)), exit, )?; @@ -543,7 +560,7 @@ impl Vecs { self.height_to_dateindex.compute_transform( starting_indexes.height, - self.height_to_date_fixed.vec(), + &self.height_to_date_fixed, |(h, d, ..)| (h, DateIndex::try_from(d).unwrap()), exit, )?; @@ -561,28 +578,24 @@ impl Vecs { self.dateindex_to_first_height .compute_inverse_more_to_less( starting_indexes.height, - self.height_to_dateindex.vec(), + &self.height_to_dateindex, exit, )?; self.dateindex_to_dateindex.compute_range( starting_dateindex, - self.dateindex_to_first_height.vec(), + &self.dateindex_to_first_height, |di| (di, di), exit, )?; - self.dateindex_to_date.compute_range( - starting_dateindex, - self.dateindex_to_dateindex.vec(), - |di| (di, Date::from(di)), - exit, - )?; + self.dateindex_to_date + .compute_if_necessary(starting_dateindex, exit)?; self.dateindex_to_height_count.compute_count_from_indexes( starting_dateindex, - self.dateindex_to_first_height.vec(), - indexer_vecs.height_to_weight.vec(), + &self.dateindex_to_first_height, + &indexer_vecs.height_to_weight, exit, )?; @@ -598,21 +611,17 @@ impl Vecs { self.dateindex_to_weekindex.compute_range( starting_dateindex, - self.dateindex_to_dateindex.vec(), + &self.dateindex_to_dateindex, |di| (di, WeekIndex::from(di)), exit, )?; self.weekindex_to_first_dateindex - .compute_inverse_more_to_less( - starting_dateindex, - self.dateindex_to_weekindex.vec(), - exit, - )?; + .compute_inverse_more_to_less(starting_dateindex, &self.dateindex_to_weekindex, exit)?; self.weekindex_to_weekindex.compute_range( starting_weekindex, - self.weekindex_to_first_dateindex.vec(), + &self.weekindex_to_first_dateindex, |wi| (wi, wi), exit, )?; @@ -620,8 +629,8 @@ impl Vecs { self.weekindex_to_dateindex_count .compute_count_from_indexes( starting_weekindex, - self.weekindex_to_first_dateindex.vec(), - self.dateindex_to_date.vec(), + &self.weekindex_to_first_dateindex, + &self.dateindex_to_date, exit, )?; @@ -637,7 +646,7 @@ impl Vecs { self.height_to_difficultyepoch.compute_range( starting_indexes.height, - self.height_to_height.vec(), + &self.height_to_height, |h| (h, DifficultyEpoch::from(h)), exit, )?; @@ -645,13 +654,13 @@ impl Vecs { self.difficultyepoch_to_first_height .compute_inverse_more_to_less( starting_indexes.height, - self.height_to_difficultyepoch.vec(), + &self.height_to_difficultyepoch, exit, )?; self.difficultyepoch_to_difficultyepoch.compute_range( starting_difficultyepoch, - self.difficultyepoch_to_first_height.vec(), + &self.difficultyepoch_to_first_height, |i| (i, i), exit, )?; @@ -659,8 +668,8 @@ impl Vecs { self.difficultyepoch_to_height_count .compute_count_from_indexes( starting_difficultyepoch, - self.difficultyepoch_to_first_height.vec(), - self.height_to_date.vec(), + &self.difficultyepoch_to_first_height, + &self.height_to_date, exit, )?; @@ -676,7 +685,7 @@ impl Vecs { self.dateindex_to_monthindex.compute_range( starting_dateindex, - self.dateindex_to_dateindex.vec(), + &self.dateindex_to_dateindex, |di| (di, MonthIndex::from(di)), exit, )?; @@ -684,13 +693,13 @@ impl Vecs { self.monthindex_to_first_dateindex .compute_inverse_more_to_less( starting_dateindex, - self.dateindex_to_monthindex.vec(), + &self.dateindex_to_monthindex, exit, )?; self.monthindex_to_monthindex.compute_range( starting_monthindex, - self.monthindex_to_first_dateindex.vec(), + &self.monthindex_to_first_dateindex, |mi| (mi, mi), exit, )?; @@ -698,8 +707,8 @@ impl Vecs { self.monthindex_to_dateindex_count .compute_count_from_indexes( starting_monthindex, - self.monthindex_to_first_dateindex.vec(), - self.dateindex_to_date.vec(), + &self.monthindex_to_first_dateindex, + &self.dateindex_to_date, exit, )?; @@ -715,7 +724,7 @@ impl Vecs { self.monthindex_to_quarterindex.compute_range( starting_monthindex, - self.monthindex_to_monthindex.vec(), + &self.monthindex_to_monthindex, |mi| (mi, QuarterIndex::from(mi)), exit, )?; @@ -723,7 +732,7 @@ impl Vecs { self.quarterindex_to_first_monthindex .compute_inverse_more_to_less( starting_monthindex, - self.monthindex_to_quarterindex.vec(), + &self.monthindex_to_quarterindex, exit, )?; @@ -731,7 +740,7 @@ impl Vecs { self.quarterindex_to_quarterindex.compute_range( starting_quarterindex, - self.quarterindex_to_first_monthindex.vec(), + &self.quarterindex_to_first_monthindex, |i| (i, i), exit, )?; @@ -739,8 +748,8 @@ impl Vecs { self.quarterindex_to_monthindex_count .compute_count_from_indexes( starting_quarterindex, - self.quarterindex_to_first_monthindex.vec(), - self.monthindex_to_monthindex.vec(), + &self.quarterindex_to_first_monthindex, + &self.monthindex_to_monthindex, exit, )?; @@ -756,7 +765,7 @@ impl Vecs { self.monthindex_to_yearindex.compute_range( starting_monthindex, - self.monthindex_to_monthindex.vec(), + &self.monthindex_to_monthindex, |i| (i, YearIndex::from(i)), exit, )?; @@ -764,13 +773,13 @@ impl Vecs { self.yearindex_to_first_monthindex .compute_inverse_more_to_less( starting_monthindex, - self.monthindex_to_yearindex.vec(), + &self.monthindex_to_yearindex, exit, )?; self.yearindex_to_yearindex.compute_range( starting_yearindex, - self.yearindex_to_first_monthindex.vec(), + &self.yearindex_to_first_monthindex, |i| (i, i), exit, )?; @@ -778,8 +787,8 @@ impl Vecs { self.yearindex_to_monthindex_count .compute_count_from_indexes( starting_yearindex, - self.yearindex_to_first_monthindex.vec(), - self.monthindex_to_monthindex.vec(), + &self.yearindex_to_first_monthindex, + &self.monthindex_to_monthindex, exit, )?; // --- @@ -794,7 +803,7 @@ impl Vecs { self.height_to_halvingepoch.compute_range( starting_indexes.height, - self.height_to_height.vec(), + &self.height_to_height, |h| (h, HalvingEpoch::from(h)), exit, )?; @@ -802,13 +811,13 @@ impl Vecs { self.halvingepoch_to_first_height .compute_inverse_more_to_less( starting_indexes.height, - self.height_to_halvingepoch.vec(), + &self.height_to_halvingepoch, exit, )?; self.halvingepoch_to_halvingepoch.compute_range( starting_halvingepoch, - self.halvingepoch_to_first_height.vec(), + &self.halvingepoch_to_first_height, |i| (i, i), exit, )?; @@ -825,7 +834,7 @@ impl Vecs { self.yearindex_to_decadeindex.compute_range( starting_yearindex, - self.yearindex_to_yearindex.vec(), + &self.yearindex_to_yearindex, |i| (i, DecadeIndex::from(i)), exit, )?; @@ -833,13 +842,13 @@ impl Vecs { self.decadeindex_to_first_yearindex .compute_inverse_more_to_less( starting_yearindex, - self.yearindex_to_decadeindex.vec(), + &self.yearindex_to_decadeindex, exit, )?; self.decadeindex_to_decadeindex.compute_range( starting_decadeindex, - self.decadeindex_to_first_yearindex.vec(), + &self.decadeindex_to_first_yearindex, |i| (i, i), exit, )?; @@ -847,8 +856,8 @@ impl Vecs { self.decadeindex_to_yearindex_count .compute_count_from_indexes( starting_decadeindex, - self.decadeindex_to_first_yearindex.vec(), - self.yearindex_to_yearindex.vec(), + &self.decadeindex_to_first_yearindex, + &self.yearindex_to_yearindex, exit, )?; @@ -865,61 +874,61 @@ impl Vecs { }) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { vec![ - self.dateindex_to_date.any_vec(), - self.dateindex_to_dateindex.any_vec(), - self.dateindex_to_first_height.any_vec(), - self.dateindex_to_height_count.any_vec(), - self.dateindex_to_monthindex.any_vec(), - self.dateindex_to_weekindex.any_vec(), - self.decadeindex_to_decadeindex.any_vec(), - self.decadeindex_to_first_yearindex.any_vec(), - self.decadeindex_to_yearindex_count.any_vec(), - self.difficultyepoch_to_difficultyepoch.any_vec(), - self.difficultyepoch_to_first_height.any_vec(), - self.difficultyepoch_to_height_count.any_vec(), - self.emptyoutputindex_to_emptyoutputindex.any_vec(), - self.halvingepoch_to_first_height.any_vec(), - self.halvingepoch_to_halvingepoch.any_vec(), - self.height_to_date.any_vec(), - self.height_to_date_fixed.any_vec(), - self.height_to_dateindex.any_vec(), - self.height_to_difficultyepoch.any_vec(), - self.height_to_halvingepoch.any_vec(), - self.height_to_height.any_vec(), - self.height_to_timestamp_fixed.any_vec(), - self.height_to_txindex_count.any_vec(), - self.inputindex_to_inputindex.any_vec(), - self.monthindex_to_dateindex_count.any_vec(), - self.monthindex_to_first_dateindex.any_vec(), - self.monthindex_to_monthindex.any_vec(), - self.monthindex_to_quarterindex.any_vec(), - self.monthindex_to_yearindex.any_vec(), - self.opreturnindex_to_opreturnindex.any_vec(), - self.outputindex_to_outputindex.any_vec(), - self.p2aindex_to_p2aindex.any_vec(), - self.p2msindex_to_p2msindex.any_vec(), - self.p2pk33index_to_p2pk33index.any_vec(), - self.p2pk65index_to_p2pk65index.any_vec(), - self.p2pkhindex_to_p2pkhindex.any_vec(), - self.p2shindex_to_p2shindex.any_vec(), - self.p2trindex_to_p2trindex.any_vec(), - self.p2wpkhindex_to_p2wpkhindex.any_vec(), - self.p2wshindex_to_p2wshindex.any_vec(), - self.quarterindex_to_first_monthindex.any_vec(), - self.quarterindex_to_monthindex_count.any_vec(), - self.quarterindex_to_quarterindex.any_vec(), - self.txindex_to_height.any_vec(), - self.txindex_to_txindex.any_vec(), - self.unknownoutputindex_to_unknownoutputindex.any_vec(), - self.weekindex_to_dateindex_count.any_vec(), - self.weekindex_to_first_dateindex.any_vec(), - self.weekindex_to_weekindex.any_vec(), - self.yearindex_to_decadeindex.any_vec(), - self.yearindex_to_first_monthindex.any_vec(), - self.yearindex_to_monthindex_count.any_vec(), - self.yearindex_to_yearindex.any_vec(), + &self.dateindex_to_date, + &self.dateindex_to_dateindex, + &self.dateindex_to_first_height, + &self.dateindex_to_height_count, + &self.dateindex_to_monthindex, + &self.dateindex_to_weekindex, + &self.decadeindex_to_decadeindex, + &self.decadeindex_to_first_yearindex, + &self.decadeindex_to_yearindex_count, + &self.difficultyepoch_to_difficultyepoch, + &self.difficultyepoch_to_first_height, + &self.difficultyepoch_to_height_count, + &self.emptyoutputindex_to_emptyoutputindex, + &self.halvingepoch_to_first_height, + &self.halvingepoch_to_halvingepoch, + &self.height_to_date, + &self.height_to_date_fixed, + &self.height_to_dateindex, + &self.height_to_difficultyepoch, + &self.height_to_halvingepoch, + &self.height_to_height, + &self.height_to_timestamp_fixed, + &self.height_to_txindex_count, + &self.inputindex_to_inputindex, + &self.monthindex_to_dateindex_count, + &self.monthindex_to_first_dateindex, + &self.monthindex_to_monthindex, + &self.monthindex_to_quarterindex, + &self.monthindex_to_yearindex, + &self.opreturnindex_to_opreturnindex, + &self.outputindex_to_outputindex, + &self.p2aindex_to_p2aindex, + &self.p2msindex_to_p2msindex, + &self.p2pk33index_to_p2pk33index, + &self.p2pk65index_to_p2pk65index, + &self.p2pkhindex_to_p2pkhindex, + &self.p2shindex_to_p2shindex, + &self.p2trindex_to_p2trindex, + &self.p2wpkhindex_to_p2wpkhindex, + &self.p2wshindex_to_p2wshindex, + &self.quarterindex_to_first_monthindex, + &self.quarterindex_to_monthindex_count, + &self.quarterindex_to_quarterindex, + &self.txindex_to_height, + &self.txindex_to_txindex, + &self.unknownoutputindex_to_unknownoutputindex, + &self.weekindex_to_dateindex_count, + &self.weekindex_to_first_dateindex, + &self.weekindex_to_weekindex, + &self.yearindex_to_decadeindex, + &self.yearindex_to_first_monthindex, + &self.yearindex_to_monthindex_count, + &self.yearindex_to_yearindex, ] } } diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index d4d98decf..e7ce67d12 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -7,10 +7,10 @@ use brk_core::{ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{Compressed, VecIterator, Version}; +use brk_vec::{AnyIterableVec, Compressed, Computation, EagerVec, VecIterator, Version}; use super::{ - EagerVec, Indexes, + Indexes, grouped::{ ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions, }, @@ -69,7 +69,11 @@ const VERSION: Version = Version::ZERO; const VERSION_IN_SATS: Version = Version::ONE; impl Vecs { - pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import( + path: &Path, + computation: Computation, + compressed: Compressed, + ) -> color_eyre::Result { fs::create_dir_all(path)?; let mut fetched_path = path.to_owned(); @@ -336,7 +340,7 @@ impl Vecs { let mut height_to_timestamp_iter = indexer_vecs.height_to_timestamp.iter(); self.height_to_ohlc_in_cents.compute_transform( starting_indexes.height, - indexer_vecs.height_to_timestamp.vec(), + indexer_vecs.height_to_timestamp.iter_vec(), |(h, t, ..)| { let ohlc = fetcher .get_height( @@ -353,42 +357,42 @@ impl Vecs { self.height_to_open_in_cents.compute_transform( starting_indexes.height, - self.height_to_ohlc_in_cents.vec(), + self.height_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.height_to_high_in_cents.compute_transform( starting_indexes.height, - self.height_to_ohlc_in_cents.vec(), + self.height_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.height_to_low_in_cents.compute_transform( starting_indexes.height, - self.height_to_ohlc_in_cents.vec(), + self.height_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.height_to_close_in_cents.compute_transform( starting_indexes.height, - self.height_to_ohlc_in_cents.vec(), + self.height_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.height_to_ohlc.compute_transform( starting_indexes.height, - self.height_to_ohlc_in_cents.vec(), + self.height_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), exit, )?; self.dateindex_to_ohlc_in_cents.compute_transform( starting_indexes.dateindex, - indexes.dateindex_to_date.vec(), + indexes.dateindex_to_date.iter_vec(), |(di, d, ..)| { let ohlc = fetcher.get_date(d).unwrap(); (di, ohlc) @@ -398,35 +402,35 @@ impl Vecs { self.dateindex_to_open_in_cents.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc_in_cents.vec(), + self.dateindex_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, )?; self.dateindex_to_high_in_cents.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc_in_cents.vec(), + self.dateindex_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, )?; self.dateindex_to_low_in_cents.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc_in_cents.vec(), + self.dateindex_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, )?; self.dateindex_to_close_in_cents.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc_in_cents.vec(), + self.dateindex_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, )?; self.dateindex_to_ohlc.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc_in_cents.vec(), + self.dateindex_to_ohlc_in_cents.iter_vec(), |(di, ohlc, ..)| (di, OHLCDollars::from(ohlc)), exit, )?; @@ -439,7 +443,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc.vec(), + self.dateindex_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, ) @@ -454,7 +458,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc.vec(), + self.dateindex_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, ) @@ -469,7 +473,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc.vec(), + self.dateindex_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, ) @@ -484,7 +488,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.dateindex_to_ohlc.vec(), + self.dateindex_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, ) @@ -499,7 +503,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.height_to_ohlc.vec(), + self.height_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.close), exit, ) @@ -514,7 +518,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.height_to_ohlc.vec(), + self.height_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.high), exit, ) @@ -529,7 +533,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.height_to_ohlc.vec(), + self.height_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.low), exit, ) @@ -544,7 +548,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.height_to_ohlc.vec(), + self.height_to_ohlc.iter_vec(), |(di, ohlc, ..)| (di, ohlc.open), exit, ) @@ -556,7 +560,7 @@ impl Vecs { let mut weekindex_min_iter = self.timeindexes_to_low.weekindex.unwrap_min().iter(); self.weekindex_to_ohlc.compute_transform( starting_indexes.weekindex, - self.timeindexes_to_close.weekindex.unwrap_last().vec(), + self.timeindexes_to_close.weekindex.unwrap_last().iter_vec(), |(i, close, ..)| { ( i, @@ -588,7 +592,7 @@ impl Vecs { self.chainindexes_to_close .difficultyepoch .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -608,7 +612,10 @@ impl Vecs { let mut monthindex_min_iter = self.timeindexes_to_low.monthindex.unwrap_min().iter(); self.monthindex_to_ohlc.compute_transform( starting_indexes.monthindex, - self.timeindexes_to_close.monthindex.unwrap_last().vec(), + self.timeindexes_to_close + .monthindex + .unwrap_last() + .iter_vec(), |(i, close, ..)| { ( i, @@ -629,7 +636,10 @@ impl Vecs { let mut quarterindex_min_iter = self.timeindexes_to_low.quarterindex.unwrap_min().iter(); self.quarterindex_to_ohlc.compute_transform( starting_indexes.quarterindex, - self.timeindexes_to_close.quarterindex.unwrap_last().vec(), + self.timeindexes_to_close + .quarterindex + .unwrap_last() + .iter_vec(), |(i, close, ..)| { ( i, @@ -649,7 +659,7 @@ impl Vecs { let mut yearindex_min_iter = self.timeindexes_to_low.yearindex.unwrap_min().iter(); self.yearindex_to_ohlc.compute_transform( starting_indexes.yearindex, - self.timeindexes_to_close.yearindex.unwrap_last().vec(), + self.timeindexes_to_close.yearindex.unwrap_last().iter_vec(), |(i, close, ..)| { ( i, @@ -672,7 +682,10 @@ impl Vecs { let mut decadeindex_min_iter = self.timeindexes_to_low.decadeindex.unwrap_min().iter(); self.decadeindex_to_ohlc.compute_transform( starting_indexes.decadeindex, - self.timeindexes_to_close.decadeindex.unwrap_last().vec(), + self.timeindexes_to_close + .decadeindex + .unwrap_last() + .iter_vec(), |(i, close, ..)| { ( i, @@ -695,7 +708,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.chainindexes_to_open.height.vec(), + self.chainindexes_to_open.height.iter_vec(), |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), exit, ) @@ -710,7 +723,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.chainindexes_to_low.height.vec(), + self.chainindexes_to_low.height.iter_vec(), |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), exit, ) @@ -725,7 +738,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.chainindexes_to_high.height.vec(), + self.chainindexes_to_high.height.iter_vec(), |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), exit, ) @@ -740,7 +753,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.height, - self.chainindexes_to_close.height.vec(), + self.chainindexes_to_close.height.iter_vec(), |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), exit, ) @@ -755,7 +768,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.timeindexes_to_open.dateindex.vec(), + self.timeindexes_to_open.dateindex.iter_vec(), |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)), exit, ) @@ -770,7 +783,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.timeindexes_to_low.dateindex.vec(), + self.timeindexes_to_low.dateindex.iter_vec(), |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)), exit, ) @@ -785,7 +798,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.timeindexes_to_high.dateindex.vec(), + self.timeindexes_to_high.dateindex.iter_vec(), |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)), exit, ) @@ -800,7 +813,7 @@ impl Vecs { |v, _, _, starting_indexes, exit| { v.compute_transform( starting_indexes.dateindex, - self.timeindexes_to_close.dateindex.vec(), + self.timeindexes_to_close.dateindex.iter_vec(), |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)), exit, ) @@ -812,7 +825,7 @@ impl Vecs { let mut height_min_iter = self.chainindexes_to_low_in_sats.height.iter(); self.height_to_ohlc_in_sats.compute_transform( starting_indexes.height, - self.chainindexes_to_close_in_sats.height.vec(), + self.chainindexes_to_close_in_sats.height.iter_vec(), |(i, close, ..)| { ( i, @@ -832,7 +845,7 @@ impl Vecs { let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter(); self.dateindex_to_ohlc_in_sats.compute_transform( starting_indexes.dateindex, - self.timeindexes_to_close_in_sats.dateindex.vec(), + self.timeindexes_to_close_in_sats.dateindex.iter_vec(), |(i, close, ..)| { ( i, @@ -867,7 +880,7 @@ impl Vecs { self.timeindexes_to_close_in_sats .weekindex .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -902,7 +915,7 @@ impl Vecs { self.chainindexes_to_close_in_sats .difficultyepoch .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -937,7 +950,7 @@ impl Vecs { self.timeindexes_to_close_in_sats .monthindex .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -972,7 +985,7 @@ impl Vecs { self.timeindexes_to_close_in_sats .quarterindex .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -1007,7 +1020,7 @@ impl Vecs { self.timeindexes_to_close_in_sats .yearindex .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -1045,7 +1058,7 @@ impl Vecs { self.timeindexes_to_close_in_sats .decadeindex .unwrap_last() - .vec(), + .iter_vec(), |(i, close, ..)| { ( i, @@ -1063,7 +1076,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { vec![ vec![ self.dateindex_to_close_in_cents.any_vec(), @@ -1095,22 +1108,22 @@ impl Vecs { // self.halvingepoch_to_ohlc_in_sats.any_vec(), self.decadeindex_to_ohlc_in_sats.any_vec(), ], - self.timeindexes_to_close.any_vecs(), - self.timeindexes_to_high.any_vecs(), - self.timeindexes_to_low.any_vecs(), - self.timeindexes_to_open.any_vecs(), - self.chainindexes_to_close.any_vecs(), - self.chainindexes_to_high.any_vecs(), - self.chainindexes_to_low.any_vecs(), - self.chainindexes_to_open.any_vecs(), - self.timeindexes_to_close_in_sats.any_vecs(), - self.timeindexes_to_high_in_sats.any_vecs(), - self.timeindexes_to_low_in_sats.any_vecs(), - self.timeindexes_to_open_in_sats.any_vecs(), - self.chainindexes_to_close_in_sats.any_vecs(), - self.chainindexes_to_high_in_sats.any_vecs(), - self.chainindexes_to_low_in_sats.any_vecs(), - self.chainindexes_to_open_in_sats.any_vecs(), + self.timeindexes_to_close.vecs(), + self.timeindexes_to_high.vecs(), + self.timeindexes_to_low.vecs(), + self.timeindexes_to_open.vecs(), + self.chainindexes_to_close.vecs(), + self.chainindexes_to_high.vecs(), + self.chainindexes_to_low.vecs(), + self.chainindexes_to_open.vecs(), + self.timeindexes_to_close_in_sats.vecs(), + self.timeindexes_to_high_in_sats.vecs(), + self.timeindexes_to_low_in_sats.vecs(), + self.timeindexes_to_open_in_sats.vecs(), + self.chainindexes_to_close_in_sats.vecs(), + self.chainindexes_to_high_in_sats.vecs(), + self.chainindexes_to_low_in_sats.vecs(), + self.chainindexes_to_open_in_sats.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/mining.rs b/crates/brk_computer/src/storage/vecs/mining.rs index 4224cbcde..1ab2a0df7 100644 --- a/crates/brk_computer/src/storage/vecs/mining.rs +++ b/crates/brk_computer/src/storage/vecs/mining.rs @@ -3,7 +3,7 @@ use std::{fs, path::Path}; use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{Compressed, VecIterator, Version}; +use brk_vec::{Compressed, Computation, VecIterator, Version}; use super::{ Indexes, @@ -19,7 +19,11 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import( + path: &Path, + computation: Computation, + compressed: Compressed, + ) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { @@ -113,11 +117,11 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ - self.indexes_to_difficulty.any_vecs(), - self.indexes_to_difficultyepoch.any_vecs(), - self.indexes_to_halvingepoch.any_vecs(), + self.indexes_to_difficulty.vecs(), + self.indexes_to_difficultyepoch.vecs(), + self.indexes_to_halvingepoch.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 2a45193d8..d3ace65f8 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -3,26 +3,24 @@ use std::{fs, path::Path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyVec, Compressed}; +use brk_vec::{AnyCollectableVec, AnyVec, CollectableVec, Compressed, Computation}; -pub mod blocks; -pub mod grouped; +// pub mod blocks; +// pub mod grouped; pub mod indexes; -pub mod marketprice; -pub mod mining; -pub mod transactions; -pub mod vec; +// pub mod marketprice; +// pub mod mining; +// pub mod transactions; pub use indexes::Indexes; -pub use vec::*; #[derive(Clone)] pub struct Vecs { pub indexes: indexes::Vecs, - pub blocks: blocks::Vecs, - pub mining: mining::Vecs, - pub transactions: transactions::Vecs, - pub marketprice: Option, + // pub blocks: blocks::Vecs, + // pub mining: mining::Vecs, + // pub transactions: transactions::Vecs, + // pub marketprice: Option, } impl Vecs { @@ -36,17 +34,18 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - blocks: blocks::Vecs::forced_import(path, compressed)?, - indexes: indexes::Vecs::forced_import(path, compressed)?, - mining: mining::Vecs::forced_import(path, compressed)?, - transactions: transactions::Vecs::forced_import( - path, - indexer, - computation, - compressed, - fetch, - )?, - marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()), + // blocks: blocks::Vecs::forced_import(path, computation, compressed)?, + indexes: indexes::Vecs::forced_import(path, computation, compressed)?, + // mining: mining::Vecs::forced_import(path, computation, compressed)?, + // transactions: transactions::Vecs::forced_import( + // path, + // indexer, + // computation, + // compressed, + // fetch, + // )?, + // marketprice: fetch + // .then(|| marketprice::Vecs::forced_import(path, computation, compressed).unwrap()), }) } @@ -59,40 +58,40 @@ impl Vecs { ) -> color_eyre::Result<()> { let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?; - self.blocks - .compute(indexer, &self.indexes, &starting_indexes, exit)?; + // self.blocks + // .compute(indexer, &self.indexes, &starting_indexes, exit)?; - self.mining - .compute(indexer, &self.indexes, &starting_indexes, exit)?; + // self.mining + // .compute(indexer, &self.indexes, &starting_indexes, exit)?; - if let Some(marketprice) = self.marketprice.as_mut() { - marketprice.compute( - indexer, - &self.indexes, - &starting_indexes, - fetcher.unwrap(), - exit, - )?; - } + // if let Some(marketprice) = self.marketprice.as_mut() { + // marketprice.compute( + // indexer, + // &self.indexes, + // &starting_indexes, + // fetcher.unwrap(), + // exit, + // )?; + // } - self.transactions.compute( - indexer, - &self.indexes, - &starting_indexes, - self.marketprice.as_ref(), - exit, - )?; + // self.transactions.compute( + // indexer, + // &self.indexes, + // &starting_indexes, + // self.marketprice.as_ref(), + // exit, + // )?; Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ - self.indexes.any_vecs(), - self.blocks.any_vecs(), - self.mining.any_vecs(), - self.transactions.any_vecs(), - self.marketprice.as_ref().map_or(vec![], |v| v.any_vecs()), + self.indexes.vecs(), + // self.blocks.vecs(), + // self.mining.vecs(), + // self.transactions.vecs(), + // self.marketprice.as_ref().map_or(vec![], |v| v.vecs()), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index f001fa7da..769177012 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -6,10 +6,13 @@ use brk_core::{ use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::bitcoin; -use brk_vec::{AnyVec, Compressed, StoredIndex, VecIterator, Version}; +use brk_vec::{ + AnyIterableVec, AnyVec, Compressed, Computation, ComputedVec, ComputedVecFrom2, EagerVec, + StoredIndex, VecIterator, Version, +}; use super::{ - Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes, + Indexes, grouped::{ ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions, @@ -49,6 +52,7 @@ pub struct Vecs { pub indexes_to_unknownoutput_count: ComputedVecsFromHeight, pub inputindex_to_value: ComputedVecFrom2, + // Bitcoin and dollar version too pub indexes_to_input_count: ComputedVecsFromTxindex, pub txindex_to_is_coinbase: EagerVec, pub indexes_to_output_count: ComputedVecsFromTxindex, @@ -541,23 +545,8 @@ impl Vecs { exit, )?; - // let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter(); - self.inputindex_to_value.compute_if_necessary( - starting_indexes.inputindex, - // indexer.vecs().inputindex_to_outputindex.vec(), - // |(inputindex, outputindex, ..)| { - // let value = if outputindex == OutputIndex::COINBASE { - // Sats::ZERO - // } else if let Some(value) = outputindex_to_value_iter.get(outputindex) { - // value.into_inner() - // } else { - // dbg!(inputindex, outputindex); - // panic!() - // }; - // (inputindex, value) - // }, - exit, - )?; + self.inputindex_to_value + .compute_if_necessary(starting_indexes.inputindex, exit)?; self.indexes_to_output_value.compute_all( indexer, @@ -898,7 +887,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + pub fn vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ vec![ self.inputindex_to_value.any_vec(), @@ -906,32 +895,32 @@ impl Vecs { self.txindex_to_vsize.any_vec(), self.txindex_to_weight.any_vec(), ], - self.indexes_to_tx_count.any_vecs(), - self.indexes_to_coinbase.any_vecs(), - self.indexes_to_fee.any_vecs(), - self.indexes_to_feerate.any_vecs(), - self.indexes_to_input_value.any_vecs(), - self.indexes_to_output_value.any_vecs(), - self.indexes_to_subsidy.any_vecs(), - self.indexes_to_tx_v1.any_vecs(), - self.indexes_to_tx_v2.any_vecs(), - self.indexes_to_tx_v3.any_vecs(), - self.indexes_to_tx_vsize.any_vecs(), - self.indexes_to_tx_weight.any_vecs(), - self.indexes_to_input_count.any_vecs(), - self.indexes_to_output_count.any_vecs(), - self.indexes_to_p2a_count.any_vecs(), - self.indexes_to_p2ms_count.any_vecs(), - self.indexes_to_p2pk33_count.any_vecs(), - self.indexes_to_p2pk65_count.any_vecs(), - self.indexes_to_p2pkh_count.any_vecs(), - self.indexes_to_p2sh_count.any_vecs(), - self.indexes_to_p2tr_count.any_vecs(), - self.indexes_to_p2wpkh_count.any_vecs(), - self.indexes_to_p2wsh_count.any_vecs(), - self.indexes_to_opreturn_count.any_vecs(), - self.indexes_to_unknownoutput_count.any_vecs(), - self.indexes_to_emptyoutput_count.any_vecs(), + self.indexes_to_tx_count.vecs(), + self.indexes_to_coinbase.vecs(), + self.indexes_to_fee.vecs(), + self.indexes_to_feerate.vecs(), + self.indexes_to_input_value.vecs(), + self.indexes_to_output_value.vecs(), + self.indexes_to_subsidy.vecs(), + self.indexes_to_tx_v1.vecs(), + self.indexes_to_tx_v2.vecs(), + self.indexes_to_tx_v3.vecs(), + self.indexes_to_tx_vsize.vecs(), + self.indexes_to_tx_weight.vecs(), + self.indexes_to_input_count.vecs(), + self.indexes_to_output_count.vecs(), + self.indexes_to_p2a_count.vecs(), + self.indexes_to_p2ms_count.vecs(), + self.indexes_to_p2pk33_count.vecs(), + self.indexes_to_p2pk65_count.vecs(), + self.indexes_to_p2pkh_count.vecs(), + self.indexes_to_p2sh_count.vecs(), + self.indexes_to_p2tr_count.vecs(), + self.indexes_to_p2wpkh_count.vecs(), + self.indexes_to_p2wsh_count.vecs(), + self.indexes_to_opreturn_count.vecs(), + self.indexes_to_unknownoutput_count.vecs(), + self.indexes_to_emptyoutput_count.vecs(), ] .concat() } diff --git a/crates/brk_indexer/src/indexes.rs b/crates/brk_indexer/src/indexes.rs index 9a31151a3..749b7108c 100644 --- a/crates/brk_indexer/src/indexes.rs +++ b/crates/brk_indexer/src/indexes.rs @@ -5,10 +5,10 @@ use brk_core::{ P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; -use brk_vec::{StoredIndex, StoredType, VecIterator}; +use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType}; use color_eyre::eyre::ContextCompat; -use crate::{IndexedVec, Stores, Vecs}; +use crate::{Stores, Vecs}; #[derive(Debug, Default, Clone)] pub struct Indexes { diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 166dbc556..aaabd48e3 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -18,7 +18,7 @@ pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; use brk_exit::Exit; -use brk_vec::{Compressed, DynamicVec, VecIterator}; +use brk_vec::{AnyVec, Compressed, GenericStoredVec, VecIterator}; use color_eyre::eyre::{ContextCompat, eyre}; use fjall::TransactionalKeyspace; use log::{error, info}; @@ -648,7 +648,7 @@ impl Indexer { let mut txindex_to_tx_and_txid: BTreeMap = BTreeMap::default(); let mut txindex_to_txid_iter = vecs - .txindex_to_txid.iter(); + .txindex_to_txid.into_iter(); txid_prefix_to_txid_and_block_txindex_and_prev_txindex .into_iter() diff --git a/crates/brk_indexer/src/stores/mod.rs b/crates/brk_indexer/src/stores/mod.rs index 5156ea4f4..06848aad9 100644 --- a/crates/brk_indexer/src/stores/mod.rs +++ b/crates/brk_indexer/src/stores/mod.rs @@ -4,7 +4,7 @@ use brk_core::{ AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex, TxidPrefix, }; -use brk_vec::{Value, VecIterator, Version}; +use brk_vec::{AnyIterableVec, Value, Version}; use fjall::{PersistMode, TransactionalKeyspace}; use crate::Indexes; diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 2c970629a..6c46d586c 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -7,15 +7,11 @@ use brk_core::{ P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight, }; -use brk_vec::{AnyVec, Compressed, Result, Version}; +use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec, Result, Version}; use rayon::prelude::*; use crate::Indexes; -mod base; - -pub use base::*; - #[derive(Clone)] pub struct Vecs { pub emptyoutputindex_to_txindex: IndexedVec, @@ -442,69 +438,69 @@ impl Vecs { } pub fn flush(&mut self, height: Height) -> Result<()> { - self.mut_any_vecs() + self.mut_vecs() .into_par_iter() .try_for_each(|vec| vec.flush(height)) } pub fn starting_height(&mut self) -> Height { - self.mut_any_vecs() + self.mut_vecs() .into_iter() .map(|vec| vec.height().map(Height::incremented).unwrap_or_default()) .min() .unwrap() } - pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { vec![ - self.emptyoutputindex_to_txindex.any_vec(), - self.height_to_blockhash.any_vec(), - self.height_to_difficulty.any_vec(), - self.height_to_first_emptyoutputindex.any_vec(), - self.height_to_first_inputindex.any_vec(), - self.height_to_first_opreturnindex.any_vec(), - self.height_to_first_outputindex.any_vec(), - self.height_to_first_p2aindex.any_vec(), - self.height_to_first_p2msindex.any_vec(), - self.height_to_first_p2pk33index.any_vec(), - self.height_to_first_p2pk65index.any_vec(), - self.height_to_first_p2pkhindex.any_vec(), - self.height_to_first_p2shindex.any_vec(), - self.height_to_first_p2trindex.any_vec(), - self.height_to_first_p2wpkhindex.any_vec(), - self.height_to_first_p2wshindex.any_vec(), - self.height_to_first_txindex.any_vec(), - self.height_to_first_unknownoutputindex.any_vec(), - self.height_to_timestamp.any_vec(), - self.height_to_total_size.any_vec(), - self.height_to_weight.any_vec(), - self.inputindex_to_outputindex.any_vec(), - self.opreturnindex_to_txindex.any_vec(), - self.outputindex_to_outputtype.any_vec(), - self.outputindex_to_outputtypeindex.any_vec(), - self.outputindex_to_value.any_vec(), - self.p2aindex_to_p2abytes.any_vec(), - self.p2msindex_to_txindex.any_vec(), - self.p2pk33index_to_p2pk33bytes.any_vec(), - self.p2pk65index_to_p2pk65bytes.any_vec(), - self.p2pkhindex_to_p2pkhbytes.any_vec(), - self.p2shindex_to_p2shbytes.any_vec(), - self.p2trindex_to_p2trbytes.any_vec(), - self.p2wpkhindex_to_p2wpkhbytes.any_vec(), - self.p2wshindex_to_p2wshbytes.any_vec(), - self.txindex_to_base_size.any_vec(), - self.txindex_to_first_inputindex.any_vec(), - self.txindex_to_first_outputindex.any_vec(), - self.txindex_to_is_explicitly_rbf.any_vec(), - self.txindex_to_rawlocktime.any_vec(), - self.txindex_to_total_size.any_vec(), - self.txindex_to_txid.any_vec(), - self.txindex_to_txversion.any_vec(), - self.unknownoutputindex_to_txindex.any_vec(), + &self.emptyoutputindex_to_txindex, + &self.height_to_blockhash, + &self.height_to_difficulty, + &self.height_to_first_emptyoutputindex, + &self.height_to_first_inputindex, + &self.height_to_first_opreturnindex, + &self.height_to_first_outputindex, + &self.height_to_first_p2aindex, + &self.height_to_first_p2msindex, + &self.height_to_first_p2pk33index, + &self.height_to_first_p2pk65index, + &self.height_to_first_p2pkhindex, + &self.height_to_first_p2shindex, + &self.height_to_first_p2trindex, + &self.height_to_first_p2wpkhindex, + &self.height_to_first_p2wshindex, + &self.height_to_first_txindex, + &self.height_to_first_unknownoutputindex, + &self.height_to_timestamp, + &self.height_to_total_size, + &self.height_to_weight, + &self.inputindex_to_outputindex, + &self.opreturnindex_to_txindex, + &self.outputindex_to_outputtype, + &self.outputindex_to_outputtypeindex, + &self.outputindex_to_value, + &self.p2aindex_to_p2abytes, + &self.p2msindex_to_txindex, + &self.p2pk33index_to_p2pk33bytes, + &self.p2pk65index_to_p2pk65bytes, + &self.p2pkhindex_to_p2pkhbytes, + &self.p2shindex_to_p2shbytes, + &self.p2trindex_to_p2trbytes, + &self.p2wpkhindex_to_p2wpkhbytes, + &self.p2wshindex_to_p2wshbytes, + &self.txindex_to_base_size, + &self.txindex_to_first_inputindex, + &self.txindex_to_first_outputindex, + &self.txindex_to_is_explicitly_rbf, + &self.txindex_to_rawlocktime, + &self.txindex_to_total_size, + &self.txindex_to_txid, + &self.txindex_to_txversion, + &self.unknownoutputindex_to_txindex, ] } - fn mut_any_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { + fn mut_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> { vec![ &mut self.emptyoutputindex_to_txindex, &mut self.height_to_blockhash, diff --git a/crates/brk_query/examples/main.rs b/crates/brk_query/examples/main.rs index 0c7af83a7..4504fa243 100644 --- a/crates/brk_query/examples/main.rs +++ b/crates/brk_query/examples/main.rs @@ -1,8 +1,9 @@ use std::path::Path; -use brk_computer::{Computation, Computer}; +use brk_computer::Computer; use brk_indexer::Indexer; use brk_query::{Index, Query}; +use brk_vec::Computation; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index 01d85a9e2..b48aa6499 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -5,7 +5,7 @@ use brk_computer::Computer; use brk_indexer::Indexer; -use brk_vec::AnyVec; +use brk_vec::AnyCollectableVec; use tabled::settings::Style; mod format; @@ -34,13 +34,12 @@ impl<'a> Query<'a> { indexer .vecs() - .any_vecs() + .vecs() .into_iter() .for_each(|vec| vec_trees.insert(vec)); computer .vecs() - .any_vecs() .into_iter() .for_each(|vec| vec_trees.insert(vec)); @@ -51,7 +50,7 @@ impl<'a> Query<'a> { } } - pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyVec)> { + pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyCollectableVec)> { let tuples = ids .iter() .flat_map(|s| { @@ -86,7 +85,7 @@ impl<'a> Query<'a> { pub fn format( &self, - vecs: Vec<(String, &&dyn AnyVec)>, + vecs: Vec<(String, &&dyn AnyCollectableVec)>, from: Option, to: Option, format: Option, diff --git a/crates/brk_query/src/vec_trees.rs b/crates/brk_query/src/vec_trees.rs index 4178ffc77..c4d90099a 100644 --- a/crates/brk_query/src/vec_trees.rs +++ b/crates/brk_query/src/vec_trees.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_vec::AnyVec; +use brk_vec::AnyCollectableVec; use derive_deref::{Deref, DerefMut}; use super::index::Index; @@ -13,7 +13,7 @@ pub struct VecTrees<'a> { impl<'a> VecTrees<'a> { // Not the most performant or type safe but only built once so that's okay - pub fn insert(&mut self, vec: &'a dyn AnyVec) { + pub fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { let name = vec.name(); let split = name.split("_to_").collect::>(); if split.len() != 2 { @@ -88,7 +88,7 @@ impl<'a> VecTrees<'a> { } #[derive(Default, Deref, DerefMut)] -pub struct IndexToVec<'a>(BTreeMap); +pub struct IndexToVec<'a>(BTreeMap); #[derive(Default, Deref, DerefMut)] -pub struct IdToVec<'a>(BTreeMap); +pub struct IdToVec<'a>(BTreeMap); diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 42294d7fd..b81784558 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -10,12 +10,13 @@ repository.workspace = true axum = { workspace = true } brk_computer = { workspace = true } brk_exit = { workspace = true } -brk_fetcher = { workspace = true } brk_core = { workspace = true } +brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_query = { workspace = true } +brk_vec = { workspace = true } clap = { workspace = true } clap_derive = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index b04bbadd1..1c3bb9a54 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -1,6 +1,6 @@ use std::{path::Path, thread::sleep, time::Duration}; -use brk_computer::{Computation, Computer}; +use brk_computer::Computer; use brk_core::default_bitcoin_path; use brk_exit::Exit; use brk_fetcher::Fetcher; @@ -10,6 +10,7 @@ use brk_parser::{ rpc::{self, RpcApi}, }; use brk_server::{Server, Website}; +use brk_vec::Computation; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 46fd7587e..a4feaa850 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -10,6 +10,11 @@ repository.workspace = true [dependencies] arc-swap = "1.7.1" +brk_core = { workspace = true } +brk_exit = { workspace = true } +clap = { workspace = true } +clap_derive = { workspace = true } +log = { workspace = true } memmap2 = "0.9.5" rayon = { workspace = true } serde = { workspace = true } diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index d8ca84fae..7a41a565d 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -1,6 +1,8 @@ use std::{fs, path::Path}; -use brk_vec::{Compressed, DynamicVec, GenericVec, StoredVec, VecIterator, Version}; +use brk_vec::{ + AnyVec, CollectableVec, Compressed, GenericStoredVec, StoredVec, VecIterator, Version, +}; fn main() -> Result<(), Box> { let _ = fs::remove_dir_all("./vec"); @@ -16,7 +18,7 @@ fn main() -> Result<(), Box> { vec.push(v); }); - let mut iter = vec.iter(); + let mut iter = vec.into_iter(); dbg!(iter.get(0)); dbg!(iter.get(20)); dbg!(iter.get(21)); @@ -27,7 +29,7 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = StoredVec::forced_import(Path::new("./vec"), version, compressed)?; - let mut iter = vec.iter(); + let mut iter = vec.into_iter(); dbg!(iter.get(0)); dbg!(iter.get(0)); @@ -40,7 +42,7 @@ fn main() -> Result<(), Box> { vec.push(21); vec.push(22); - let mut iter = vec.iter(); + let mut iter = vec.into_iter(); dbg!(iter.get(20)); dbg!(iter.get(21)); @@ -53,7 +55,7 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = StoredVec::forced_import(Path::new("./vec"), version, compressed)?; - let mut iter = vec.iter(); + let mut iter = vec.into_iter(); dbg!(iter.get(0)); dbg!(iter.get(20)); @@ -62,7 +64,7 @@ fn main() -> Result<(), Box> { vec.truncate_if_needed(14)?; - let mut iter = vec.iter(); + let mut iter = vec.into_iter(); iter.get(0); iter.get(5); @@ -71,7 +73,7 @@ fn main() -> Result<(), Box> { dbg!(vec.collect_signed_range(Some(-5), None)?); vec.push(vec.len() as u32); - dbg!(VecIterator::last(vec.iter())); + dbg!(VecIterator::last(vec.into_iter())); dbg!(vec.into_iter().collect::>()); } diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 30066cc42..6c84b3cdb 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -8,225 +8,8 @@ mod structs; mod traits; mod variants; -use std::{path::Path, time::Duration}; - -use arc_swap::ArcSwap; pub use enums::*; pub use memmap2::Mmap; pub use structs::*; pub use traits::*; -use variants::*; - -#[derive(Debug, Clone)] -pub enum StoredVec { - Raw(RawVec), - Compressed(CompressedVec), -} - -impl StoredVec -where - I: StoredIndex, - T: StoredType, -{ - pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { - if *compressed { - Ok(Self::Compressed(CompressedVec::forced_import( - path, version, - )?)) - } else { - Ok(Self::Raw(RawVec::forced_import(path, version)?)) - } - } - - pub fn iter(&self) -> StoredVecIterator<'_, I, T> { - self.into_iter() - } - - pub fn iter_at(&self, i: I) -> StoredVecIterator<'_, I, T> { - let mut iter = self.into_iter(); - iter.set(i); - iter - } -} - -impl DynamicVec for StoredVec -where - I: StoredIndex, - T: StoredType, -{ - type I = I; - type T = T; - - #[inline] - fn read_(&self, index: usize, guard: &Mmap) -> Result> { - match self { - StoredVec::Raw(v) => v.read_(index, guard), - StoredVec::Compressed(v) => v.read_(index, guard), - } - } - - #[inline] - fn mmap(&self) -> &ArcSwap { - match self { - StoredVec::Raw(v) => v.mmap(), - StoredVec::Compressed(v) => v.mmap(), - } - } - - #[inline] - fn stored_len(&self) -> usize { - match self { - StoredVec::Raw(v) => v.stored_len(), - StoredVec::Compressed(v) => v.stored_len(), - } - } - - #[inline] - fn pushed(&self) -> &[T] { - match self { - StoredVec::Raw(v) => v.pushed(), - StoredVec::Compressed(v) => v.pushed(), - } - } - #[inline] - fn mut_pushed(&mut self) -> &mut Vec { - match self { - StoredVec::Raw(v) => v.mut_pushed(), - StoredVec::Compressed(v) => v.mut_pushed(), - } - } - - #[inline] - fn path(&self) -> &Path { - match self { - StoredVec::Raw(v) => v.path(), - StoredVec::Compressed(v) => v.path(), - } - } -} - -impl GenericVec for StoredVec -where - I: StoredIndex, - T: StoredType, -{ - fn collect_range(&self, from: Option, to: Option) -> Result> { - match self { - StoredVec::Raw(v) => v.collect_range(from, to), - StoredVec::Compressed(v) => v.collect_range(from, to), - } - } - - fn flush(&mut self) -> Result<()> { - match self { - StoredVec::Raw(v) => v.flush(), - StoredVec::Compressed(v) => v.flush(), - } - } - - fn truncate_if_needed(&mut self, index: I) -> Result<()> { - match self { - StoredVec::Raw(v) => v.truncate_if_needed(index), - StoredVec::Compressed(v) => v.truncate_if_needed(index), - } - } - - #[inline] - fn version(&self) -> Version { - match self { - StoredVec::Raw(v) => v.version(), - StoredVec::Compressed(v) => v.version(), - } - } -} - -impl AnyVec for StoredVec -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - fn len(&self) -> usize { - DynamicVec::len(self) - } - - #[inline] - fn index_type_to_string(&self) -> &str { - GenericVec::index_type_to_string(self) - } - - #[inline] - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - GenericVec::collect_range_serde_json(self, from, to) - } - - #[inline] - fn modified_time(&self) -> Result { - GenericVec::modified_time(self) - } - - fn name(&self) -> String { - GenericVec::name(self) - } -} - -#[derive(Debug)] -pub enum StoredVecIterator<'a, I, T> { - Raw(RawVecIterator<'a, I, T>), - Compressed(CompressedVecIterator<'a, I, T>), -} - -impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T> -where - I: StoredIndex, - T: StoredType, -{ - type Item = (I, Value<'a, T>); - fn next(&mut self) -> Option { - match self { - Self::Compressed(i) => i.next(), - Self::Raw(i) => i.next(), - } - } -} - -impl BaseVecIterator for StoredVecIterator<'_, I, T> -where - I: StoredIndex, - T: StoredType, -{ - #[inline] - fn mut_index(&mut self) -> &mut usize { - match self { - Self::Compressed(iter) => iter.mut_index(), - Self::Raw(iter) => iter.mut_index(), - } - } - - fn len(&self) -> usize { - match self { - Self::Compressed(i) => i.len(), - Self::Raw(i) => i.len(), - } - } -} - -impl<'a, I, T> IntoIterator for &'a StoredVec -where - I: StoredIndex, - T: StoredType, -{ - type Item = (I, Value<'a, T>); - type IntoIter = StoredVecIterator<'a, I, T>; - - fn into_iter(self) -> Self::IntoIter { - match self { - StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()), - StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()), - } - } -} +pub use variants::*; diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs index e28a2cae1..79662708d 100644 --- a/crates/brk_vec/src/traits/any.rs +++ b/crates/brk_vec/src/traits/any.rs @@ -1,24 +1,73 @@ use std::time::Duration; -use crate::Result; +use crate::{Result, Version}; + +use super::{BoxedVecIterator, StoredIndex, StoredType}; pub trait AnyVec: Send + Sync { + fn version(&self) -> Version; fn name(&self) -> String; - fn index_type_to_string(&self) -> &str; fn len(&self) -> usize; fn is_empty(&self) -> bool { self.len() == 0 } - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result>; fn modified_time(&self) -> Result; - fn any_vec(&self) -> &dyn AnyVec + fn index_type_to_string(&self) -> &str; +} + +pub trait AnyIterableVec: AnyVec { + #[allow(clippy::wrong_self_convention)] + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> where - Self: Sized, + I: StoredIndex, + T: StoredType + 'a; + + fn iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredType + 'a, { - self + self.boxed_iter() + } + + fn iter_at<'a>(&'a self, i: I) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredType + 'a, + { + let mut iter = self.boxed_iter(); + iter.set(i); + iter + } + + fn iter_at_<'a>(&'a self, i: usize) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredType + 'a, + { + let mut iter = self.boxed_iter(); + iter.set_(i); + iter } } + +pub trait CloneableAnyIterableVec: AnyIterableVec { + fn boxed_clone(&self) -> Box>; +} + +impl CloneableAnyIterableVec for U +where + U: 'static + AnyIterableVec + Clone, +{ + fn boxed_clone(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.boxed_clone() + } +} + +pub type BoxedAnyIterableVec = Box>; diff --git a/crates/brk_vec/src/traits/collectable.rs b/crates/brk_vec/src/traits/collectable.rs new file mode 100644 index 000000000..0ee63de5f --- /dev/null +++ b/crates/brk_vec/src/traits/collectable.rs @@ -0,0 +1,71 @@ +use crate::{Error, Result}; + +use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType}; + +pub trait CollectableVec: AnyVec + AnyIterableVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range(&self, from: Option, to: Option) -> Result> { + let len = self.len(); + let from = from.unwrap_or_default(); + let to = to.map_or(len, |i| i.min(len)); + + if from >= len || from >= to { + return Ok(vec![]); + } + + Ok(self + .iter_at_(from) + .take(to - from) + .map(|(_, v)| v.into_inner()) + .collect::>()) + } + + #[inline] + fn i64_to_usize(i: i64, len: usize) -> usize { + if i >= 0 { + i as usize + } else { + let v = len as i64 + i; + if v < 0 { 0 } else { v as usize } + } + } + + #[doc(hidden)] + fn collect_signed_range(&self, from: Option, to: Option) -> Result> { + let len = self.len(); + let from = from.map(|i| Self::i64_to_usize(i, len)); + let to = to.map(|i| Self::i64_to_usize(i, len)); + self.collect_range(from, to) + } + + #[inline] + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + self.collect_signed_range(from, to)? + .into_iter() + .map(|v| serde_json::to_value(v).map_err(Error::from)) + .collect::>>() + } +} + +impl CollectableVec for V +where + V: AnyVec + AnyIterableVec, + I: StoredIndex, + T: StoredType, +{ +} + +pub trait AnyCollectableVec: AnyVec { + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result>; +} diff --git a/crates/brk_vec/src/traits/dynamic.rs b/crates/brk_vec/src/traits/dynamic.rs deleted file mode 100644 index de95b3eff..000000000 --- a/crates/brk_vec/src/traits/dynamic.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::path::Path; - -use arc_swap::ArcSwap; -use memmap2::Mmap; - -use crate::{Result, Value}; - -use super::{StoredIndex, StoredType}; - -pub trait DynamicVec: Send + Sync { - type I: StoredIndex; - type T: StoredType; - const SIZE_OF_T: usize = size_of::(); - - #[inline] - fn read(&self, index: Self::I, mmap: &Mmap) -> Result> { - self.read_(index.to_usize()?, mmap) - } - fn read_(&self, index: usize, mmap: &Mmap) -> Result>; - - #[inline] - fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result>> { - self.get_or_read_(index.to_usize()?, mmap) - } - #[inline] - fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result>> { - let stored_len = mmap.len() / Self::SIZE_OF_T; - - if index >= stored_len { - let pushed = self.pushed(); - let j = index - stored_len; - if j >= pushed.len() { - return Ok(None); - } - Ok(pushed.get(j).map(Value::Ref)) - } else { - Ok(self.read_(index, mmap)?.map(Value::Owned)) - } - } - - #[inline] - fn len(&self) -> usize { - self.stored_len() + self.pushed_len() - } - #[inline] - fn is_empty(&self) -> bool { - self.len() == 0 - } - - fn mmap(&self) -> &ArcSwap; - - fn stored_len(&self) -> usize; - - fn pushed(&self) -> &[Self::T]; - #[inline] - fn pushed_len(&self) -> usize { - self.pushed().len() - } - fn mut_pushed(&mut self) -> &mut Vec; - #[inline] - fn push(&mut self, value: Self::T) { - self.mut_pushed().push(value) - } - - fn path(&self) -> &Path; -} diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index dd0615648..cc976944c 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -6,17 +6,70 @@ use std::{ time::{self, Duration}, }; +use arc_swap::ArcSwap; use memmap2::Mmap; -use crate::{Error, Result, Version}; +use crate::{Result, Value}; -use super::{DynamicVec, StoredIndex, StoredType}; +use super::{StoredIndex, StoredType}; -pub trait GenericVec: DynamicVec +pub trait GenericStoredVec: Send + Sync where I: StoredIndex, T: StoredType, { + const SIZE_OF_T: usize = size_of::(); + + #[inline] + fn read(&self, index: I, mmap: &Mmap) -> Result> { + self.read_(index.to_usize()?, mmap) + } + fn read_(&self, index: usize, mmap: &Mmap) -> Result>; + + #[inline] + fn get_or_read(&self, index: I, mmap: &Mmap) -> Result>> { + self.get_or_read_(index.to_usize()?, mmap) + } + #[inline] + fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result>> { + let stored_len = mmap.len() / Self::SIZE_OF_T; + + if index >= stored_len { + let pushed = self.pushed(); + let j = index - stored_len; + if j >= pushed.len() { + return Ok(None); + } + Ok(pushed.get(j).map(Value::Ref)) + } else { + Ok(self.read_(index, mmap)?.map(Value::Owned)) + } + } + + #[inline] + fn len_(&self) -> usize { + self.stored_len() + self.pushed_len() + } + + fn mmap(&self) -> &ArcSwap; + + fn stored_len(&self) -> usize; + + fn pushed(&self) -> &[T]; + #[inline] + fn pushed_len(&self) -> usize { + self.pushed().len() + } + fn mut_pushed(&mut self) -> &mut Vec; + #[inline] + fn push(&mut self, value: T) { + self.mut_pushed().push(value) + } + + fn path(&self) -> &Path; + + // --- + fn open_file(&self) -> io::Result { Self::open_file_(&self.path_vec()) } @@ -74,54 +127,17 @@ where } #[inline] - fn has(&self, index: Self::I) -> Result { + fn has(&self, index: I) -> Result { Ok(self.has_(index.to_usize()?)) } #[inline] fn has_(&self, index: usize) -> bool { - index < self.len() - } - - #[inline] - fn index_type_to_string(&self) -> &str { - Self::I::to_string() + index < self.len_() } fn flush(&mut self) -> Result<()>; - fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>; - - fn collect_range(&self, from: Option, to: Option) -> Result>; - - #[inline] - fn i64_to_usize(i: i64, len: usize) -> usize { - if i >= 0 { - i as usize - } else { - let v = len as i64 + i; - if v < 0 { 0 } else { v as usize } - } - } - - #[doc(hidden)] - fn collect_signed_range(&self, from: Option, to: Option) -> Result> { - let len = self.len(); - let from = from.map(|i| Self::i64_to_usize(i, len)); - let to = to.map(|i| Self::i64_to_usize(i, len)); - self.collect_range(from, to) - } - - #[inline] - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - self.collect_signed_range(from, to)? - .into_iter() - .map(|v| serde_json::to_value(v).map_err(Error::from)) - .collect::>>() - } + fn truncate_if_needed(&mut self, index: I) -> Result<()>; #[inline] fn path_vec(&self) -> PathBuf { @@ -143,7 +159,7 @@ where } #[inline] - fn name(&self) -> String { + fn name_(&self) -> String { self.path() .file_name() .unwrap() @@ -152,13 +168,11 @@ where .to_owned() } - fn modified_time(&self) -> Result { + fn modified_time_(&self) -> Result { Ok(self .path_vec() .metadata()? .modified()? .duration_since(time::UNIX_EPOCH)?) } - - fn version(&self) -> Version; } diff --git a/crates/brk_vec/src/traits/stored_index.rs b/crates/brk_vec/src/traits/index.rs similarity index 99% rename from crates/brk_vec/src/traits/stored_index.rs rename to crates/brk_vec/src/traits/index.rs index 752109574..c0fb5d470 100644 --- a/crates/brk_vec/src/traits/stored_index.rs +++ b/crates/brk_vec/src/traits/index.rs @@ -23,6 +23,7 @@ where fn to_string<'a>() -> &'a str; fn decremented(self) -> Option; } + impl StoredIndex for I where I: Debug diff --git a/crates/brk_vec/src/traits/iterator.rs b/crates/brk_vec/src/traits/iterator.rs index 04ed32d4e..354ad2c2a 100644 --- a/crates/brk_vec/src/traits/iterator.rs +++ b/crates/brk_vec/src/traits/iterator.rs @@ -32,9 +32,9 @@ pub trait BaseVecIterator: Iterator { } } -pub trait VecIterator<'a>: BaseVecIterator)> + 'a { +pub trait VecIterator<'a>: BaseVecIterator)> { type I: StoredIndex; - type T: StoredType; + type T: StoredType + 'a; #[inline] fn set(&mut self, i: Self::I) { @@ -77,10 +77,13 @@ pub trait VecIterator<'a>: BaseVecIterator)> impl<'a, I, T, Iter> VecIterator<'a> for Iter where - Iter: BaseVecIterator)> + 'a, + Iter: BaseVecIterator)>, I: StoredIndex, T: StoredType + 'a, { type I = I; type T = T; } + +pub type BoxedVecIterator<'a, I, T> = + Box)> + 'a>; diff --git a/crates/brk_vec/src/traits/mod.rs b/crates/brk_vec/src/traits/mod.rs index b7c1a0ad7..d07c2a598 100644 --- a/crates/brk_vec/src/traits/mod.rs +++ b/crates/brk_vec/src/traits/mod.rs @@ -1,13 +1,13 @@ mod any; -mod dynamic; +mod collectable; mod generic; +mod index; mod iterator; -mod stored_index; -mod stored_type; +mod r#type; pub use any::*; -pub use dynamic::*; +pub use collectable::*; pub use generic::*; +pub use index::*; pub use iterator::*; -pub use stored_index::*; -pub use stored_type::*; +pub use r#type::*; diff --git a/crates/brk_vec/src/traits/stored_type.rs b/crates/brk_vec/src/traits/type.rs similarity index 99% rename from crates/brk_vec/src/traits/stored_type.rs rename to crates/brk_vec/src/traits/type.rs index bb9dd1495..3acb2f7bf 100644 --- a/crates/brk_vec/src/traits/stored_type.rs +++ b/crates/brk_vec/src/traits/type.rs @@ -17,6 +17,7 @@ where + Serialize, { } + impl StoredType for T where T: Sized + Debug diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 3b7f2016b..82250a229 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -3,6 +3,7 @@ use std::{ mem, path::Path, sync::Arc, + time::Duration, }; use arc_swap::{ArcSwap, Guard}; @@ -11,8 +12,9 @@ use rayon::prelude::*; use zstd::DEFAULT_COMPRESSION_LEVEL; use crate::{ - BaseVecIterator, CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error, - GenericVec, RawVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version, + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, + CompressedPageMetadata, CompressedPagesMetadata, Error, GenericStoredVec, RawVec, Result, + StoredIndex, StoredType, UnsafeSlice, Value, Version, }; const ONE_KIB: usize = 1024; @@ -152,14 +154,11 @@ where } } -impl DynamicVec for CompressedVec +impl GenericStoredVec for CompressedVec where I: StoredIndex, T: StoredType, { - type I = I; - type T = T; - #[inline] fn read_(&self, index: usize, mmap: &Mmap) -> Result> { let page_index = Self::index_to_page_index(index); @@ -194,28 +193,6 @@ where fn path(&self) -> &Path { self.inner.path() } -} - -impl GenericVec for CompressedVec -where - I: StoredIndex, - T: StoredType, -{ - fn collect_range(&self, from: Option, to: Option) -> Result> { - let stored_len = self.stored_len(); - let from = from.unwrap_or_default(); - let to = to.map_or(stored_len, |i| i.min(stored_len)); - - if from >= stored_len || from >= to { - return Ok(vec![]); - } - - Ok(self - .iter_at_(from) - .take(to - from) - .map(|(_, v)| v.into_inner()) - .collect::>()) - } fn flush(&mut self) -> Result<()> { let pushed_len = self.pushed_len(); @@ -352,11 +329,37 @@ where Ok(()) } +} +impl AnyVec for CompressedVec +where + I: StoredIndex, + T: StoredType, +{ #[inline] fn version(&self) -> Version { self.inner.version() } + + #[inline] + fn name(&self) -> String { + self.name_() + } + + #[inline] + fn len(&self) -> usize { + self.len_() + } + + #[inline] + fn modified_time(&self) -> Result { + self.modified_time_() + } + + #[inline] + fn index_type_to_string(&self) -> &str { + I::to_string() + } } impl Clone for CompressedVec { @@ -474,3 +477,30 @@ where } } } + +impl AnyIterableVec for CompressedVec +where + I: StoredIndex, + T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for CompressedVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/mod.rs b/crates/brk_vec/src/variants/computed.rs similarity index 57% rename from crates/brk_computer/src/storage/vecs/vec/mod.rs rename to crates/brk_vec/src/variants/computed.rs index f23ba13f9..c9147e7e1 100644 --- a/crates/brk_computer/src/storage/vecs/vec/mod.rs +++ b/crates/brk_vec/src/variants/computed.rs @@ -4,21 +4,17 @@ use brk_exit::Exit; use clap_derive::ValueEnum; use serde::{Deserialize, Serialize}; -mod _type; -mod eager; -mod lazy1; -mod lazy2; -mod lazy3; - -pub use _type::*; use brk_core::StoredPhantom; -use brk_vec::{ - AnyVec, Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version, + +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, + BoxedVecIterator, CollectableVec, Compressed, Result, StoredIndex, StoredType, Value, Version, +}; + +use super::{ + ComputeFrom1, ComputeFrom2, ComputeFrom3, EagerVec, LazyVecFrom1, LazyVecFrom1Iterator, + LazyVecFrom2, LazyVecFrom2Iterator, LazyVecFrom3, LazyVecFrom3Iterator, StoredVecIterator, }; -pub use eager::*; -pub use lazy1::*; -pub use lazy2::*; -pub use lazy3::*; #[derive( Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum, @@ -41,16 +37,16 @@ impl Computation { #[derive(Clone)] pub enum Dependencies { - From1(StoredVec, ComputeFrom1), + From1(BoxedAnyIterableVec, ComputeFrom1), From2( - (StoredVec, StoredVec), + (BoxedAnyIterableVec, BoxedAnyIterableVec), ComputeFrom2, ), From3( ( - StoredVec, - StoredVec, - StoredVec, + BoxedAnyIterableVec, + BoxedAnyIterableVec, + BoxedAnyIterableVec, ), ComputeFrom3, ), @@ -91,9 +87,9 @@ where name: &str, version: Version, compressed: Compressed, - source: StoredVec, + source: BoxedAnyIterableVec, compute: ComputeFrom1, - ) -> brk_vec::Result { + ) -> Result { Ok(match mode { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, version, compressed)?, @@ -112,10 +108,10 @@ where mode: Computation, version: Version, compressed: Compressed, - source1: StoredVec, - source2: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, compute: ComputeFrom2, - ) -> brk_vec::Result { + ) -> Result { Ok(match mode { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, version, compressed)?, @@ -134,11 +130,11 @@ where name: &str, version: Version, compressed: Compressed, - source1: StoredVec, - source2: StoredVec, - source3: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, + source3: BoxedAnyIterableVec, compute: ComputeFrom3, - ) -> brk_vec::Result { + ) -> Result { Ok(match mode { Computation::Eager => Self::Eager { vec: EagerVec::forced_import(path, version, compressed)?, @@ -166,7 +162,7 @@ where let version = source.version(); let mut iter = source.iter(); let t = |i: I| { - compute(i.unwrap_to_usize(), &mut iter) + compute(i.unwrap_to_usize(), &mut *iter) .map(|v| (i, v)) .unwrap() }; @@ -177,7 +173,7 @@ where let mut iter1 = source1.iter(); let mut iter2 = source2.iter(); let t = |i: I| { - compute(i.unwrap_to_usize(), &mut iter1, &mut iter2) + compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2) .map(|v| (i, v)) .unwrap() }; @@ -189,7 +185,7 @@ where let mut iter2 = source2.iter(); let mut iter3 = source3.iter(); let t = |i: I| { - compute(i.unwrap_to_usize(), &mut iter1, &mut iter2, &mut iter3) + compute(i.unwrap_to_usize(), &mut *iter1, &mut *iter2, &mut *iter3) .map(|v| (i, v)) .unwrap() }; @@ -210,6 +206,15 @@ where S3I: StoredIndex, S3T: StoredType, { + fn version(&self) -> Version { + match self { + ComputedVec::Eager { vec, .. } => vec.version(), + ComputedVec::LazyFrom1(v) => v.version(), + ComputedVec::LazyFrom2(v) => v.version(), + ComputedVec::LazyFrom3(v) => v.version(), + } + } + fn name(&self) -> String { match self { ComputedVec::Eager { vec, .. } => vec.name(), @@ -232,14 +237,6 @@ where } } - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result> { - todo!() - } - fn modified_time(&self) -> Result { match self { ComputedVec::Eager { vec, .. } => vec.modified_time(), @@ -249,3 +246,131 @@ where } } } + +pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { + Eager(StoredVecIterator<'a, I, T>), + LazyFrom1(LazyVecFrom1Iterator<'a, I, T, S1I, S1T>), + LazyFrom2(LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>), + LazyFrom3(LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>), +} + +impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator + for ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + type Item = (I, Value<'a, T>); + fn next(&mut self) -> Option { + match self { + Self::Eager(i) => i.next(), + Self::LazyFrom1(i) => i.next(), + Self::LazyFrom2(i) => i.next(), + Self::LazyFrom3(i) => i.next(), + } + } +} + +impl BaseVecIterator + for ComputedVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + #[inline] + fn mut_index(&mut self) -> &mut usize { + match self { + Self::Eager(i) => i.mut_index(), + Self::LazyFrom1(i) => i.mut_index(), + Self::LazyFrom2(i) => i.mut_index(), + Self::LazyFrom3(i) => i.mut_index(), + } + } + + fn len(&self) -> usize { + match self { + Self::Eager(i) => i.len(), + Self::LazyFrom1(i) => i.len(), + Self::LazyFrom2(i) => i.len(), + Self::LazyFrom3(i) => i.len(), + } + } +} + +impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> IntoIterator + for &'a ComputedVec +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + type Item = (I, Value<'a, T>); + type IntoIter = ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; + + fn into_iter(self) -> Self::IntoIter { + match self { + ComputedVec::Eager { vec, .. } => ComputedVecIterator::Eager(vec.into_iter()), + ComputedVec::LazyFrom1(v) => ComputedVecIterator::LazyFrom1(v.into_iter()), + ComputedVec::LazyFrom2(v) => ComputedVecIterator::LazyFrom2(v.into_iter()), + ComputedVec::LazyFrom3(v) => ComputedVecIterator::LazyFrom3(v.into_iter()), + } + } +} + +impl AnyIterableVec + for ComputedVec +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec + for ComputedVec +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/eager.rs b/crates/brk_vec/src/variants/eager.rs similarity index 81% rename from crates/brk_computer/src/storage/vecs/vec/eager.rs rename to crates/brk_vec/src/variants/eager.rs index e1b425cc8..92342ab60 100644 --- a/crates/brk_computer/src/storage/vecs/vec/eager.rs +++ b/crates/brk_vec/src/variants/eager.rs @@ -9,12 +9,14 @@ use std::{ use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex}; use brk_exit::Exit; -use brk_vec::{ - Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, - StoredVecIterator, Value, VecIterator, Version, -}; use log::info; +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error, + GenericStoredVec, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, + VecIterator, Version, +}; + const ONE_KIB: usize = 1024; const ONE_MIB: usize = ONE_KIB * ONE_KIB; const MAX_CACHE_SIZE: usize = 210 * ONE_MIB; @@ -32,11 +34,7 @@ where { const SIZE_OF: usize = size_of::(); - pub fn forced_import( - path: &Path, - version: Version, - compressed: Compressed, - ) -> brk_vec::Result { + pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { let inner = StoredVec::forced_import(path, version, compressed)?; Ok(Self { @@ -105,21 +103,21 @@ where self.inner.modified_time() } - pub fn vec(&self) -> &StoredVec { - &self.inner - } + // pub fn vec(&self) -> &StoredVec { + // &self.inner + // } - pub fn mut_vec(&mut self) -> &StoredVec { - &mut self.inner - } + // pub fn mut_vec(&mut self) -> &StoredVec { + // &mut self.inner + // } - pub fn any_vec(&self) -> &dyn brk_vec::AnyVec { - &self.inner - } + // pub fn any_vec(&self) -> &dyn AnyVec { + // &self.inner + // } - pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyVec { - &mut self.inner - } + // pub fn mut_any_vec(&mut self) -> &mut dyn AnyVec { + // &mut self.inner + // } pub fn path(&self) -> &Path { self.inner.path() @@ -175,7 +173,7 @@ where pub fn compute_range( &mut self, max_from: I, - other: &StoredVec, + other: &impl AnyIterableVec, t: F, exit: &Exit, ) -> Result<()> @@ -189,7 +187,7 @@ where pub fn compute_transform( &mut self, max_from: A, - other: &StoredVec, + other: &impl AnyIterableVec, mut t: F, exit: &Exit, ) -> Result<()> @@ -214,7 +212,7 @@ where pub fn compute_inverse_more_to_less( &mut self, max_from: T, - other: &StoredVec, + other: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -226,7 +224,8 @@ where )?; let index = max_from.min( - VecIterator::last(self.inner.iter()).map_or_else(T::default, |(_, v)| v.into_inner()), + VecIterator::last(self.inner.into_iter()) + .map_or_else(T::default, |(_, v)| v.into_inner()), ); let mut prev_i = None; other.iter_at(index).try_for_each(|(v, i)| -> Result<()> { @@ -247,8 +246,8 @@ where pub fn compute_inverse_less_to_more( &mut self, max_from: T, - first_indexes: &StoredVec, - indexes_count: &StoredVec, + first_indexes: &impl AnyIterableVec, + indexes_count: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -280,8 +279,8 @@ where pub fn compute_count_from_indexes( &mut self, max_from: I, - first_indexes: &StoredVec, - other_to_else: &StoredVec, + first_indexes: &impl AnyIterableVec, + other_to_else: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -303,8 +302,8 @@ where pub fn compute_filtered_count_from_indexes( &mut self, max_from: I, - first_indexes: &StoredVec, - other_to_else: &StoredVec, + first_indexes: &impl AnyIterableVec, + other_to_else: &impl AnyIterableVec, filter: F, exit: &Exit, ) -> Result<()> @@ -333,8 +332,8 @@ where fn compute_filtered_count_from_indexes_( &mut self, max_from: I, - first_indexes: &StoredVec, - other_to_else: &StoredVec, + first_indexes: &impl AnyIterableVec, + other_to_else: &impl AnyIterableVec, mut filter: Option bool + '_>>, exit: &Exit, ) -> Result<()> @@ -351,7 +350,7 @@ where >::Error: error::Error + 'static, { self.validate_computed_version_or_reset_file( - Version::ZERO + self.inner.version() + first_indexes.version(), // + last_indexes.version(), + Version::ZERO + self.inner.version() + first_indexes.version(), )?; let mut other_iter = first_indexes.iter(); @@ -379,8 +378,8 @@ where pub fn compute_is_first_ordered( &mut self, max_from: I, - self_to_other: &StoredVec, - other_to_self: &StoredVec, + self_to_other: &impl AnyIterableVec, + other_to_self: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -411,9 +410,9 @@ where pub fn compute_sum_from_indexes( &mut self, max_from: I, - first_indexes: &StoredVec, - indexes_count: &StoredVec, - source: &StoredVec, + first_indexes: &impl AnyIterableVec, + indexes_count: &impl AnyIterableVec, + source: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> where @@ -454,7 +453,7 @@ where pub fn compute_from_sats( &mut self, max_from: I, - sats: &StoredVec, + sats: &impl AnyIterableVec, exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( @@ -475,8 +474,8 @@ impl EagerVec { pub fn compute_from_bitcoin( &mut self, max_from: Height, - bitcoin: &StoredVec, - price: &StoredVec>, + bitcoin: &impl AnyIterableVec, + price: &impl AnyIterableVec>, exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( @@ -499,9 +498,9 @@ impl EagerVec { pub fn compute_from_bitcoin( &mut self, max_from: TxIndex, - bitcoin: &StoredVec, - i_to_height: &StoredVec, - price: &StoredVec>, + bitcoin: &impl AnyIterableVec, + i_to_height: &impl AnyIterableVec, + price: &impl AnyIterableVec>, exit: &Exit, ) -> Result<()> { self.validate_computed_version_or_reset_file( @@ -534,3 +533,62 @@ where self.inner.into_iter() } } + +impl AnyVec for EagerVec +where + I: StoredIndex, + T: StoredType, +{ + #[inline] + fn version(&self) -> Version { + self.inner.version() + } + + #[inline] + fn name(&self) -> String { + self.inner.name() + } + + #[inline] + fn len(&self) -> usize { + self.inner.len() + } + + #[inline] + fn modified_time(&self) -> Result { + self.inner.modified_time() + } + + #[inline] + fn index_type_to_string(&self) -> &str { + I::to_string() + } +} + +impl AnyIterableVec for EagerVec +where + I: StoredIndex, + T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + I: StoredIndex, + T: StoredType + 'a, + { + Box::new(self.inner.into_iter()) + } +} + +impl AnyCollectableVec for EagerVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +} diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_vec/src/variants/indexed.rs similarity index 63% rename from crates/brk_indexer/src/vecs/base.rs rename to crates/brk_vec/src/variants/indexed.rs index d75b233eb..99f997b6a 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -2,21 +2,20 @@ use std::{ cmp::Ordering, fmt::Debug, path::{Path, PathBuf}, + time::Duration, }; -use brk_vec::{ - Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec, - StoredVecIterator, Value, VecIterator, Version, +use brk_core::Height; + +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error, + GenericStoredVec, Mmap, Result, StoredIndex, StoredType, StoredVec, Value, Version, }; -use super::Height; +use super::StoredVecIterator; #[derive(Debug, Clone)] -pub struct IndexedVec -where - I: StoredIndex, - T: StoredType, -{ +pub struct IndexedVec { height: Option, inner: StoredVec, } @@ -26,11 +25,7 @@ where I: StoredIndex, T: StoredType, { - pub fn forced_import( - path: &Path, - version: Version, - compressed: Compressed, - ) -> brk_vec::Result { + pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { Ok(Self { height: Height::try_from(Self::path_height_(path).as_path()).ok(), inner: StoredVec::forced_import(path, version, compressed)?, @@ -44,7 +39,8 @@ where #[inline] pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> { - match self.inner.len().cmp(&index.to_usize()?) { + let len = self.inner.len(); + match len.cmp(&index.to_usize()?) { Ordering::Greater => { // dbg!(len, index, &self.pathbuf); // panic!(); @@ -55,13 +51,13 @@ where Ok(()) } Ordering::Less => { - dbg!(index, value, self.inner.len(), self.path_height()); + dbg!(index, value, len, self.path_height()); Err(Error::IndexTooHigh) } } } - pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<()> { + pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> { if self.height.is_none_or(|self_height| self_height != height) { height.write(&self.path_height())?; } @@ -82,18 +78,6 @@ where &mut self.inner } - pub fn any_vec(&self) -> &dyn brk_vec::AnyVec { - &self.inner - } - - pub fn len(&self) -> usize { - self.inner.len() - } - - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - #[inline] pub fn hasnt(&self, index: I) -> Result { self.inner.has(index).map(|b| !b) @@ -108,19 +92,40 @@ where fn path_height_(path: &Path) -> PathBuf { path.join("height") } +} - pub fn iter(&self) -> StoredVecIterator<'_, I, T> { - self.into_iter() +impl AnyVec for IndexedVec +where + I: StoredIndex, + T: StoredType, +{ + #[inline] + fn version(&self) -> Version { + self.inner.version() } - pub fn iter_at(&self, i: I) -> StoredVecIterator<'_, I, T> { - let mut iter = self.into_iter(); - iter.set(i); - iter + #[inline] + fn name(&self) -> String { + self.inner.name() + } + + #[inline] + fn len(&self) -> usize { + self.inner.len() + } + + #[inline] + fn modified_time(&self) -> Result { + self.inner.modified_time() + } + + #[inline] + fn index_type_to_string(&self) -> &str { + I::to_string() } } -pub trait AnyIndexedVec: Send + Sync { +pub trait AnyIndexedVec: AnyVec { fn height(&self) -> brk_core::Result; fn flush(&mut self, height: Height) -> Result<()>; } @@ -151,3 +156,30 @@ where self.inner.into_iter() } } + +impl AnyIterableVec for IndexedVec +where + I: StoredIndex, + T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for IndexedVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy1.rs b/crates/brk_vec/src/variants/lazy1.rs similarity index 61% rename from crates/brk_computer/src/storage/vecs/vec/lazy1.rs rename to crates/brk_vec/src/variants/lazy1.rs index bb6e31511..9121dd378 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy1.rs +++ b/crates/brk_vec/src/variants/lazy1.rs @@ -1,7 +1,8 @@ use std::marker::PhantomData; -use brk_vec::{ - AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version, }; pub type ComputeFrom1 = @@ -11,7 +12,7 @@ pub type ComputeFrom1 = pub struct LazyVecFrom1 { name: String, version: Version, - source: StoredVec, + source: BoxedAnyIterableVec, compute: ComputeFrom1, phantom: PhantomData, } @@ -26,7 +27,7 @@ where pub fn init( name: &str, version: Version, - source: StoredVec, + source: BoxedAnyIterableVec, compute: ComputeFrom1, ) -> Self { Self { @@ -43,13 +44,13 @@ where } } -pub struct LazyVecIterator<'a, I, T, S1I, S1T> { +pub struct LazyVecFrom1Iterator<'a, I, T, S1I, S1T> { lazy: &'a LazyVecFrom1, - source: StoredVecIterator<'a, S1I, S1T>, + source: BoxedVecIterator<'a, S1I, S1T>, index: usize, } -impl<'a, I, T, S1I, S1T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T> +impl<'a, I, T, S1I, S1T> Iterator for LazyVecFrom1Iterator<'a, I, T, S1I, S1T> where I: StoredIndex, T: StoredType + 'a, @@ -59,7 +60,7 @@ where type Item = (I, Value<'a, T>); fn next(&mut self) -> Option { - let opt = (self.lazy.compute)(self.index, &mut self.lazy.source.iter()) + let opt = (self.lazy.compute)(self.index, &mut *self.source) .map(|v| (I::from(self.index), Value::Owned(v))); if opt.is_some() { self.index += 1; @@ -68,7 +69,7 @@ where } } -impl BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T> +impl BaseVecIterator for LazyVecFrom1Iterator<'_, I, T, S1I, S1T> where I: StoredIndex, T: StoredType, @@ -82,7 +83,7 @@ where #[inline] fn len(&self) -> usize { - todo!(); + self.source.len() } } @@ -94,10 +95,10 @@ where S1T: StoredType, { type Item = (I, Value<'a, T>); - type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T>; + type IntoIter = LazyVecFrom1Iterator<'a, I, T, S1I, S1T>; fn into_iter(self) -> Self::IntoIter { - LazyVecIterator { + LazyVecFrom1Iterator { lazy: self, source: self.source.iter(), index: 0, @@ -112,6 +113,10 @@ where S1I: StoredIndex, S1T: StoredType, { + fn version(&self) -> Version { + self.version() + } + fn name(&self) -> String { self.name.clone() } @@ -124,16 +129,38 @@ where self.source.len() } - fn modified_time(&self) -> brk_vec::Result { + fn modified_time(&self) -> Result { self.source.modified_time() } +} +impl AnyIterableVec for LazyVecFrom1 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for LazyVecFrom1 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, +{ fn collect_range_serde_json( &self, from: Option, to: Option, - ) -> brk_vec::Result> { - todo!() - // self. + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) } } diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy2.rs b/crates/brk_vec/src/variants/lazy2.rs similarity index 60% rename from crates/brk_computer/src/storage/vecs/vec/lazy2.rs rename to crates/brk_vec/src/variants/lazy2.rs index 289d1dbf4..18dc4ba42 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy2.rs +++ b/crates/brk_vec/src/variants/lazy2.rs @@ -1,7 +1,8 @@ use std::marker::PhantomData; -use brk_vec::{ - AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version, }; pub type ComputeFrom2 = for<'a> fn( @@ -14,8 +15,8 @@ pub type ComputeFrom2 = for<'a> fn( pub struct LazyVecFrom2 { name: String, version: Version, - source1: StoredVec, - source2: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, compute: ComputeFrom2, phantom: PhantomData, } @@ -32,8 +33,8 @@ where pub fn init( name: &str, version: Version, - source1: StoredVec, - source2: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, compute: ComputeFrom2, ) -> Self { Self { @@ -51,14 +52,14 @@ where } } -pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> { +pub struct LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> { lazy: &'a LazyVecFrom2, - source1: StoredVecIterator<'a, S1I, S1T>, - source2: StoredVecIterator<'a, S2I, S2T>, + source1: BoxedVecIterator<'a, S1I, S1T>, + source2: BoxedVecIterator<'a, S2I, S2T>, index: usize, } -impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T> +impl<'a, I, T, S1I, S1T, S2I, S2T> Iterator for LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, T: StoredType + 'a, @@ -70,12 +71,8 @@ where type Item = (I, Value<'a, T>); fn next(&mut self) -> Option { - let opt = (self.lazy.compute)( - self.index, - &mut self.lazy.source1.iter(), - &mut self.lazy.source2.iter(), - ) - .map(|v| (I::from(self.index), Value::Owned(v))); + let opt = (self.lazy.compute)(self.index, &mut *self.source1, &mut *self.source2) + .map(|v| (I::from(self.index), Value::Owned(v))); if opt.is_some() { self.index += 1; } @@ -83,7 +80,8 @@ where } } -impl BaseVecIterator for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T> +impl BaseVecIterator + for LazyVecFrom2Iterator<'_, I, T, S1I, S1T, S2I, S2T> where I: StoredIndex, T: StoredType, @@ -99,7 +97,7 @@ where #[inline] fn len(&self) -> usize { - todo!(); + self.source1.len().min(self.source2.len()) } } @@ -113,10 +111,10 @@ where S2T: StoredType, { type Item = (I, Value<'a, T>); - type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T>; + type IntoIter = LazyVecFrom2Iterator<'a, I, T, S1I, S1T, S2I, S2T>; fn into_iter(self) -> Self::IntoIter { - LazyVecIterator { + LazyVecFrom2Iterator { lazy: self, source1: self.source1.iter(), source2: self.source2.iter(), @@ -134,6 +132,10 @@ where S2I: StoredIndex, S2T: StoredType, { + fn version(&self) -> Version { + self.version() + } + fn name(&self) -> String { self.name.clone() } @@ -146,19 +148,45 @@ where self.source1.len().min(self.source2.len()) } - fn modified_time(&self) -> brk_vec::Result { + fn modified_time(&self) -> Result { Ok(self .source1 .modified_time()? .min(self.source2.modified_time()?)) } +} +impl AnyIterableVec for LazyVecFrom2 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for LazyVecFrom2 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ fn collect_range_serde_json( &self, from: Option, to: Option, - ) -> brk_vec::Result> { - todo!() - // self. + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) } } diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy3.rs b/crates/brk_vec/src/variants/lazy3.rs similarity index 59% rename from crates/brk_computer/src/storage/vecs/vec/lazy3.rs rename to crates/brk_vec/src/variants/lazy3.rs index 07e0ebfc7..f512bc6f9 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy3.rs +++ b/crates/brk_vec/src/variants/lazy3.rs @@ -1,7 +1,8 @@ use std::marker::PhantomData; -use brk_vec::{ - AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec, + BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version, }; pub type ComputeFrom3 = for<'a> fn( @@ -15,9 +16,9 @@ pub type ComputeFrom3 = for<'a> fn( pub struct LazyVecFrom3 { name: String, version: Version, - source1: StoredVec, - source2: StoredVec, - source3: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, + source3: BoxedAnyIterableVec, compute: ComputeFrom3, phantom: PhantomData, } @@ -36,9 +37,9 @@ where pub fn init( name: &str, version: Version, - source1: StoredVec, - source2: StoredVec, - source3: StoredVec, + source1: BoxedAnyIterableVec, + source2: BoxedAnyIterableVec, + source3: BoxedAnyIterableVec, compute: ComputeFrom3, ) -> Self { Self { @@ -57,16 +58,16 @@ where } } -pub struct LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { +pub struct LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { lazy: &'a LazyVecFrom3, - source1: StoredVecIterator<'a, S1I, S1T>, - source2: StoredVecIterator<'a, S2I, S2T>, - source3: StoredVecIterator<'a, S3I, S3T>, + source1: BoxedVecIterator<'a, S1I, S1T>, + source2: BoxedVecIterator<'a, S2I, S2T>, + source3: BoxedVecIterator<'a, S3I, S3T>, index: usize, } impl<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> Iterator - for LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> + for LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, T: StoredType + 'a, @@ -82,9 +83,9 @@ where fn next(&mut self) -> Option { let opt = (self.lazy.compute)( self.index, - &mut self.lazy.source1.iter(), - &mut self.lazy.source2.iter(), - &mut self.lazy.source3.iter(), + &mut *self.source1, + &mut *self.source2, + &mut *self.source3, ) .map(|v| (I::from(self.index), Value::Owned(v))); if opt.is_some() { @@ -95,7 +96,7 @@ where } impl BaseVecIterator - for LazyVecIterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> + for LazyVecFrom3Iterator<'_, I, T, S1I, S1T, S2I, S2T, S3I, S3T> where I: StoredIndex, T: StoredType, @@ -113,7 +114,10 @@ where #[inline] fn len(&self) -> usize { - todo!(); + self.source1 + .len() + .min(self.source2.len()) + .min(self.source3.len()) } } @@ -130,10 +134,10 @@ where S3T: StoredType, { type Item = (I, Value<'a, T>); - type IntoIter = LazyVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; + type IntoIter = LazyVecFrom3Iterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T>; fn into_iter(self) -> Self::IntoIter { - LazyVecIterator { + LazyVecFrom3Iterator { lazy: self, source1: self.source1.iter(), source2: self.source2.iter(), @@ -154,6 +158,10 @@ where S3I: StoredIndex, S3T: StoredType, { + fn version(&self) -> Version { + self.version() + } + fn name(&self) -> String { self.name.clone() } @@ -163,22 +171,58 @@ where } fn len(&self) -> usize { - self.source1.len().min(self.source2.len()) + self.source1 + .len() + .min(self.source2.len()) + .min(self.source3.len()) } - fn modified_time(&self) -> brk_vec::Result { + fn modified_time(&self) -> Result { Ok(self .source1 .modified_time()? - .min(self.source2.modified_time()?)) + .min(self.source2.modified_time()?) + .min(self.source3.modified_time()?)) } +} +impl AnyIterableVec + for LazyVecFrom3 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec + for LazyVecFrom3 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ fn collect_range_serde_json( &self, from: Option, to: Option, - ) -> brk_vec::Result> { - todo!() - // self. + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) } } diff --git a/crates/brk_vec/src/variants/mod.rs b/crates/brk_vec/src/variants/mod.rs index 325b8f96c..191a28472 100644 --- a/crates/brk_vec/src/variants/mod.rs +++ b/crates/brk_vec/src/variants/mod.rs @@ -1,5 +1,19 @@ mod compressed; +mod computed; +mod eager; +mod indexed; +mod lazy1; +mod lazy2; +mod lazy3; mod raw; +mod stored; pub use compressed::*; +pub use computed::*; +pub use eager::*; +pub use indexed::*; +pub use lazy1::*; +pub use lazy2::*; +pub use lazy3::*; pub use raw::*; +pub use stored::*; diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 95fc66fae..a71868c19 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -4,6 +4,7 @@ use std::{ mem, path::{Path, PathBuf}, sync::Arc, + time::Duration, }; use arc_swap::{ArcSwap, Guard}; @@ -11,8 +12,8 @@ use memmap2::Mmap; use rayon::prelude::*; use crate::{ - BaseVecIterator, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice, - Value, Version, + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, + Error, GenericStoredVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version, }; #[derive(Debug)] @@ -79,19 +80,16 @@ where } } -impl DynamicVec for RawVec +impl GenericStoredVec for RawVec where I: StoredIndex, T: StoredType, { - type I = I; - type T = T; - #[inline] fn read_(&self, index: usize, mmap: &Mmap) -> Result> { let index = index * Self::SIZE_OF_T; let slice = &mmap[index..(index + Self::SIZE_OF_T)]; - Self::T::try_read_from_bytes(slice) + T::try_read_from_bytes(slice) .map(|v| Some(v)) .map_err(Error::from) } @@ -119,28 +117,6 @@ where fn path(&self) -> &Path { self.pathbuf.as_path() } -} - -impl GenericVec for RawVec -where - I: StoredIndex, - T: StoredType, -{ - fn collect_range(&self, from: Option, to: Option) -> Result> { - let stored_len = self.stored_len(); - let from = from.unwrap_or_default(); - let to = to.map_or(stored_len, |i| i.min(stored_len)); - - if from >= stored_len || from >= to { - return Ok(vec![]); - } - - Ok(self - .iter_at_(from) - .take(to - from) - .map(|(_, v)| v.into_inner()) - .collect::>()) - } fn flush(&mut self) -> Result<()> { let pushed_len = self.pushed_len(); @@ -188,10 +164,41 @@ where Ok(()) } + // #[inline] + // fn version(&self) -> Version { + // self.version + // } +} + +impl AnyVec for RawVec +where + I: StoredIndex, + T: StoredType, +{ #[inline] fn version(&self) -> Version { self.version } + + #[inline] + fn name(&self) -> String { + self.name_() + } + + #[inline] + fn len(&self) -> usize { + self.len_() + } + + #[inline] + fn modified_time(&self) -> Result { + self.modified_time_() + } + + #[inline] + fn index_type_to_string(&self) -> &str { + I::to_string() + } } impl Clone for RawVec { @@ -270,3 +277,30 @@ where } } } + +impl AnyIterableVec for RawVec +where + I: StoredIndex, + T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for RawVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +} diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs new file mode 100644 index 000000000..6ef5c53a8 --- /dev/null +++ b/crates/brk_vec/src/variants/stored.rs @@ -0,0 +1,223 @@ +use std::{path::Path, time::Duration}; + +use arc_swap::ArcSwap; +use memmap2::Mmap; + +use crate::{ + AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec, + Compressed, GenericStoredVec, Result, StoredIndex, StoredType, Value, Version, +}; + +use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator}; + +#[derive(Debug, Clone)] +pub enum StoredVec { + Raw(RawVec), + Compressed(CompressedVec), +} + +impl StoredVec +where + I: StoredIndex, + T: StoredType, +{ + pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { + if *compressed { + Ok(Self::Compressed(CompressedVec::forced_import( + path, version, + )?)) + } else { + Ok(Self::Raw(RawVec::forced_import(path, version)?)) + } + } +} + +impl GenericStoredVec for StoredVec +where + I: StoredIndex, + T: StoredType, +{ + #[inline] + fn read_(&self, index: usize, guard: &Mmap) -> Result> { + match self { + StoredVec::Raw(v) => v.read_(index, guard), + StoredVec::Compressed(v) => v.read_(index, guard), + } + } + + #[inline] + fn mmap(&self) -> &ArcSwap { + match self { + StoredVec::Raw(v) => v.mmap(), + StoredVec::Compressed(v) => v.mmap(), + } + } + + #[inline] + fn stored_len(&self) -> usize { + match self { + StoredVec::Raw(v) => v.stored_len(), + StoredVec::Compressed(v) => v.stored_len(), + } + } + + #[inline] + fn pushed(&self) -> &[T] { + match self { + StoredVec::Raw(v) => v.pushed(), + StoredVec::Compressed(v) => v.pushed(), + } + } + #[inline] + fn mut_pushed(&mut self) -> &mut Vec { + match self { + StoredVec::Raw(v) => v.mut_pushed(), + StoredVec::Compressed(v) => v.mut_pushed(), + } + } + + #[inline] + fn path(&self) -> &Path { + match self { + StoredVec::Raw(v) => v.path(), + StoredVec::Compressed(v) => v.path(), + } + } + + fn flush(&mut self) -> Result<()> { + match self { + StoredVec::Raw(v) => v.flush(), + StoredVec::Compressed(v) => v.flush(), + } + } + + fn truncate_if_needed(&mut self, index: I) -> Result<()> { + match self { + StoredVec::Raw(v) => v.truncate_if_needed(index), + StoredVec::Compressed(v) => v.truncate_if_needed(index), + } + } +} + +impl AnyVec for StoredVec +where + I: StoredIndex, + T: StoredType, +{ + #[inline] + fn version(&self) -> Version { + match self { + StoredVec::Raw(v) => v.version(), + StoredVec::Compressed(v) => v.version(), + } + } + + #[inline] + fn index_type_to_string(&self) -> &str { + I::to_string() + } + + #[inline] + fn len(&self) -> usize { + self.pushed_len() + self.stored_len() + } + + #[inline] + fn modified_time(&self) -> Result { + match self { + StoredVec::Raw(v) => v.modified_time(), + StoredVec::Compressed(v) => v.modified_time(), + } + } + + fn name(&self) -> String { + match self { + StoredVec::Raw(v) => v.name(), + StoredVec::Compressed(v) => v.name(), + } + } +} + +#[derive(Debug)] +pub enum StoredVecIterator<'a, I, T> { + Raw(RawVecIterator<'a, I, T>), + Compressed(CompressedVecIterator<'a, I, T>), +} + +impl<'a, I, T> Iterator for StoredVecIterator<'a, I, T> +where + I: StoredIndex, + T: StoredType, +{ + type Item = (I, Value<'a, T>); + fn next(&mut self) -> Option { + match self { + Self::Compressed(i) => i.next(), + Self::Raw(i) => i.next(), + } + } +} + +impl BaseVecIterator for StoredVecIterator<'_, I, T> +where + I: StoredIndex, + T: StoredType, +{ + #[inline] + fn mut_index(&mut self) -> &mut usize { + match self { + Self::Compressed(iter) => iter.mut_index(), + Self::Raw(iter) => iter.mut_index(), + } + } + + fn len(&self) -> usize { + match self { + Self::Compressed(i) => i.len(), + Self::Raw(i) => i.len(), + } + } +} + +impl<'a, I, T> IntoIterator for &'a StoredVec +where + I: StoredIndex, + T: StoredType, +{ + type Item = (I, Value<'a, T>); + type IntoIter = StoredVecIterator<'a, I, T>; + + fn into_iter(self) -> Self::IntoIter { + match self { + StoredVec::Compressed(v) => StoredVecIterator::Compressed(v.into_iter()), + StoredVec::Raw(v) => StoredVecIterator::Raw(v.into_iter()), + } + } +} + +impl AnyIterableVec for StoredVec +where + I: StoredIndex, + T: StoredType, +{ + fn boxed_iter<'a>(&'a self) -> BoxedVecIterator<'a, I, T> + where + T: 'a, + { + Box::new(self.into_iter()) + } +} + +impl AnyCollectableVec for StoredVec +where + I: StoredIndex, + T: StoredType, +{ + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + CollectableVec::collect_range_serde_json(self, from, to) + } +}