diff --git a/Cargo.lock b/Cargo.lock index 672605d8e..11a72bd5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -435,6 +435,7 @@ dependencies = [ "fjall", "log", "serde", + "serde_json", ] [[package]] @@ -571,7 +572,6 @@ name = "brk_vec" version = "0.0.31" dependencies = [ "arc-swap", - "axum", "memmap2", "rayon", "serde", diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 39d20acf6..53f658002 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -20,3 +20,4 @@ color-eyre = { workspace = true } fjall = { workspace = true } log = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 71749d7fe..8436c83f7 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -223,7 +223,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ vec![ self.height_to_interval.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index da68eb39d..4b8233134 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -523,8 +523,8 @@ where self.total.as_mut().unwrap() } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { - let mut v: Vec<&dyn brk_vec::AnyStoredVec> = vec![]; + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { + let mut v: Vec<&dyn brk_vec::AnyVec> = vec![]; if let Some(first) = self.first.as_ref() { v.push(first.any_vec()); diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index 08f8ac156..4c091e7b8 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, Version}; +use brk_vec::{AnyVec, Compressed, Result, Version}; use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; @@ -140,7 +140,7 @@ where Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ vec![self.dateindex.any_vec()], self.dateindex_extra.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 424fc1499..be13052bb 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -5,7 +5,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; @@ -180,7 +180,7 @@ where Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]), self.height_extra.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 19b9a8b9a..9847197b7 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_core::{DifficultyEpoch, Height}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, Version}; +use brk_vec::{AnyVec, Compressed, Result, Version}; use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; @@ -86,7 +86,7 @@ where Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ vec![self.height.any_vec()], self.height_extra.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index 78abf3e68..ec66f9684 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -6,7 +6,7 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ComputedType, EagerVec, Indexes, indexes}; @@ -188,7 +188,7 @@ where Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]), self.height.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs index 80230f485..57d89caa6 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_core::{Bitcoin, Dollars, Height, Sats}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ EagerVec, marketprice, @@ -145,7 +145,7 @@ impl ComputedValueVecsFromHeight { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ self.sats.any_vecs(), self.bitcoin.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs index 6991ab71a..c66fc4bb6 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs @@ -3,7 +3,7 @@ use std::path::Path; use brk_core::{Bitcoin, Dollars, Sats, TxIndex}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; +use brk_vec::{AnyVec, Compressed, Result, StoredVec, Version}; use crate::storage::{ EagerVec, marketprice, @@ -147,7 +147,7 @@ impl ComputedValueVecsFromTxindex { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ self.sats.any_vecs(), self.bitcoin.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index f3f9e29f0..6296e148d 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -865,7 +865,7 @@ impl Vecs { }) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { vec![ self.dateindex_to_date.any_vec(), self.dateindex_to_dateindex.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index b66505363..d4d98decf 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -1063,7 +1063,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { vec![ vec![ self.dateindex_to_close_in_cents.any_vec(), diff --git a/crates/brk_computer/src/storage/vecs/mining.rs b/crates/brk_computer/src/storage/vecs/mining.rs index e939d4fb0..4224cbcde 100644 --- a/crates/brk_computer/src/storage/vecs/mining.rs +++ b/crates/brk_computer/src/storage/vecs/mining.rs @@ -113,7 +113,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ self.indexes_to_difficulty.any_vecs(), self.indexes_to_difficultyepoch.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index a192d31e5..2a45193d8 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -3,7 +3,7 @@ use std::{fs, path::Path}; use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use brk_vec::{AnyStoredVec, Compressed}; +use brk_vec::{AnyVec, Compressed}; pub mod blocks; pub mod grouped; @@ -86,7 +86,7 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { [ self.indexes.any_vecs(), self.blocks.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 2b3070a7b..f001fa7da 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -6,7 +6,7 @@ use brk_core::{ use brk_exit::Exit; use brk_indexer::Indexer; use brk_parser::bitcoin; -use brk_vec::{Compressed, StoredIndex, VecIterator, Version}; +use brk_vec::{AnyVec, Compressed, StoredIndex, VecIterator, Version}; use super::{ Computation, ComputedVec, ComputedVecFrom2, EagerVec, Indexes, @@ -113,8 +113,9 @@ impl Vecs { .add_total(), )?, inputindex_to_value: ComputedVec::forced_import_or_init_from_2( + path, + "inputindex_to_value", computation, - &path.join("inputindex_to_value"), Version::ZERO, compressed, indexer.vecs().outputindex_to_value.vec().clone(), @@ -728,6 +729,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2ms_count.compute_all( indexer, indexes, @@ -742,6 +744,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2pk33_count.compute_all( indexer, indexes, @@ -756,6 +759,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2pk65_count.compute_all( indexer, indexes, @@ -770,6 +774,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2pkh_count.compute_all( indexer, indexes, @@ -784,6 +789,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2sh_count.compute_all( indexer, indexes, @@ -798,6 +804,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2tr_count.compute_all( indexer, indexes, @@ -812,6 +819,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2wpkh_count.compute_all( indexer, indexes, @@ -826,6 +834,7 @@ impl Vecs { ) }, )?; + self.indexes_to_p2wsh_count.compute_all( indexer, indexes, @@ -840,6 +849,7 @@ impl Vecs { ) }, )?; + self.indexes_to_opreturn_count.compute_all( indexer, indexes, @@ -854,6 +864,7 @@ impl Vecs { ) }, )?; + self.indexes_to_unknownoutput_count.compute_all( indexer, indexes, @@ -868,6 +879,7 @@ impl Vecs { ) }, )?; + self.indexes_to_emptyoutput_count.compute_all( indexer, indexes, @@ -886,13 +898,13 @@ impl Vecs { Ok(()) } - pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyVec> { [ vec![ + self.inputindex_to_value.any_vec(), self.txindex_to_is_coinbase.any_vec(), - // self.inputindex_to_value.any_vec(), - self.txindex_to_weight.any_vec(), self.txindex_to_vsize.any_vec(), + self.txindex_to_weight.any_vec(), ], self.indexes_to_tx_count.any_vecs(), self.indexes_to_coinbase.any_vecs(), diff --git a/crates/brk_computer/src/storage/vecs/vec/eager.rs b/crates/brk_computer/src/storage/vecs/vec/eager.rs index 7c7db8a49..e1b425cc8 100644 --- a/crates/brk_computer/src/storage/vecs/vec/eager.rs +++ b/crates/brk_computer/src/storage/vecs/vec/eager.rs @@ -4,6 +4,7 @@ use std::{ fmt::Debug, ops::Add, path::{Path, PathBuf}, + time::Duration, }; use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, StoredUsize, TxIndex}; @@ -96,8 +97,12 @@ where self.inner.is_empty() } - fn file_name(&self) -> String { - self.inner.file_name() + pub fn name(&self) -> String { + self.inner.name() + } + + pub fn modified_time(&self) -> Result { + self.inner.modified_time() } pub fn vec(&self) -> &StoredVec { @@ -108,11 +113,11 @@ where &mut self.inner } - pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec { + pub fn any_vec(&self) -> &dyn brk_vec::AnyVec { &self.inner } - pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyStoredVec { + pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyVec { &mut self.inner } @@ -133,7 +138,7 @@ where version.write(path.as_ref())?; if self.is_empty() { - info!("Computing {}...", self.file_name()) + info!("Computing {}...", self.name()) } Ok(()) diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy1.rs b/crates/brk_computer/src/storage/vecs/vec/lazy1.rs index 9c67f11e1..bb6e31511 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy1.rs +++ b/crates/brk_computer/src/storage/vecs/vec/lazy1.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use brk_vec::{ - BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, + AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; pub type ComputeFrom1 = @@ -9,6 +9,7 @@ pub type ComputeFrom1 = #[derive(Clone)] pub struct LazyVecFrom1 { + name: String, version: Version, source: StoredVec, compute: ComputeFrom1, @@ -23,11 +24,13 @@ where S1T: StoredType, { pub fn init( + name: &str, version: Version, source: StoredVec, compute: ComputeFrom1, ) -> Self { Self { + name: name.to_owned(), version, source, compute, @@ -101,3 +104,36 @@ where } } } + +impl AnyVec for LazyVecFrom1 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, +{ + fn name(&self) -> String { + self.name.clone() + } + + fn index_type_to_string(&self) -> &str { + I::to_string() + } + + fn len(&self) -> usize { + self.source.len() + } + + fn modified_time(&self) -> brk_vec::Result { + self.source.modified_time() + } + + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> brk_vec::Result> { + todo!() + // self. + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy2.rs b/crates/brk_computer/src/storage/vecs/vec/lazy2.rs index d4561661a..289d1dbf4 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy2.rs +++ b/crates/brk_computer/src/storage/vecs/vec/lazy2.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use brk_vec::{ - BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, + AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; pub type ComputeFrom2 = for<'a> fn( @@ -12,6 +12,7 @@ pub type ComputeFrom2 = for<'a> fn( #[derive(Clone)] pub struct LazyVecFrom2 { + name: String, version: Version, source1: StoredVec, source2: StoredVec, @@ -29,12 +30,14 @@ where S2T: StoredType, { pub fn init( + name: &str, version: Version, source1: StoredVec, source2: StoredVec, compute: ComputeFrom2, ) -> Self { Self { + name: name.to_string(), version, source1, source2, @@ -121,3 +124,41 @@ where } } } + +impl AnyVec for LazyVecFrom2 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, +{ + fn name(&self) -> String { + self.name.clone() + } + + fn index_type_to_string(&self) -> &str { + I::to_string() + } + + fn len(&self) -> usize { + self.source1.len().min(self.source2.len()) + } + + fn modified_time(&self) -> brk_vec::Result { + Ok(self + .source1 + .modified_time()? + .min(self.source2.modified_time()?)) + } + + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> brk_vec::Result> { + todo!() + // self. + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/lazy3.rs b/crates/brk_computer/src/storage/vecs/vec/lazy3.rs index 0a6baf8ef..07e0ebfc7 100644 --- a/crates/brk_computer/src/storage/vecs/vec/lazy3.rs +++ b/crates/brk_computer/src/storage/vecs/vec/lazy3.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use brk_vec::{ - BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, + AnyVec, BaseVecIterator, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value, Version, }; pub type ComputeFrom3 = for<'a> fn( @@ -13,6 +13,7 @@ pub type ComputeFrom3 = for<'a> fn( #[derive(Clone)] pub struct LazyVecFrom3 { + name: String, version: Version, source1: StoredVec, source2: StoredVec, @@ -33,6 +34,7 @@ where S3T: StoredType, { pub fn init( + name: &str, version: Version, source1: StoredVec, source2: StoredVec, @@ -40,6 +42,7 @@ where compute: ComputeFrom3, ) -> Self { Self { + name: name.to_string(), version, source1, source2, @@ -139,3 +142,43 @@ where } } } + +impl AnyVec for LazyVecFrom3 +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + fn name(&self) -> String { + self.name.clone() + } + + fn index_type_to_string(&self) -> &str { + I::to_string() + } + + fn len(&self) -> usize { + self.source1.len().min(self.source2.len()) + } + + fn modified_time(&self) -> brk_vec::Result { + Ok(self + .source1 + .modified_time()? + .min(self.source2.modified_time()?)) + } + + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> brk_vec::Result> { + todo!() + // self. + } +} diff --git a/crates/brk_computer/src/storage/vecs/vec/mod.rs b/crates/brk_computer/src/storage/vecs/vec/mod.rs index db26330ea..f23ba13f9 100644 --- a/crates/brk_computer/src/storage/vecs/vec/mod.rs +++ b/crates/brk_computer/src/storage/vecs/vec/mod.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{path::Path, time::Duration}; use brk_exit::Exit; use clap_derive::ValueEnum; @@ -12,7 +12,9 @@ mod lazy3; pub use _type::*; use brk_core::StoredPhantom; -use brk_vec::{Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version}; +use brk_vec::{ + AnyVec, Compressed, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version, +}; pub use eager::*; pub use lazy1::*; pub use lazy2::*; @@ -38,7 +40,7 @@ impl Computation { } #[derive(Clone)] -enum Dependencies { +pub enum Dependencies { From1(StoredVec, ComputeFrom1), From2( (StoredVec, StoredVec), @@ -70,7 +72,6 @@ pub enum ComputedVec { LazyFrom1(LazyVecFrom1), LazyFrom2(LazyVecFrom2), LazyFrom3(LazyVecFrom3), - // Lazy4 } impl ComputedVec @@ -87,6 +88,7 @@ where pub fn forced_import_or_init_from_1( mode: Computation, path: &Path, + name: &str, version: Version, compressed: Compressed, source: StoredVec, @@ -97,13 +99,17 @@ where vec: EagerVec::forced_import(path, version, compressed)?, deps: Dependencies::From1(source, compute), }, - Computation::Lazy => Self::LazyFrom1(LazyVecFrom1::init(version, source, compute)), + Computation::Lazy => { + Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute)) + } }) } + #[allow(clippy::too_many_arguments)] pub fn forced_import_or_init_from_2( - mode: Computation, path: &Path, + name: &str, + mode: Computation, version: Version, compressed: Compressed, source1: StoredVec, @@ -116,7 +122,7 @@ where deps: Dependencies::From2((source1, source2), compute), }, Computation::Lazy => { - Self::LazyFrom2(LazyVecFrom2::init(version, source1, source2, compute)) + Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute)) } }) } @@ -125,6 +131,7 @@ where pub fn forced_import_or_init_from_3( mode: Computation, path: &Path, + name: &str, version: Version, compressed: Compressed, source1: StoredVec, @@ -138,7 +145,7 @@ where deps: Dependencies::From3((source1, source2, source3), compute), }, Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init( - version, source1, source2, source3, compute, + name, version, source1, source2, source3, compute, )), }) } @@ -191,3 +198,54 @@ where } } } + +impl AnyVec for ComputedVec +where + I: StoredIndex, + T: StoredType, + S1I: StoredIndex, + S1T: StoredType, + S2I: StoredIndex, + S2T: StoredType, + S3I: StoredIndex, + S3T: StoredType, +{ + fn name(&self) -> String { + match self { + ComputedVec::Eager { vec, .. } => vec.name(), + ComputedVec::LazyFrom1(v) => v.name(), + ComputedVec::LazyFrom2(v) => v.name(), + ComputedVec::LazyFrom3(v) => v.name(), + } + } + + fn index_type_to_string(&self) -> &str { + I::to_string() + } + + fn len(&self) -> usize { + match self { + ComputedVec::Eager { vec, .. } => vec.len(), + ComputedVec::LazyFrom1(v) => v.len(), + ComputedVec::LazyFrom2(v) => v.len(), + ComputedVec::LazyFrom3(v) => v.len(), + } + } + + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result> { + todo!() + } + + fn modified_time(&self) -> Result { + match self { + ComputedVec::Eager { vec, .. } => vec.modified_time(), + ComputedVec::LazyFrom1(v) => v.modified_time(), + ComputedVec::LazyFrom2(v) => v.modified_time(), + ComputedVec::LazyFrom3(v) => v.modified_time(), + } + } +} diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index c7f8fe73b..d75b233eb 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -82,7 +82,7 @@ where &mut self.inner } - pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec { + pub fn any_vec(&self) -> &dyn brk_vec::AnyVec { &self.inner } diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index c841be780..2c970629a 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -7,7 +7,7 @@ use brk_core::{ P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight, }; -use brk_vec::{AnyStoredVec, Compressed, Result, Version}; +use brk_vec::{AnyVec, Compressed, Result, Version}; use rayon::prelude::*; use crate::Indexes; @@ -455,7 +455,7 @@ impl Vecs { .unwrap() } - pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyVec> { vec![ self.emptyoutputindex_to_txindex.any_vec(), self.height_to_blockhash.any_vec(), diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index ffff4226e..01d85a9e2 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -5,7 +5,7 @@ use brk_computer::Computer; use brk_indexer::Indexer; -use brk_vec::AnyStoredVec; +use brk_vec::AnyVec; use tabled::settings::Style; mod format; @@ -51,7 +51,7 @@ impl<'a> Query<'a> { } } - pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyStoredVec)> { + pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyVec)> { let tuples = ids .iter() .flat_map(|s| { @@ -86,7 +86,7 @@ impl<'a> Query<'a> { pub fn format( &self, - vecs: Vec<(String, &&dyn AnyStoredVec)>, + vecs: Vec<(String, &&dyn AnyVec)>, from: Option, to: Option, format: Option, diff --git a/crates/brk_query/src/vec_trees.rs b/crates/brk_query/src/vec_trees.rs index b5088947a..4178ffc77 100644 --- a/crates/brk_query/src/vec_trees.rs +++ b/crates/brk_query/src/vec_trees.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use brk_vec::AnyStoredVec; +use brk_vec::AnyVec; use derive_deref::{Deref, DerefMut}; use super::index::Index; @@ -13,11 +13,11 @@ pub struct VecTrees<'a> { impl<'a> VecTrees<'a> { // Not the most performant or type safe but only built once so that's okay - pub fn insert(&mut self, vec: &'a dyn AnyStoredVec) { - let file_name = vec.file_name(); - let split = file_name.split("_to_").collect::>(); + pub fn insert(&mut self, vec: &'a dyn AnyVec) { + let name = vec.name(); + let split = name.split("_to_").collect::>(); if split.len() != 2 { - dbg!(&file_name, &split); + dbg!(&name, &split); panic!(); } let str = vec @@ -32,7 +32,7 @@ impl<'a> VecTrees<'a> { }) .unwrap(); if split[0] != index.to_string().to_lowercase() { - dbg!(&file_name, split[0], index.to_string()); + dbg!(&name, split[0], index.to_string()); panic!(); } let key = split[1].to_string().replace("_", "-"); @@ -42,7 +42,7 @@ impl<'a> VecTrees<'a> { .or_default() .insert(index.clone(), vec); if prev.is_some() { - dbg!(&key, str, file_name); + dbg!(&key, str, name); panic!() } let prev = self @@ -51,7 +51,7 @@ impl<'a> VecTrees<'a> { .or_default() .insert(key.clone(), vec); if prev.is_some() { - dbg!(&key, str, file_name); + dbg!(&key, str, name); panic!() } } @@ -88,7 +88,7 @@ impl<'a> VecTrees<'a> { } #[derive(Default, Deref, DerefMut)] -pub struct IndexToVec<'a>(BTreeMap); +pub struct IndexToVec<'a>(BTreeMap); #[derive(Default, Deref, DerefMut)] -pub struct IdToVec<'a>(BTreeMap); +pub struct IdToVec<'a>(BTreeMap); diff --git a/crates/brk_server/src/api/query/mod.rs b/crates/brk_server/src/api/query/mod.rs index d5c3f340a..4eb129dc0 100644 --- a/crates/brk_server/src/api/query/mod.rs +++ b/crates/brk_server/src/api/query/mod.rs @@ -53,9 +53,9 @@ fn req_to_response_res( if to.is_none() { let not_modified = vecs .iter() - .map(|(_, vec)| headers.check_if_modified_since(&vec.path_vec())) + .map(|(_, vec)| headers.check_if_modified_since_(vec.modified_time()?)) .all(|res| { - res.is_ok_and(|(modified, date_modified)| { + res.ok().is_some_and(|(modified, date_modified)| { if date_modified_opt.is_none_or(|dm| dm > date_modified) { date_modified_opt.replace(date_modified); } diff --git a/crates/brk_server/src/traits/header_map.rs b/crates/brk_server/src/traits/header_map.rs index 07db81a4c..ec433b802 100644 --- a/crates/brk_server/src/traits/header_map.rs +++ b/crates/brk_server/src/traits/header_map.rs @@ -1,4 +1,7 @@ -use std::{path::Path, time}; +use std::{ + path::Path, + time::{self, Duration}, +}; use axum::http::{ HeaderMap, @@ -28,6 +31,10 @@ pub trait HeaderMapExtended { fn get_if_modified_since(&self) -> Option; fn check_if_modified_since(&self, path: &Path) -> color_eyre::Result<(ModifiedState, DateTime)>; + fn check_if_modified_since_( + &self, + duration: Duration, + ) -> color_eyre::Result<(ModifiedState, DateTime)>; fn insert_cache_control_immutable(&mut self); fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64); @@ -118,11 +125,17 @@ impl HeaderMapExtended for HeaderMap { &self, path: &Path, ) -> color_eyre::Result<(ModifiedState, DateTime)> { - let duration = path - .metadata()? - .modified()? - .duration_since(time::UNIX_EPOCH) - .unwrap(); + self.check_if_modified_since_( + path.metadata()? + .modified()? + .duration_since(time::UNIX_EPOCH)?, + ) + } + + fn check_if_modified_since_( + &self, + duration: Duration, + ) -> color_eyre::Result<(ModifiedState, DateTime)> { let date = Timestamp::new(duration.as_secs() as i64, 0) .unwrap() .to_zoned(TimeZone::UTC) diff --git a/crates/brk_vec/Cargo.toml b/crates/brk_vec/Cargo.toml index 734a1d506..46fd7587e 100644 --- a/crates/brk_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -9,7 +9,6 @@ license.workspace = true repository.workspace = true [dependencies] -axum = { workspace = true } arc-swap = "1.7.1" memmap2 = "0.9.5" rayon = { workspace = true } diff --git a/crates/brk_vec/src/enums/error.rs b/crates/brk_vec/src/enums/error.rs index 645e5be53..f80135b4c 100644 --- a/crates/brk_vec/src/enums/error.rs +++ b/crates/brk_vec/src/enums/error.rs @@ -1,6 +1,7 @@ use std::{ fmt::{self, Debug}, io, + time::SystemTimeError, }; use crate::Version; @@ -23,9 +24,16 @@ pub enum Error { UnsupportedUnflushedState, RangeFromAfterTo(usize, usize), DifferentCompressionMode, + SystemTimeError, ToSerdeJsonValueError(serde_json::Error), } +impl From for Error { + fn from(_: SystemTimeError) -> Self { + Self::SystemTimeError + } +} + impl From for Error { fn from(value: io::Error) -> Self { Self::IO(value) @@ -74,6 +82,7 @@ impl fmt::Display for Error { ) } Error::ZeroCopyError => write!(f, "Zero copy convert error"), + Error::SystemTimeError => write!(f, "SystemTimeError"), Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"), Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"), Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"), diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 90524dea7..30066cc42 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -8,10 +8,9 @@ mod structs; mod traits; mod variants; -use std::path::{Path, PathBuf}; +use std::{path::Path, time::Duration}; use arc_swap::ArcSwap; -use axum::response::Response; pub use enums::*; pub use memmap2::Mmap; pub use structs::*; @@ -141,22 +140,7 @@ where } } -pub trait AnyStoredVec: Send + Sync { - fn file_name(&self) -> String; - fn index_type_to_string(&self) -> &str; - fn len(&self) -> usize; - fn is_empty(&self) -> bool; - fn flush(&mut self) -> Result<()>; - fn collect_range_serde_json( - &self, - from: Option, - to: Option, - ) -> Result>; - fn collect_range_response(&self, from: Option, to: Option) -> Result; - fn path_vec(&self) -> PathBuf; -} - -impl AnyStoredVec for StoredVec +impl AnyVec for StoredVec where I: StoredIndex, T: StoredType, @@ -166,20 +150,11 @@ where DynamicVec::len(self) } - #[inline] - fn is_empty(&self) -> bool { - DynamicVec::is_empty(self) - } - #[inline] fn index_type_to_string(&self) -> &str { GenericVec::index_type_to_string(self) } - fn flush(&mut self) -> Result<()> { - GenericVec::flush(self) - } - #[inline] fn collect_range_serde_json( &self, @@ -190,17 +165,12 @@ where } #[inline] - fn collect_range_response(&self, from: Option, to: Option) -> Result { - GenericVec::collect_range_response(self, from, to) + fn modified_time(&self) -> Result { + GenericVec::modified_time(self) } - #[inline] - fn path_vec(&self) -> PathBuf { - GenericVec::path_vec(self) - } - - fn file_name(&self) -> String { - GenericVec::file_name(self) + fn name(&self) -> String { + GenericVec::name(self) } } diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs new file mode 100644 index 000000000..e28a2cae1 --- /dev/null +++ b/crates/brk_vec/src/traits/any.rs @@ -0,0 +1,24 @@ +use std::time::Duration; + +use crate::Result; + +pub trait AnyVec: Send + Sync { + fn name(&self) -> String; + fn index_type_to_string(&self) -> &str; + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn collect_range_serde_json( + &self, + from: Option, + to: Option, + ) -> Result>; + fn modified_time(&self) -> Result; + fn any_vec(&self) -> &dyn AnyVec + where + Self: Sized, + { + self + } +} diff --git a/crates/brk_vec/src/traits/generic.rs b/crates/brk_vec/src/traits/generic.rs index c5960082f..dd0615648 100644 --- a/crates/brk_vec/src/traits/generic.rs +++ b/crates/brk_vec/src/traits/generic.rs @@ -3,12 +3,9 @@ use std::{ io::{self, Seek, SeekFrom, Write}, path::{Path, PathBuf}, sync::Arc, + time::{self, Duration}, }; -use axum::{ - Json, - response::{IntoResponse, Response}, -}; use memmap2::Mmap; use crate::{Error, Result, Version}; @@ -96,11 +93,6 @@ where fn collect_range(&self, from: Option, to: Option) -> Result>; - // #[inline] - // fn collect_inclusive_range(&self, from: I, to: I) -> Result> { - // self.collect_range(Some(from.to_usize()?), Some(to.to_usize()? + 1)) - // } - #[inline] fn i64_to_usize(i: i64, len: usize) -> usize { if i >= 0 { @@ -119,15 +111,6 @@ where self.collect_range(from, to) } - #[inline] - fn collect_range_axum_json( - &self, - from: Option, - to: Option, - ) -> Result>> { - Ok(Json(self.collect_signed_range(from, to)?)) - } - #[inline] fn collect_range_serde_json( &self, @@ -140,11 +123,6 @@ where .collect::>>() } - #[inline] - fn collect_range_response(&self, from: Option, to: Option) -> Result { - Ok(self.collect_range_axum_json(from, to)?.into_response()) - } - #[inline] fn path_vec(&self) -> PathBuf { Self::path_vec_(self.path()) @@ -165,7 +143,7 @@ where } #[inline] - fn file_name(&self) -> String { + fn name(&self) -> String { self.path() .file_name() .unwrap() @@ -174,5 +152,13 @@ where .to_owned() } + fn modified_time(&self) -> Result { + Ok(self + .path_vec() + .metadata()? + .modified()? + .duration_since(time::UNIX_EPOCH)?) + } + fn version(&self) -> Version; } diff --git a/crates/brk_vec/src/traits/mod.rs b/crates/brk_vec/src/traits/mod.rs index c8891be5c..b7c1a0ad7 100644 --- a/crates/brk_vec/src/traits/mod.rs +++ b/crates/brk_vec/src/traits/mod.rs @@ -1,9 +1,11 @@ +mod any; mod dynamic; mod generic; mod iterator; mod stored_index; mod stored_type; +pub use any::*; pub use dynamic::*; pub use generic::*; pub use iterator::*;