From 52cfbf60d42331b2efe94f49b11e9e4c98abbe73 Mon Sep 17 00:00:00 2001 From: nym21 Date: Thu, 20 Mar 2025 21:40:06 +0100 Subject: [PATCH] computer: part 2 --- Cargo.lock | 20 +- crates/brk_cli/src/run.rs | 8 +- crates/brk_computer/src/storage/vecs/base.rs | 6 +- .../brk_computer/src/storage/vecs/blocks.rs | 22 +- .../brk_computer/src/storage/vecs/indexes.rs | 438 +++++++++++++++++- crates/brk_computer/src/storage/vecs/mod.rs | 1 + .../src/storage/vecs/stats/generic.rs | 232 ++++++++-- .../src/storage/vecs/stats/grouped.rs | 98 +++- .../src/storage/vecs/stats/stored_type.rs | 10 +- crates/brk_core/src/structs/dateindex.rs | 8 +- crates/brk_core/src/structs/decadeindex.rs | 37 ++ .../brk_core/src/structs/difficultyepoch.rs | 16 + crates/brk_core/src/structs/halvingepoch.rs | 16 + crates/brk_core/src/structs/height.rs | 2 +- crates/brk_core/src/structs/monthindex.rs | 22 + crates/brk_core/src/structs/timestamp.rs | 17 +- crates/brk_core/src/structs/weekindex.rs | 40 ++ crates/brk_core/src/structs/yearindex.rs | 34 ++ crates/brk_query/src/index.rs | 28 +- crates/brk_query/src/tree.rs | 2 +- crates/brk_server/Cargo.toml | 2 +- crates/brk_vec/src/lib.rs | 34 +- crates/brk_vec/src/traits/stored_index.rs | 6 + update.sh | 5 + 24 files changed, 997 insertions(+), 107 deletions(-) create mode 100755 update.sh diff --git a/Cargo.lock b/Cargo.lock index 52156bc39..5b368c932 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1813,9 +1813,9 @@ dependencies = [ [[package]] name = "oxc-miette" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112f8458565c773a1f6555c14d6cd0255f56c85dd372932029e4baeb15308bbe" +checksum = "b8c278d00ecc50ee84aba4768a7ab74eb325dff4dca8c0581495b850d53480ba" dependencies = [ "cfg-if", "owo-colors 4.2.0", @@ -2494,9 +2494,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" dependencies = [ "bitflags", "errno", @@ -2848,9 +2848,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" dependencies = [ "fastrand", "getrandom 0.3.2", @@ -3269,9 +3269,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] name = "windows-sys" @@ -3430,9 +3430,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938cc23ac49778ac8340e366ddc422b2227ea176edb447e23fc0627608dddadd" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" dependencies = [ "aes", "arbitrary", diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 599fe86ae..448112f83 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -109,11 +109,11 @@ pub struct RunConfig { #[arg(short, long, value_name = "BOOL")] compressed: Option, - /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: false, saved + /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved #[arg(short, long, value_name = "BOOL")] fetch: Option, - /// Website served by the server (if active), default: none, saved + /// Website served by the server (if active), default: kibo.money, saved #[arg(short, long)] website: Option, @@ -390,11 +390,11 @@ impl RunConfig { } pub fn website(&self) -> Website { - self.website.unwrap_or_default() + self.website.unwrap_or(Website::KiboMoney) } pub fn fetch(&self) -> bool { - self.fetch.is_some_and(|b| b) + self.fetch.is_none_or(|b| b) } pub fn compressed(&self) -> bool { diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index f9070b64a..6359a1a74 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -82,7 +82,7 @@ where self.vec.version() } - fn len(&self) -> usize { + pub fn len(&self) -> usize { self.vec.len() } @@ -98,6 +98,10 @@ where &self.vec } + pub fn mut_any_vec(&mut self) -> &mut dyn AnyStorableVec { + &mut self.vec + } + pub fn get(&mut self, index: I) -> Result> { self.vec.get(index) } diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 64448d708..a96c91475 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{CheckedSub, Height, Timestamp}; +use brk_core::{CheckedSub, Dateindex, Height, Timestamp}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; @@ -14,8 +14,8 @@ use super::{ pub struct Vecs { pub height_to_block_interval: StorableVec, pub indexes_to_block_interval_stats: StorableVecGeneatorByIndex, - pub dateindex_to_block_count: StorableVec, - pub dateindex_to_total_block_count: StorableVec, + pub dateindex_to_block_count: StorableVec, + pub dateindex_to_total_block_count: StorableVec, } impl Vecs { @@ -61,15 +61,15 @@ impl Vecs { self.height_to_block_interval.compute_transform( starting_indexes.height, indexer_vecs.height_to_timestamp.mut_vec(), - |(height, timestamp, height_to_timestamp, ..)| { + |(height, timestamp, _, height_to_timestamp)| { let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| { let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap(); - if prev_timestamp >= timestamp { - Timestamp::ZERO - } else { - timestamp.checked_sub(prev_timestamp).unwrap() - } + dbg!((timestamp, prev_timestamp)); + timestamp + .checked_sub(prev_timestamp) + .unwrap_or(Timestamp::ZERO) }); + dbg!((height, interval)); (height, interval) }, exit, @@ -77,10 +77,10 @@ impl Vecs { self.indexes_to_block_interval_stats.compute( &mut self.height_to_block_interval, - indexer, indexes, starting_indexes, - ); + exit, + )?; Ok(()) } diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index 963a4a0d9..82c99ce3c 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -1,6 +1,9 @@ use std::{fs, ops::Deref, path::Path}; -use brk_core::{Date, Dateindex, Height, Txindex, Txinindex, Txoutindex}; +use brk_core::{ + Date, Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Txindex, + Txinindex, Txoutindex, Weekindex, Yearindex, +}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; @@ -15,13 +18,37 @@ pub struct Vecs { pub dateindex_to_dateindex: StorableVec, pub dateindex_to_first_height: StorableVec, pub dateindex_to_last_height: StorableVec, + pub dateindex_to_monthindex: StorableVec, + pub dateindex_to_weekindex: StorableVec, + pub decadeindex_to_decadeindex: StorableVec, + pub decadeindex_to_first_yearindex: StorableVec, + pub decadeindex_to_last_yearindex: StorableVec, + pub difficultyepoch_to_difficultyepoch: StorableVec, + pub difficultyepoch_to_first_height: StorableVec, + pub difficultyepoch_to_last_height: StorableVec, + pub halvingepoch_to_first_height: StorableVec, + pub halvingepoch_to_halvingepoch: StorableVec, + pub halvingepoch_to_last_height: StorableVec, pub height_to_dateindex: StorableVec, + pub height_to_difficultyepoch: StorableVec, pub height_to_fixed_date: StorableVec, + pub height_to_halvingepoch: StorableVec, pub height_to_height: StorableVec, pub height_to_last_txindex: StorableVec, pub height_to_real_date: StorableVec, + pub monthindex_to_first_dateindex: StorableVec, + pub monthindex_to_last_dateindex: StorableVec, + pub monthindex_to_monthindex: StorableVec, + pub monthindex_to_yearindex: StorableVec, pub txindex_to_last_txinindex: StorableVec, pub txindex_to_last_txoutindex: StorableVec, + pub weekindex_to_first_dateindex: StorableVec, + pub weekindex_to_last_dateindex: StorableVec, + pub weekindex_to_weekindex: StorableVec, + pub yearindex_to_decadeindex: StorableVec, + pub yearindex_to_first_monthindex: StorableVec, + pub yearindex_to_last_monthindex: StorableVec, + pub yearindex_to_yearindex: StorableVec, } impl Vecs { @@ -74,7 +101,6 @@ impl Vecs { Version::from(1), compressed, )?, - txindex_to_last_txinindex: StorableVec::forced_import( &path.join("txindex_to_last_txinindex"), Version::from(1), @@ -85,6 +111,126 @@ impl Vecs { Version::from(1), compressed, )?, + difficultyepoch_to_first_height: StorableVec::forced_import( + &path.join("difficultyepoch_to_first_height"), + Version::from(1), + compressed, + )?, + difficultyepoch_to_last_height: StorableVec::forced_import( + &path.join("difficultyepoch_to_last_height"), + Version::from(1), + compressed, + )?, + halvingepoch_to_first_height: StorableVec::forced_import( + &path.join("halvingepoch_to_first_height"), + Version::from(1), + compressed, + )?, + halvingepoch_to_last_height: StorableVec::forced_import( + &path.join("halvingepoch_to_last_height"), + Version::from(1), + compressed, + )?, + weekindex_to_first_dateindex: StorableVec::forced_import( + &path.join("weekindex_to_first_dateindex"), + Version::from(1), + compressed, + )?, + weekindex_to_last_dateindex: StorableVec::forced_import( + &path.join("weekindex_to_last_dateindex"), + Version::from(1), + compressed, + )?, + monthindex_to_first_dateindex: StorableVec::forced_import( + &path.join("monthindex_to_first_dateindex"), + Version::from(1), + compressed, + )?, + monthindex_to_last_dateindex: StorableVec::forced_import( + &path.join("monthindex_to_last_dateindex"), + Version::from(1), + compressed, + )?, + yearindex_to_first_monthindex: StorableVec::forced_import( + &path.join("yearindex_to_first_monthindex"), + Version::from(1), + compressed, + )?, + yearindex_to_last_monthindex: StorableVec::forced_import( + &path.join("yearindex_to_last_monthindex"), + Version::from(1), + compressed, + )?, + decadeindex_to_first_yearindex: StorableVec::forced_import( + &path.join("decadeindex_to_first_yearindex"), + Version::from(1), + compressed, + )?, + decadeindex_to_last_yearindex: StorableVec::forced_import( + &path.join("decadeindex_to_last_yearindex"), + Version::from(1), + compressed, + )?, + dateindex_to_weekindex: StorableVec::forced_import( + &path.join("dateindex_to_weekindex"), + Version::from(1), + compressed, + )?, + dateindex_to_monthindex: StorableVec::forced_import( + &path.join("dateindex_to_monthindex"), + Version::from(1), + compressed, + )?, + monthindex_to_yearindex: StorableVec::forced_import( + &path.join("monthindex_to_yearindex"), + Version::from(1), + compressed, + )?, + yearindex_to_decadeindex: StorableVec::forced_import( + &path.join("yearindex_to_decadeindex"), + Version::from(1), + compressed, + )?, + height_to_difficultyepoch: StorableVec::forced_import( + &path.join("height_to_difficultyepoch"), + Version::from(1), + compressed, + )?, + height_to_halvingepoch: StorableVec::forced_import( + &path.join("height_to_halvingepoch"), + Version::from(1), + compressed, + )?, + weekindex_to_weekindex: StorableVec::forced_import( + &path.join("weekindex_to_weekindex"), + Version::from(1), + compressed, + )?, + monthindex_to_monthindex: StorableVec::forced_import( + &path.join("monthindex_to_monthindex"), + Version::from(1), + compressed, + )?, + yearindex_to_yearindex: StorableVec::forced_import( + &path.join("yearindex_to_yearindex"), + Version::from(1), + compressed, + )?, + decadeindex_to_decadeindex: StorableVec::forced_import( + &path.join("decadeindex_to_decadeindex"), + Version::from(1), + compressed, + )?, + difficultyepoch_to_difficultyepoch: StorableVec::forced_import( + &path.join("difficultyepoch_to_difficultyepoch"), + Version::from(1), + compressed, + )?, + halvingepoch_to_halvingepoch: StorableVec::forced_import( + &path.join("halvingepoch_to_halvingepoch"), + Version::from(1), + compressed, + )?, }) } @@ -132,6 +278,12 @@ impl Vecs { exit, )?; + let starting_dateindex = self + .height_to_dateindex + .get(starting_indexes.height.decremented().unwrap_or_default())? + .copied() + .unwrap_or_default(); + self.height_to_dateindex.compute_transform( starting_indexes.height, self.height_to_fixed_date.mut_vec(), @@ -139,11 +291,15 @@ impl Vecs { exit, )?; - let starting_dateindex = self + let starting_dateindex = if let Some(dateindex) = self .height_to_dateindex .get(starting_indexes.height.decremented().unwrap_or_default())? .copied() - .unwrap_or_default(); + { + starting_dateindex.min(dateindex) + } else { + starting_dateindex + }; self.dateindex_to_first_height .compute_inverse_more_to_less( @@ -152,6 +308,8 @@ impl Vecs { exit, )?; + let date_count = self.dateindex_to_first_height.len(); + self.dateindex_to_last_height .compute_last_index_from_first( starting_dateindex, @@ -197,7 +355,242 @@ impl Vecs { exit, )?; - Ok(Indexes::from((starting_indexes, starting_dateindex))) + // --- + + let starting_weekindex = self + .dateindex_to_weekindex + .get(starting_dateindex)? + .copied() + .unwrap_or_default(); + + self.dateindex_to_weekindex.compute_transform( + starting_dateindex, + self.dateindex_to_dateindex.mut_vec(), + |(di, ..)| (di, Weekindex::from(di)), + exit, + )?; + + self.weekindex_to_first_dateindex + .compute_inverse_more_to_less( + starting_dateindex, + self.dateindex_to_weekindex.mut_vec(), + exit, + )?; + + self.weekindex_to_last_dateindex + .compute_last_index_from_first( + starting_weekindex, + self.weekindex_to_first_dateindex.mut_vec(), + date_count, + exit, + )?; + + self.weekindex_to_weekindex.compute_transform( + starting_weekindex, + self.weekindex_to_first_dateindex.mut_vec(), + |(wi, ..)| (wi, wi), + exit, + )?; + + // --- + + let starting_monthindex = self + .dateindex_to_monthindex + .get(starting_dateindex)? + .copied() + .unwrap_or_default(); + + self.dateindex_to_monthindex.compute_transform( + starting_dateindex, + self.dateindex_to_dateindex.mut_vec(), + |(di, ..)| (di, Monthindex::from(di)), + exit, + )?; + + self.monthindex_to_first_dateindex + .compute_inverse_more_to_less( + starting_dateindex, + self.dateindex_to_monthindex.mut_vec(), + exit, + )?; + + let month_count = self.monthindex_to_first_dateindex.len(); + + self.monthindex_to_last_dateindex + .compute_last_index_from_first( + starting_monthindex, + self.monthindex_to_first_dateindex.mut_vec(), + date_count, + exit, + )?; + + self.monthindex_to_monthindex.compute_transform( + starting_monthindex, + self.monthindex_to_first_dateindex.mut_vec(), + |(mi, ..)| (mi, mi), + exit, + )?; + + // --- + + let starting_yearindex = self + .monthindex_to_yearindex + .get(starting_monthindex)? + .copied() + .unwrap_or_default(); + + self.monthindex_to_yearindex.compute_transform( + starting_monthindex, + self.monthindex_to_monthindex.mut_vec(), + |(mi, ..)| (mi, Yearindex::from(mi)), + exit, + )?; + + self.yearindex_to_first_monthindex + .compute_inverse_more_to_less( + starting_monthindex, + self.monthindex_to_yearindex.mut_vec(), + exit, + )?; + + let year_count = self.yearindex_to_first_monthindex.len(); + + self.yearindex_to_last_monthindex + .compute_last_index_from_first( + starting_yearindex, + self.yearindex_to_first_monthindex.mut_vec(), + month_count, + exit, + )?; + + self.yearindex_to_yearindex.compute_transform( + starting_yearindex, + self.yearindex_to_first_monthindex.mut_vec(), + |(yi, ..)| (yi, yi), + exit, + )?; + + // --- + + let starting_decadeindex = self + .yearindex_to_decadeindex + .get(starting_yearindex)? + .copied() + .unwrap_or_default(); + + self.yearindex_to_decadeindex.compute_transform( + starting_yearindex, + self.yearindex_to_yearindex.mut_vec(), + |(yi, ..)| (yi, Decadeindex::from(yi)), + exit, + )?; + + self.decadeindex_to_first_yearindex + .compute_inverse_more_to_less( + starting_yearindex, + self.yearindex_to_decadeindex.mut_vec(), + exit, + )?; + + self.decadeindex_to_last_yearindex + .compute_last_index_from_first( + starting_decadeindex, + self.decadeindex_to_first_yearindex.mut_vec(), + year_count, + exit, + )?; + + self.decadeindex_to_decadeindex.compute_transform( + starting_decadeindex, + self.decadeindex_to_first_yearindex.mut_vec(), + |(di, ..)| (di, di), + exit, + )?; + + // --- + + let starting_difficultyepoch = self + .height_to_difficultyepoch + .get(starting_indexes.height)? + .copied() + .unwrap_or_default(); + + self.height_to_difficultyepoch.compute_transform( + starting_indexes.height, + self.height_to_height.mut_vec(), + |(h, ..)| (h, Difficultyepoch::from(h)), + exit, + )?; + + self.difficultyepoch_to_first_height + .compute_inverse_more_to_less( + starting_indexes.height, + self.height_to_difficultyepoch.mut_vec(), + exit, + )?; + + self.difficultyepoch_to_last_height + .compute_last_index_from_first( + starting_difficultyepoch, + self.difficultyepoch_to_first_height.mut_vec(), + height_count, + exit, + )?; + + self.difficultyepoch_to_difficultyepoch.compute_transform( + starting_difficultyepoch, + self.difficultyepoch_to_first_height.mut_vec(), + |(de, ..)| (de, de), + exit, + )?; + + // --- + + let starting_halvingepoch = self + .height_to_halvingepoch + .get(starting_indexes.height)? + .copied() + .unwrap_or_default(); + + self.height_to_halvingepoch.compute_transform( + starting_indexes.height, + self.height_to_height.mut_vec(), + |(h, ..)| (h, Halvingepoch::from(h)), + exit, + )?; + + self.halvingepoch_to_first_height + .compute_inverse_more_to_less( + starting_indexes.height, + self.height_to_halvingepoch.mut_vec(), + exit, + )?; + + self.halvingepoch_to_last_height + .compute_last_index_from_first( + starting_halvingepoch, + self.halvingepoch_to_first_height.mut_vec(), + height_count, + exit, + )?; + + self.halvingepoch_to_halvingepoch.compute_transform( + starting_halvingepoch, + self.halvingepoch_to_first_height.mut_vec(), + |(he, ..)| (he, he), + exit, + )?; + + Ok(Indexes { + indexes: starting_indexes, + dateindex: starting_dateindex, + weekindex: starting_weekindex, + monthindex: starting_monthindex, + yearindex: starting_yearindex, + decadeindex: starting_decadeindex, + difficultyepoch: starting_difficultyepoch, + halvingepoch: starting_halvingepoch, + }) } pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { @@ -213,6 +606,30 @@ impl Vecs { self.height_to_real_date.any_vec(), self.txindex_to_last_txinindex.any_vec(), self.txindex_to_last_txoutindex.any_vec(), + self.difficultyepoch_to_first_height.any_vec(), + self.difficultyepoch_to_last_height.any_vec(), + self.halvingepoch_to_first_height.any_vec(), + self.halvingepoch_to_last_height.any_vec(), + self.weekindex_to_first_dateindex.any_vec(), + self.weekindex_to_last_dateindex.any_vec(), + self.monthindex_to_first_dateindex.any_vec(), + self.monthindex_to_last_dateindex.any_vec(), + self.yearindex_to_first_monthindex.any_vec(), + self.yearindex_to_last_monthindex.any_vec(), + self.decadeindex_to_first_yearindex.any_vec(), + self.decadeindex_to_last_yearindex.any_vec(), + self.dateindex_to_weekindex.any_vec(), + self.dateindex_to_monthindex.any_vec(), + self.monthindex_to_yearindex.any_vec(), + self.yearindex_to_decadeindex.any_vec(), + self.height_to_difficultyepoch.any_vec(), + self.height_to_halvingepoch.any_vec(), + self.weekindex_to_weekindex.any_vec(), + self.monthindex_to_monthindex.any_vec(), + self.yearindex_to_yearindex.any_vec(), + self.decadeindex_to_decadeindex.any_vec(), + self.difficultyepoch_to_difficultyepoch.any_vec(), + self.halvingepoch_to_halvingepoch.any_vec(), ] } } @@ -220,6 +637,12 @@ impl Vecs { pub struct Indexes { indexes: brk_indexer::Indexes, pub dateindex: Dateindex, + pub weekindex: Weekindex, + pub monthindex: Monthindex, + pub yearindex: Yearindex, + pub decadeindex: Decadeindex, + pub difficultyepoch: Difficultyepoch, + pub halvingepoch: Halvingepoch, } impl Deref for Indexes { @@ -228,8 +651,3 @@ impl Deref for Indexes { &self.indexes } } -impl From<(brk_indexer::Indexes, Dateindex)> for Indexes { - fn from((indexes, dateindex): (brk_indexer::Indexes, Dateindex)) -> Self { - Self { indexes, dateindex } - } -} diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index 2b2418792..e873110cf 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -66,6 +66,7 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ self.indexes.as_any_vecs(), + self.blocks.as_any_vecs(), self.transactions.as_any_vecs(), self.marketprice .as_ref() diff --git a/crates/brk_computer/src/storage/vecs/stats/generic.rs b/crates/brk_computer/src/storage/vecs/stats/generic.rs index d80cc02c9..beb0ef7be 100644 --- a/crates/brk_computer/src/storage/vecs/stats/generic.rs +++ b/crates/brk_computer/src/storage/vecs/stats/generic.rs @@ -1,19 +1,19 @@ use std::path::Path; use brk_exit::Exit; -use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, Version}; +use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, StoredType, Version}; use crate::storage::vecs::base::StorableVec; -use super::StoredType; +use super::ComputedType; #[derive(Clone, Debug)] -pub struct StorableVecGeneator +pub struct StorableVecBuilder where I: StoredIndex, - T: StoredType, + T: ComputedType, { - // first: Option>, + first: Option>, average: Option>, sum: Option>, max: Option>, @@ -26,10 +26,10 @@ where last: Option>, } -impl StorableVecGeneator +impl StorableVecBuilder where I: StoredIndex, - T: StoredType, + T: ComputedType, { pub fn forced_import( path: &Path, @@ -37,17 +37,23 @@ where options: StorableVecGeneatorOptions, ) -> color_eyre::Result { let name = path.file_name().unwrap().to_str().unwrap().to_string(); + let key = I::to_string().split("::").last().unwrap().to_lowercase(); - // let prefix = |s: &str| path.with_file_name(format!("{s}_{name}")); - let suffix = |s: &str| path.with_file_name(format!("{name}_{s}")); + let prefix = |s: &str| path.with_file_name(format!("{key}_to_{s}_{name}")); + let suffix = |s: &str| path.with_file_name(format!("{key}_to_{name}_{s}")); let s = Self { - // first: options.first.then(|| { - // StorableVec::forced_import(&prefix("first"), Version::from(1), compressed).unwrap() - // }), - last: options - .last - .then(|| StorableVec::forced_import(path, Version::from(1), compressed).unwrap()), + first: options.first.then(|| { + StorableVec::forced_import(&prefix("first"), Version::from(1), compressed).unwrap() + }), + last: options.last.then(|| { + StorableVec::forced_import( + &path.with_file_name(format!("{key}_to_{name}")), + Version::from(1), + compressed, + ) + .unwrap() + }), min: options.min.then(|| { StorableVec::forced_import(&suffix("min"), Version::from(1), compressed).unwrap() }), @@ -91,7 +97,8 @@ where ) -> Result<()> where I2: StoredIndex + StoredType, - T: Ord, + T: Ord + From, + f64: From, { let index = self.starting_index(max_from); @@ -99,10 +106,10 @@ where let first_index = *first_index; let last_index = *last_indexes.get(i).unwrap().unwrap(); - // if let Some(first) = self.first.as_mut() { - // let v = source.read(first_index).unwrap().unwrap(); - // first.forced_push_at(index, v.clone(), exit); - // } + if let Some(first) = self.first.as_mut() { + let v = source.get(first_index).unwrap().unwrap(); + first.forced_push_at(index, v.clone(), exit)?; + } if let Some(last) = self.last.as_mut() { let v = source.get(last_index).unwrap().unwrap(); @@ -162,11 +169,13 @@ where let len = values.len(); if let Some(average) = self.average.as_mut() { - let a = values + let len = len as f64; + let total = values .iter() - .map(|v| v.clone() / len) - .fold(T::from(0), |a, b| a + b); - average.forced_push_at(i, a, exit)?; + .map(|v| f64::from(v.clone())) + .fold(0.0, |a, b| a + b); + let avg = T::from(total / len); + average.forced_push_at(i, avg, exit)?; } if let Some(sum_vec) = self.sum.as_mut() { @@ -178,6 +187,127 @@ where Ok(()) })?; + + self.safe_flush(exit)?; + + Ok(()) + } + + pub fn from_aligned( + &mut self, + max_from: I, + source: &mut StorableVecBuilder, + first_indexes: &mut brk_vec::StorableVec, + last_indexes: &mut brk_vec::StorableVec, + exit: &Exit, + ) -> Result<()> + where + I2: StoredIndex + StoredType, + T: Ord + From, + f64: From, + { + if self._90p.is_some() + || self._75p.is_some() + || self.median.is_some() + || self._25p.is_some() + || self._10p.is_some() + { + panic!("unsupported"); + } + + let index = self.starting_index(max_from); + + first_indexes.iter_from(index, |(i, first_index)| { + let first_index = *first_index; + let last_index = *last_indexes.get(i).unwrap().unwrap(); + + if let Some(first) = self.first.as_mut() { + let v = source + .first + .as_mut() + .unwrap() + .get(last_index) + .unwrap() + .cloned() + .unwrap(); + first.forced_push_at(index, v, exit)?; + } + + if let Some(last) = self.last.as_mut() { + let v = source + .last + .as_mut() + .unwrap() + .get(last_index) + .unwrap() + .cloned() + .unwrap(); + last.forced_push_at(index, v, exit)?; + } + + let first_index = Some(first_index.to_usize()? as i64); + let last_index = Some(last_index.to_usize()? as i64); + + let needs_sum_or_average = self.sum.is_some() || self.average.is_some(); + let needs_sorted = self.max.is_some() || self.min.is_some(); + let needs_values = needs_sorted || needs_sum_or_average; + + if needs_values { + if needs_sorted { + if let Some(max) = self.max.as_mut() { + let mut values = source + .max + .as_ref() + .unwrap() + .collect_range(first_index, last_index)?; + values.sort_unstable(); + max.forced_push_at(i, values.last().unwrap().clone(), exit)?; + } + + if let Some(min) = self.min.as_mut() { + let mut values = source + .min + .as_ref() + .unwrap() + .collect_range(first_index, last_index)?; + values.sort_unstable(); + min.forced_push_at(i, values.first().unwrap().clone(), exit)?; + } + } + + if needs_sum_or_average { + if let Some(average) = self.average.as_mut() { + let values = source + .average + .as_ref() + .unwrap() + .collect_range(first_index, last_index)?; + let len = values.len() as f64; + let total = values + .into_iter() + .map(|v| f64::from(v)) + .fold(0.0, |a, b| a + b); + let avg = T::from(total / len); + average.forced_push_at(i, avg, exit)?; + } + + if let Some(sum_vec) = self.sum.as_mut() { + let values = source + .sum + .as_ref() + .unwrap() + .collect_range(first_index, last_index)?; + let sum = values.into_iter().fold(T::from(0), |a, b| a + b); + sum_vec.forced_push_at(i, sum, exit)?; + } + } + } + + Ok(()) + })?; + + self.safe_flush(exit)?; + Ok(()) } @@ -216,9 +346,9 @@ where pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { let mut v: Vec<&dyn AnyStorableVec> = vec![]; - // if let Some(first) = self.first.as_ref() { - // v.push(first.any_vec()); - // } + if let Some(first) = self.first.as_ref() { + v.push(first.any_vec()); + } if let Some(last) = self.last.as_ref() { v.push(last.any_vec()); } @@ -252,6 +382,44 @@ where v } + + pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> { + if let Some(first) = self.first.as_mut() { + first.safe_flush(exit)?; + } + if let Some(last) = self.last.as_mut() { + last.safe_flush(exit)?; + } + if let Some(min) = self.min.as_mut() { + min.safe_flush(exit)?; + } + if let Some(max) = self.max.as_mut() { + max.safe_flush(exit)?; + } + if let Some(median) = self.median.as_mut() { + median.safe_flush(exit)?; + } + if let Some(average) = self.average.as_mut() { + average.safe_flush(exit)?; + } + if let Some(sum) = self.sum.as_mut() { + sum.safe_flush(exit)?; + } + if let Some(_90p) = self._90p.as_mut() { + _90p.safe_flush(exit)?; + } + if let Some(_75p) = self._75p.as_mut() { + _75p.safe_flush(exit)?; + } + if let Some(_25p) = self._25p.as_mut() { + _25p.safe_flush(exit)?; + } + if let Some(_10p) = self._10p.as_mut() { + _10p.safe_flush(exit)?; + } + + Ok(()) + } } #[derive(Default, Clone, Copy)] @@ -265,15 +433,15 @@ pub struct StorableVecGeneatorOptions { _25p: bool, _10p: bool, min: bool, - // first: bool, + first: bool, last: bool, } impl StorableVecGeneatorOptions { - // pub fn add_first(mut self) -> Self { - // self.first = true; - // self - // } + pub fn add_first(mut self) -> Self { + self.first = true; + self + } pub fn add_last(mut self) -> Self { self.last = true; diff --git a/crates/brk_computer/src/storage/vecs/stats/grouped.rs b/crates/brk_computer/src/storage/vecs/stats/grouped.rs index 12876ff79..c5d7cfb52 100644 --- a/crates/brk_computer/src/storage/vecs/stats/grouped.rs +++ b/crates/brk_computer/src/storage/vecs/stats/grouped.rs @@ -1,60 +1,108 @@ use std::path::Path; -use brk_core::{ - Dateindex, Decadeindex, Difficultyepoch, Halvingepoch, Height, Monthindex, Weekindex, Yearindex, -}; -use brk_indexer::{Indexer, Indexes}; +use brk_core::{Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Weekindex, Yearindex}; +use brk_exit::Exit; use brk_vec::{AnyStorableVec, Compressed}; -use crate::storage::vecs::{base::StorableVec, indexes}; +use crate::storage::vecs::{Indexes, base::StorableVec, indexes}; -use super::{StorableVecGeneator, StorableVecGeneatorOptions, StoredType}; +use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] pub struct StorableVecGeneatorByIndex where - T: StoredType, + T: ComputedType + Ord, { - pub dateindex: StorableVecGeneator, - pub weekindex: StorableVecGeneator, - pub difficultyepoch: StorableVecGeneator, - pub monthindex: StorableVecGeneator, - pub yearindex: StorableVecGeneator, - pub halvingepoch: StorableVecGeneator, - pub decadeindex: StorableVecGeneator, + pub dateindex: StorableVecBuilder, + pub weekindex: StorableVecBuilder, + pub difficultyepoch: StorableVecBuilder, + pub monthindex: StorableVecBuilder, + pub yearindex: StorableVecBuilder, + // pub halvingepoch: StorableVecGeneator, // TODO + pub decadeindex: StorableVecBuilder, } impl StorableVecGeneatorByIndex where - T: StoredType, + T: ComputedType + Ord + From, + f64: From, { pub fn forced_import( path: &Path, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let dateindex = StorableVecGeneator::forced_import(path, compressed, options)?; + let dateindex = StorableVecBuilder::forced_import(path, compressed, options)?; let options = options.remove_percentiles(); Ok(Self { dateindex, - weekindex: StorableVecGeneator::forced_import(path, compressed, options)?, - difficultyepoch: StorableVecGeneator::forced_import(path, compressed, options)?, - monthindex: StorableVecGeneator::forced_import(path, compressed, options)?, - yearindex: StorableVecGeneator::forced_import(path, compressed, options)?, - halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?, - decadeindex: StorableVecGeneator::forced_import(path, compressed, options)?, + weekindex: StorableVecBuilder::forced_import(path, compressed, options)?, + difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?, + monthindex: StorableVecBuilder::forced_import(path, compressed, options)?, + yearindex: StorableVecBuilder::forced_import(path, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?, + decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?, }) } pub fn compute( &mut self, source: &mut StorableVec, - indexer: &mut Indexer, indexes: &mut indexes::Vecs, starting_indexes: &Indexes, - ) { + exit: &Exit, + ) -> color_eyre::Result<()> { + self.dateindex.compute( + starting_indexes.dateindex, + source, + indexes.dateindex_to_first_height.mut_vec(), + indexes.dateindex_to_last_height.mut_vec(), + exit, + )?; + + self.weekindex.from_aligned( + starting_indexes.weekindex, + &mut self.dateindex, + indexes.weekindex_to_first_dateindex.mut_vec(), + indexes.weekindex_to_last_dateindex.mut_vec(), + exit, + )?; + + self.monthindex.from_aligned( + starting_indexes.monthindex, + &mut self.dateindex, + indexes.monthindex_to_first_dateindex.mut_vec(), + indexes.monthindex_to_last_dateindex.mut_vec(), + exit, + )?; + + self.yearindex.from_aligned( + starting_indexes.yearindex, + &mut self.monthindex, + indexes.yearindex_to_first_monthindex.mut_vec(), + indexes.yearindex_to_last_monthindex.mut_vec(), + exit, + )?; + + self.decadeindex.from_aligned( + starting_indexes.decadeindex, + &mut self.yearindex, + indexes.decadeindex_to_first_yearindex.mut_vec(), + indexes.decadeindex_to_last_yearindex.mut_vec(), + exit, + )?; + + self.difficultyepoch.compute( + starting_indexes.difficultyepoch, + source, + indexes.difficultyepoch_to_first_height.mut_vec(), + indexes.difficultyepoch_to_last_height.mut_vec(), + exit, + )?; + + Ok(()) } pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { @@ -64,7 +112,7 @@ where self.difficultyepoch.as_any_vecs(), self.monthindex.as_any_vecs(), self.yearindex.as_any_vecs(), - self.halvingepoch.as_any_vecs(), + // self.halvingepoch.as_any_vecs(), self.decadeindex.as_any_vecs(), ] .concat() diff --git a/crates/brk_computer/src/storage/vecs/stats/stored_type.rs b/crates/brk_computer/src/storage/vecs/stats/stored_type.rs index 9061d2c92..95d41ef0b 100644 --- a/crates/brk_computer/src/storage/vecs/stats/stored_type.rs +++ b/crates/brk_computer/src/storage/vecs/stats/stored_type.rs @@ -1,11 +1,13 @@ use std::ops::{Add, Div}; -pub trait StoredType +use brk_vec::StoredType; + +pub trait ComputedType where - Self: brk_vec::StoredType + From + Div + Add, + Self: StoredType + From + Div + Add, { } -impl StoredType for T where - T: brk_vec::StoredType + From + Div + Add +impl ComputedType for T where + T: StoredType + From + Div + Add { } diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs index 3322e6c17..dfda5d490 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -4,7 +4,7 @@ use serde::Serialize; // use color_eyre::eyre::eyre; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::Error; +use crate::{CheckedSub, Error}; use super::Date; @@ -71,3 +71,9 @@ impl TryFrom for Dateindex { } } } + +impl CheckedSub for Dateindex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} diff --git a/crates/brk_core/src/structs/decadeindex.rs b/crates/brk_core/src/structs/decadeindex.rs index cb623f00b..af36005a1 100644 --- a/crates/brk_core/src/structs/decadeindex.rs +++ b/crates/brk_core/src/structs/decadeindex.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::{Date, Dateindex, Yearindex}; + #[derive( Debug, Clone, @@ -46,3 +50,36 @@ impl Add for Decadeindex { Self::from(self.0 + rhs as u8) } } + +impl From for Decadeindex { + fn from(value: Dateindex) -> Self { + Self::from(Date::from(value)) + } +} + +impl From for Decadeindex { + fn from(value: Date) -> Self { + let year = value.year(); + if year < 2000 { + panic!("unsupported") + } + Self(((year - 2000) / 10) as u8) + } +} + +impl CheckedSub for Decadeindex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +impl From for Decadeindex { + fn from(value: Yearindex) -> Self { + let v = usize::from(value); + if v == 0 { + Self(0) + } else { + Self((((v - 1) / 10) + 1) as u8) + } + } +} diff --git a/crates/brk_core/src/structs/difficultyepoch.rs b/crates/brk_core/src/structs/difficultyepoch.rs index 13d68339f..9507fc723 100644 --- a/crates/brk_core/src/structs/difficultyepoch.rs +++ b/crates/brk_core/src/structs/difficultyepoch.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::Height; + #[derive( Debug, Clone, @@ -46,3 +50,15 @@ impl Add for Difficultyepoch { Self::from(self.0 + rhs as u16) } } + +impl From for Difficultyepoch { + fn from(value: Height) -> Self { + Self((u32::from(value) / 2016) as u16) + } +} + +impl CheckedSub for Difficultyepoch { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_core/src/structs/halvingepoch.rs index bebd5869c..7c98fc2ef 100644 --- a/crates/brk_core/src/structs/halvingepoch.rs +++ b/crates/brk_core/src/structs/halvingepoch.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::Height; + #[derive( Debug, Clone, @@ -46,3 +50,15 @@ impl Add for Halvingepoch { Self::from(self.0 + rhs as u8) } } + +impl From for Halvingepoch { + fn from(value: Height) -> Self { + Self((u32::from(value) / 210_000) as u8) + } +} + +impl CheckedSub for Halvingepoch { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index c4b85e8b7..5edc88e66 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -92,7 +92,7 @@ impl Add for Height { impl CheckedSub for Height { fn checked_sub(self, rhs: Self) -> Option { - self.0.checked_sub(rhs.0).map(Self::from) + self.0.checked_sub(rhs.0).map(Self) } } diff --git a/crates/brk_core/src/structs/monthindex.rs b/crates/brk_core/src/structs/monthindex.rs index 28fc53130..892300afe 100644 --- a/crates/brk_core/src/structs/monthindex.rs +++ b/crates/brk_core/src/structs/monthindex.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::{Date, Dateindex, Yearindex}; + #[derive( Debug, Clone, @@ -46,3 +50,21 @@ impl Add for Monthindex { Self::from(self.0 + rhs as u16) } } + +impl From for Monthindex { + fn from(value: Dateindex) -> Self { + Self::from(Date::from(value)) + } +} + +impl From for Monthindex { + fn from(value: Date) -> Self { + Self(u16::from(Yearindex::from(value)) * 12 + value.month() as u16 - 1) + } +} + +impl CheckedSub for Monthindex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index 77dd493de..26e82ec37 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -77,7 +77,7 @@ impl From for Timestamp { impl CheckedSub for Timestamp { fn checked_sub(self, rhs: Self) -> Option { - self.0.checked_sub(rhs.0).map(Self::from) + self.0.checked_sub(rhs.0).map(Self) } } @@ -94,3 +94,18 @@ impl Add for Timestamp { Self(self.0 + rhs.0) } } + +impl From for Timestamp { + fn from(value: f64) -> Self { + if value < 0.0 || value > u32::MAX as f64 { + panic!() + } + Self(value as u32) + } +} + +impl From for f64 { + fn from(value: Timestamp) -> Self { + value.0 as f64 + } +} diff --git a/crates/brk_core/src/structs/weekindex.rs b/crates/brk_core/src/structs/weekindex.rs index 1f3ef677d..0be61847e 100644 --- a/crates/brk_core/src/structs/weekindex.rs +++ b/crates/brk_core/src/structs/weekindex.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::{Date, Dateindex}; + #[derive( Debug, Clone, @@ -46,3 +50,39 @@ impl Add for Weekindex { Self::from(self.0 + rhs as u16) } } + +impl From for Weekindex { + fn from(value: Dateindex) -> Self { + Self::from(Date::from(value)) + } +} + +impl From for Weekindex { + fn from(value: Date) -> Self { + let date = jiff::civil::Date::from(value).iso_week_date(); + + let mut week: u16 = 0; + let mut year = 2009; + + while date.year() > year { + let d = jiff::civil::Date::new(year, 6, 6).unwrap(); + let i = d.iso_week_date(); + let w = i.weeks_in_year(); + // dbg!(d, w); + week += w as u16; + year += 1; + } + + week += date.week() as u16; + + week -= 1; + + Self(week) + } +} + +impl CheckedSub for Weekindex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} diff --git a/crates/brk_core/src/structs/yearindex.rs b/crates/brk_core/src/structs/yearindex.rs index 6da0584d9..e82ae04da 100644 --- a/crates/brk_core/src/structs/yearindex.rs +++ b/crates/brk_core/src/structs/yearindex.rs @@ -3,6 +3,10 @@ use std::{fmt::Debug, ops::Add}; use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::CheckedSub; + +use super::{Date, Dateindex, Monthindex}; + #[derive( Debug, Clone, @@ -46,3 +50,33 @@ impl Add for Yearindex { Self::from(self.0 + rhs as u8) } } + +impl From for Yearindex { + fn from(value: Dateindex) -> Self { + Self::from(Date::from(value)) + } +} + +impl From for Yearindex { + fn from(value: Date) -> Self { + Self((value.year() - 2009) as u8) + } +} + +impl From for u16 { + fn from(value: Yearindex) -> Self { + value.0 as u16 + } +} + +impl CheckedSub for Yearindex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} + +impl From for Yearindex { + fn from(value: Monthindex) -> Self { + Self((usize::from(value) / 12) as u8) + } +} diff --git a/crates/brk_query/src/index.rs b/crates/brk_query/src/index.rs index e8e0d492a..03a2d2238 100644 --- a/crates/brk_query/src/index.rs +++ b/crates/brk_query/src/index.rs @@ -17,10 +17,16 @@ pub enum Index { Txindex, Txinindex, Txoutindex, + Weekindex, + Monthindex, + Yearindex, + Decadeindex, + Difficultyepoch, + Halvingepoch, } impl Index { - pub fn all() -> [Self; 13] { + pub fn all() -> [Self; 19] { [ Self::Addressindex, Self::Dateindex, @@ -35,13 +41,19 @@ impl Index { Self::Txindex, Self::Txinindex, Self::Txoutindex, + Self::Weekindex, + Self::Monthindex, + Self::Yearindex, + Self::Decadeindex, + Self::Difficultyepoch, + Self::Halvingepoch, ] } pub fn possible_values(&self) -> &[&str] { // Always have the "correct" id at the end match self { - Self::Dateindex => &["d", "date", "dateindex"], + Self::Dateindex => &["d", "date", "di", "dateindex"], Self::Height => &["h", "height"], Self::Txindex => &["txi", "txindex"], Self::Txinindex => &["txini", "txinindex"], @@ -54,6 +66,12 @@ impl Index { Self::P2TRindex => &["p2tri", "p2trindex"], Self::P2WPKHindex => &["p2wpkhi", "p2wpkhindex"], Self::P2WSHindex => &["p2wshi", "p2wshindex"], + Self::Weekindex => &["w", "wi", "week", "weekindex"], + Self::Monthindex => &["m", "mi", "month", "monthindex"], + Self::Yearindex => &["y", "yi", "year", "yearindex"], + Self::Decadeindex => &["decade", "decadeindex"], + Self::Difficultyepoch => &["difficulty", "difficultyepoch"], + Self::Halvingepoch => &["halving", "halvingepoch"], } } @@ -82,6 +100,12 @@ impl TryFrom<&str> for Index { v if (Self::P2TRindex).possible_values().contains(&v) => Self::P2TRindex, v if (Self::P2WPKHindex).possible_values().contains(&v) => Self::P2WPKHindex, v if (Self::P2WSHindex).possible_values().contains(&v) => Self::P2WSHindex, + v if (Self::Weekindex).possible_values().contains(&v) => Self::Weekindex, + v if (Self::Monthindex).possible_values().contains(&v) => Self::Monthindex, + v if (Self::Yearindex).possible_values().contains(&v) => Self::Yearindex, + v if (Self::Decadeindex).possible_values().contains(&v) => Self::Decadeindex, + v if (Self::Difficultyepoch).possible_values().contains(&v) => Self::Difficultyepoch, + v if (Self::Halvingepoch).possible_values().contains(&v) => Self::Halvingepoch, _ => return Err(eyre!("Bad index")), }) } diff --git a/crates/brk_query/src/tree.rs b/crates/brk_query/src/tree.rs index b99afb004..193510ff1 100644 --- a/crates/brk_query/src/tree.rs +++ b/crates/brk_query/src/tree.rs @@ -28,7 +28,7 @@ impl<'a> VecIdToIndexToVec<'a> { }) .unwrap(); if split[0] != index.to_string().to_lowercase() { - dbg!(split[0], index.to_string()); + dbg!(&file_name, split[0], index.to_string()); panic!(); } let key = split[1].to_string().replace("_", "-"); diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 9dad5dce8..c8359118c 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -25,4 +25,4 @@ oxc = { version = "0.60.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.44.1", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } -zip = "2.4.1" +zip = "2.4.2" diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index 63bfb1432..6311810cd 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -73,7 +73,7 @@ where let base = Base::import(path, version, compressed)?; if *compressed { - let pages_meta = CompressedPagesMetadata::read(Self::path_pages_meta_(path).as_path())?; + let pages_meta = Self::read_pages_meta_(path)?; Ok(Self::Compressed { base, pages_meta }) } else { @@ -81,6 +81,13 @@ where } } + fn read_pages_meta(&self) -> Result { + Self::read_pages_meta_(self.path()) + } + fn read_pages_meta_(path: &Path) -> Result { + CompressedPagesMetadata::read(Self::path_pages_meta_(path).as_path()) + } + #[inline] pub fn get(&mut self, index: I) -> Result> { self.get_(index.to_usize()?) @@ -215,7 +222,25 @@ where let page_index = Self::index_to_page_index(index); if page.as_ref().is_none_or(|b| b.0 != page_index) { - let values = self.decode_page(page_index).unwrap(); + let pages_meta = match self { + Self::Raw { .. } => None, + Self::Compressed { .. } => Some(self.read_pages_meta().unwrap()), + }; + + let values = Self::decode_page_( + self.base() + .read_stored_length() + .unwrap() + .to_usize() + .unwrap(), + page_index, + &self.base().open_file().unwrap(), + pages_meta.as_ref(), + ) + .inspect_err(|_| { + dbg!(from, to); + }) + .unwrap(); page.replace((page_index, values)); } @@ -717,7 +742,7 @@ where } pub fn index_type_to_string(&self) -> &str { - std::any::type_name::() + I::to_string() } } @@ -787,6 +812,9 @@ impl Base { path.join("vec") } + pub fn read_stored_length(&self) -> Result { + Length::try_from(self.path_length().as_path()) + } fn write_stored_length(&self) -> io::Result<()> { self.stored_len.write(&self.path_length()) } diff --git a/crates/brk_vec/src/traits/stored_index.rs b/crates/brk_vec/src/traits/stored_index.rs index efb2450dd..a083b452f 100644 --- a/crates/brk_vec/src/traits/stored_index.rs +++ b/crates/brk_vec/src/traits/stored_index.rs @@ -19,6 +19,7 @@ where + Sync, { fn to_usize(self) -> Result; + fn to_string<'a>() -> &'a str; } impl StoredIndex for I where @@ -40,4 +41,9 @@ where fn to_usize(self) -> Result { self.try_into().map_err(|_| Error::FailedKeyTryIntoUsize) } + + #[inline] + fn to_string<'a>() -> &'a str { + std::any::type_name::() + } } diff --git a/update.sh b/update.sh new file mode 100755 index 000000000..4db3d1f09 --- /dev/null +++ b/update.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +rustup update +cargo upgrade --incompatible +cargo update