diff --git a/Cargo.lock b/Cargo.lock index 290c7ef3e..993b2d666 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -541,7 +541,6 @@ dependencies = [ "tokio", "tower-http", "tracing", - "tracing-subscriber", "zip", ] @@ -1413,9 +1412,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1713,16 +1712,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1778,12 +1767,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "3.5.0" @@ -2213,7 +2196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.8.0", + "indexmap 2.9.0", ] [[package]] @@ -2675,7 +2658,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.8.0", + "indexmap 2.9.0", "serde", "serde_derive", "serde_json", @@ -2750,9 +2733,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "smawk" @@ -2999,7 +2982,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap 2.8.0", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", @@ -3097,29 +3080,15 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ - "nu-ansi-term", "sharded-slab", - "smallvec", "thread_local", "tracing-core", - "tracing-log", ] [[package]] @@ -3284,28 +3253,6 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.61.0" @@ -3522,9 +3469,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "febbe83a485467affa75a75d28dc7494acd2f819e549536c47d46b3089b56164" +checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" dependencies = [ "aes", "arbitrary", @@ -3536,7 +3483,7 @@ dependencies = [ "flate2", "getrandom 0.3.2", "hmac", - "indexmap 2.8.0", + "indexmap 2.9.0", "lzma-rs", "memchr", "pbkdf2", diff --git a/Cargo.toml b/Cargo.toml index 33bdfc2e9..29482689d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,17 +48,12 @@ shared-version = true tag-name = "v{{version}}" [workspace.metadata.dist] -# The preferred dist version to use in CI (Cargo.toml SemVer syntax) cargo-dist-version = "0.28.0" -# CI backends to support ci = "github" -# The installers to generate for each app installers = [] -# Target platforms to build apps for (Rust target-triple syntax) targets = [ "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", - "x86_64-pc-windows-msvc", ] diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index 0d6b9d82d..cee0c60f3 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -9,17 +9,19 @@ use std::{ use brk_core::CheckedSub; use brk_exit::Exit; -use brk_vec::{AnyStorableVec, Compressed, Error, Result, StoredIndex, StoredType, Version}; +use brk_vec::{ + AnyStorableVec, Compressed, Error, Result, StorableVec, StoredIndex, StoredType, Version, +}; const FLUSH_EVERY: usize = 10_000; #[derive(Debug)] -pub struct StorableVec { +pub struct ComputedVec { computed_version: Option, - vec: brk_vec::StorableVec, + vec: StorableVec, } -impl StorableVec +impl ComputedVec where I: StoredIndex, T: StoredType, @@ -316,7 +318,7 @@ where } } -impl Clone for StorableVec +impl Clone for ComputedVec where I: StoredIndex, T: StoredType, diff --git a/crates/brk_computer/src/storage/vecs/blocks.rs b/crates/brk_computer/src/storage/vecs/blocks.rs index 875faca3a..916dd4ec8 100644 --- a/crates/brk_computer/src/storage/vecs/blocks.rs +++ b/crates/brk_computer/src/storage/vecs/blocks.rs @@ -1,21 +1,21 @@ use std::{fs, path::Path}; -use brk_core::{CheckedSub, Dateindex, Height, Timestamp}; +use brk_core::{CheckedSub, Dateindex, Timestamp}; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{ - Indexes, StorableVec, indexes, - stats::{StorableVecGeneatorOptions, StorableVecsStatsFromHeight}, + ComputedVec, Indexes, + grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions}, + indexes, }; #[derive(Clone)] pub struct Vecs { - pub height_to_block_interval: StorableVec, - pub indexes_to_block_interval_stats: StorableVecsStatsFromHeight, - pub dateindex_to_block_count: StorableVec, - pub dateindex_to_total_block_count: StorableVec, + pub indexes_to_block_interval: ComputedVecsFromHeight, + pub dateindex_to_block_count: ComputedVec, + pub dateindex_to_total_block_count: ComputedVec, } impl Vecs { @@ -23,25 +23,22 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - height_to_block_interval: StorableVec::forced_import( - &path.join("height_to_block_interval"), + indexes_to_block_interval: ComputedVecsFromHeight::forced_import( + path, + "block_interval", Version::ONE, compressed, - )?, - indexes_to_block_interval_stats: StorableVecsStatsFromHeight::forced_import( - &path.join("block_interval"), - compressed, StorableVecGeneatorOptions::default() .add_percentiles() .add_minmax() .add_average(), )?, - dateindex_to_block_count: StorableVec::forced_import( + dateindex_to_block_count: ComputedVec::forced_import( &path.join("dateindex_to_block_count"), Version::ONE, compressed, )?, - dateindex_to_total_block_count: StorableVec::forced_import( + dateindex_to_total_block_count: ComputedVec::forced_import( &path.join("dateindex_to_total_block_count"), Version::ONE, compressed, @@ -58,23 +55,23 @@ impl Vecs { ) -> color_eyre::Result<()> { let indexer_vecs = indexer.mut_vecs(); - self.height_to_block_interval.compute_transform( - starting_indexes.height, - indexer_vecs.height_to_timestamp.mut_vec(), - |(height, timestamp, _, height_to_timestamp)| { - let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| { - let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap(); - timestamp - .checked_sub(prev_timestamp) - .unwrap_or(Timestamp::ZERO) - }); - (height, interval) + self.indexes_to_block_interval.compute( + |v| { + v.compute_transform( + starting_indexes.height, + indexer_vecs.height_to_timestamp.mut_vec(), + |(height, timestamp, _, height_to_timestamp)| { + let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| { + let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap(); + timestamp + .checked_sub(prev_timestamp) + .unwrap_or(Timestamp::ZERO) + }); + (height, interval) + }, + exit, + ) }, - exit, - )?; - - self.indexes_to_block_interval_stats.compute( - &mut self.height_to_block_interval, indexes, starting_indexes, exit, @@ -86,11 +83,10 @@ impl Vecs { pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ vec![ - self.height_to_block_interval.any_vec(), self.dateindex_to_block_count.any_vec(), self.dateindex_to_total_block_count.any_vec(), ], - self.indexes_to_block_interval_stats.as_any_vecs(), + self.indexes_to_block_interval.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/stats/generic.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs similarity index 91% rename from crates/brk_computer/src/storage/vecs/stats/generic.rs rename to crates/brk_computer/src/storage/vecs/grouped/builder.rs index c3c9ba548..84c36a02f 100644 --- a/crates/brk_computer/src/storage/vecs/stats/generic.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -3,40 +3,41 @@ use std::path::Path; use brk_exit::Exit; use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, StoredType, Version}; -use crate::storage::vecs::base::StorableVec; +use crate::storage::vecs::base::ComputedVec; use super::ComputedType; #[derive(Clone, Debug)] -pub struct StorableVecBuilder +pub struct ComputedVecBuilder where I: StoredIndex, T: ComputedType, { - pub first: Option>, - pub average: Option>, - pub sum: Option>, - pub max: Option>, - pub _90p: Option>, - pub _75p: Option>, - pub median: Option>, - pub _25p: Option>, - pub _10p: Option>, - pub min: Option>, - pub last: Option>, + pub first: Option>, + pub average: Option>, + pub sum: Option>, + pub max: Option>, + pub _90p: Option>, + pub _75p: Option>, + pub median: Option>, + pub _25p: Option>, + pub _10p: Option>, + pub min: Option>, + pub last: Option>, } -impl StorableVecBuilder +impl ComputedVecBuilder where I: StoredIndex, T: ComputedType, { pub fn forced_import( path: &Path, + name: &str, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let name = path.file_name().unwrap().to_str().unwrap().to_string(); + // let name = path.file_name().unwrap().to_str().unwrap().to_string(); let key = I::to_string().split("::").last().unwrap().to_lowercase(); let only_one_active = options.is_only_one_active(); @@ -59,10 +60,10 @@ where let s = Self { first: options.first.then(|| { - StorableVec::forced_import(&prefix("first"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&prefix("first"), Version::ONE, compressed).unwrap() }), last: options.last.then(|| { - StorableVec::forced_import( + ComputedVec::forced_import( &path.with_file_name(format!("{key}_to_{name}")), Version::ONE, compressed, @@ -70,31 +71,31 @@ where .unwrap() }), min: options.min.then(|| { - StorableVec::forced_import(&suffix("min"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("min"), Version::ONE, compressed).unwrap() }), max: options.max.then(|| { - StorableVec::forced_import(&suffix("max"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("max"), Version::ONE, compressed).unwrap() }), median: options.median.then(|| { - StorableVec::forced_import(&suffix("median"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("median"), Version::ONE, compressed).unwrap() }), average: options.average.then(|| { - StorableVec::forced_import(&suffix("average"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("average"), Version::ONE, compressed).unwrap() }), sum: options.sum.then(|| { - StorableVec::forced_import(&suffix("sum"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("sum"), Version::ONE, compressed).unwrap() }), _90p: options._90p.then(|| { - StorableVec::forced_import(&suffix("90p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("90p"), Version::ONE, compressed).unwrap() }), _75p: options._75p.then(|| { - StorableVec::forced_import(&suffix("75p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("75p"), Version::ONE, compressed).unwrap() }), _25p: options._25p.then(|| { - StorableVec::forced_import(&suffix("25p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("25p"), Version::ONE, compressed).unwrap() }), _10p: options._10p.then(|| { - StorableVec::forced_import(&suffix("10p"), Version::ONE, compressed).unwrap() + ComputedVec::forced_import(&suffix("10p"), Version::ONE, compressed).unwrap() }), }; @@ -104,7 +105,7 @@ where pub fn compute( &mut self, max_from: I, - source: &mut StorableVec, + source: &mut ComputedVec, first_indexes: &mut brk_vec::StorableVec, last_indexes: &mut brk_vec::StorableVec, exit: &Exit, @@ -211,7 +212,7 @@ where pub fn from_aligned( &mut self, max_from: I, - source: &mut StorableVecBuilder, + source: &mut ComputedVecBuilder, first_indexes: &mut brk_vec::StorableVec, last_indexes: &mut brk_vec::StorableVec, exit: &Exit, @@ -352,15 +353,11 @@ where fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( - self.as_any_vecs() - .into_iter() - .map(|v| v.len()) - .min() - .unwrap(), + self.any_vecs().into_iter().map(|v| v.len()).min().unwrap(), )) } - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { let mut v: Vec<&dyn AnyStorableVec> = vec![]; if let Some(first) = self.first.as_ref() { diff --git a/crates/brk_computer/src/storage/vecs/stats/from_date.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs similarity index 51% rename from crates/brk_computer/src/storage/vecs/stats/from_date.rs rename to crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index 20517e732..0e0bada1c 100644 --- a/crates/brk_computer/src/storage/vecs/stats/from_date.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -2,55 +2,68 @@ use std::path::Path; use brk_core::{Dateindex, Decadeindex, Monthindex, Quarterindex, Weekindex, Yearindex}; use brk_exit::Exit; -use brk_vec::{AnyStorableVec, Compressed}; +use brk_vec::{AnyStorableVec, Compressed, Result, Version}; -use crate::storage::vecs::{Indexes, base::StorableVec, indexes}; +use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; -use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] -pub struct StorableVecsStatsFromDate +pub struct ComputedVecsFromDateindex where T: ComputedType + PartialOrd, { - pub weekindex: StorableVecBuilder, - pub monthindex: StorableVecBuilder, - pub quarterindex: StorableVecBuilder, - pub yearindex: StorableVecBuilder, - pub decadeindex: StorableVecBuilder, + pub dateindex: ComputedVec, + pub weekindex: ComputedVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, + pub decadeindex: ComputedVecBuilder, } -impl StorableVecsStatsFromDate +impl ComputedVecsFromDateindex where T: ComputedType + Ord + From, f64: From, { pub fn forced_import( path: &Path, + name: &str, + version: Version, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { let options = options.remove_percentiles(); Ok(Self { - weekindex: StorableVecBuilder::forced_import(path, compressed, options)?, - monthindex: StorableVecBuilder::forced_import(path, compressed, options)?, - quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?, - yearindex: StorableVecBuilder::forced_import(path, compressed, options)?, - decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?, + dateindex: ComputedVec::forced_import( + &path.join(format!("dateindex_to_{name}")), + version, + compressed, + )?, + weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, }) } - pub fn compute( + pub fn compute( &mut self, - source: &mut StorableVec, + mut compute: F, indexes: &mut indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> color_eyre::Result<()> + where + F: FnMut(&mut ComputedVec) -> Result<()>, + { + compute(&mut self.dateindex)?; + self.weekindex.compute( starting_indexes.weekindex, - source, + &mut self.dateindex, indexes.weekindex_to_first_dateindex.mut_vec(), indexes.weekindex_to_last_dateindex.mut_vec(), exit, @@ -58,7 +71,7 @@ where self.monthindex.compute( starting_indexes.monthindex, - source, + &mut self.dateindex, indexes.monthindex_to_first_dateindex.mut_vec(), indexes.monthindex_to_last_dateindex.mut_vec(), exit, @@ -91,13 +104,14 @@ where Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ - self.weekindex.as_any_vecs(), - self.monthindex.as_any_vecs(), - self.quarterindex.as_any_vecs(), - self.yearindex.as_any_vecs(), - self.decadeindex.as_any_vecs(), + vec![self.dateindex.any_vec()], + self.weekindex.any_vecs(), + self.monthindex.any_vecs(), + self.quarterindex.any_vecs(), + self.yearindex.any_vecs(), + self.decadeindex.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/stats/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs similarity index 54% rename from crates/brk_computer/src/storage/vecs/stats/from_height.rs rename to crates/brk_computer/src/storage/vecs/grouped/from_height.rs index 8d9625a41..8131e94d8 100644 --- a/crates/brk_computer/src/storage/vecs/stats/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -4,63 +4,78 @@ use brk_core::{ Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Weekindex, Yearindex, }; use brk_exit::Exit; -use brk_vec::{AnyStorableVec, Compressed}; +use brk_vec::{AnyStorableVec, Compressed, Result, Version}; -use crate::storage::vecs::{Indexes, base::StorableVec, indexes}; +use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; -use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions}; +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; #[derive(Clone)] -pub struct StorableVecsStatsFromHeight +pub struct ComputedVecsFromHeight where T: ComputedType + PartialOrd, { - pub dateindex: StorableVecBuilder, - pub weekindex: StorableVecBuilder, - pub difficultyepoch: StorableVecBuilder, - pub monthindex: StorableVecBuilder, - pub quarterindex: StorableVecBuilder, - pub yearindex: StorableVecBuilder, + pub height: ComputedVec, + pub dateindex: ComputedVecBuilder, + pub weekindex: ComputedVecBuilder, + pub difficultyepoch: ComputedVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, // TODO: pub halvingepoch: StorableVecGeneator, - pub decadeindex: StorableVecBuilder, + pub decadeindex: ComputedVecBuilder, } -impl StorableVecsStatsFromHeight +impl ComputedVecsFromHeight where T: ComputedType + Ord + From, f64: From, { pub fn forced_import( path: &Path, + name: &str, + version: Version, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let dateindex = StorableVecBuilder::forced_import(path, compressed, options)?; + let height = ComputedVec::forced_import( + &path.join(format!("height_to_{name}")), + version, + compressed, + )?; + + let dateindex = ComputedVecBuilder::forced_import(path, name, compressed, options)?; let options = options.remove_percentiles(); Ok(Self { + height, dateindex, - weekindex: StorableVecBuilder::forced_import(path, compressed, options)?, - difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?, - monthindex: StorableVecBuilder::forced_import(path, compressed, options)?, - quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?, - yearindex: StorableVecBuilder::forced_import(path, compressed, options)?, - // halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?, - decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?, + weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, }) } - pub fn compute( + pub fn compute( &mut self, - source: &mut StorableVec, + mut compute: F, indexes: &mut indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, - ) -> color_eyre::Result<()> { + ) -> color_eyre::Result<()> + where + F: FnMut(&mut ComputedVec) -> Result<()>, + { + compute(&mut self.height)?; + self.dateindex.compute( starting_indexes.dateindex, - source, + &mut self.height, indexes.dateindex_to_first_height.mut_vec(), indexes.dateindex_to_last_height.mut_vec(), exit, @@ -108,7 +123,7 @@ where self.difficultyepoch.compute( starting_indexes.difficultyepoch, - source, + &mut self.height, indexes.difficultyepoch_to_first_height.mut_vec(), indexes.difficultyepoch_to_last_height.mut_vec(), exit, @@ -117,16 +132,17 @@ where Ok(()) } - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { [ - self.dateindex.as_any_vecs(), - self.weekindex.as_any_vecs(), - self.difficultyepoch.as_any_vecs(), - self.monthindex.as_any_vecs(), - self.quarterindex.as_any_vecs(), - self.yearindex.as_any_vecs(), + vec![self.height.any_vec()], + self.dateindex.any_vecs(), + self.weekindex.any_vecs(), + self.difficultyepoch.any_vecs(), + self.monthindex.any_vecs(), + self.quarterindex.any_vecs(), + self.yearindex.any_vecs(), // self.halvingepoch.as_any_vecs(), - self.decadeindex.as_any_vecs(), + self.decadeindex.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs new file mode 100644 index 000000000..aa0db04ab --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -0,0 +1,79 @@ +use std::path::Path; + +use brk_core::{Difficultyepoch, Height}; +use brk_exit::Exit; +use brk_vec::{AnyStorableVec, Compressed, Result, Version}; + +use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; + +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; + +#[derive(Clone)] +pub struct ComputedVecsFromHeightStrict +where + T: ComputedType + PartialOrd, +{ + pub height: ComputedVec, + pub difficultyepoch: ComputedVecBuilder, + // TODO: pub halvingepoch: StorableVecGeneator, +} + +impl ComputedVecsFromHeightStrict +where + T: ComputedType + Ord + From, + f64: From, +{ + pub fn forced_import( + path: &Path, + name: &str, + version: Version, + compressed: Compressed, + options: StorableVecGeneatorOptions, + ) -> color_eyre::Result { + let height = ComputedVec::forced_import( + &path.join(format!("height_to_{name}")), + version, + compressed, + )?; + + let options = options.remove_percentiles(); + + Ok(Self { + height, + difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, + }) + } + + pub fn compute( + &mut self, + mut compute: F, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> + where + F: FnMut(&mut ComputedVec) -> Result<()>, + { + compute(&mut self.height)?; + + self.difficultyepoch.compute( + starting_indexes.difficultyepoch, + &mut self.height, + indexes.difficultyepoch_to_first_height.mut_vec(), + indexes.difficultyepoch_to_last_height.mut_vec(), + exit, + )?; + + Ok(()) + } + + pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + [ + vec![self.height.any_vec()], + self.difficultyepoch.any_vecs(), + // self.halvingepoch.as_any_vecs(), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs new file mode 100644 index 000000000..2dbe7a03a --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -0,0 +1,164 @@ +use std::path::Path; + +use brk_core::{ + Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Txindex, Weekindex, + Yearindex, +}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStorableVec, Compressed, Result, Version}; + +use crate::storage::vecs::{Indexes, base::ComputedVec, indexes}; + +use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; + +#[derive(Clone)] +pub struct ComputedVecsFromTxindex +where + T: ComputedType + PartialOrd, +{ + pub txindex: ComputedVec, + pub height: ComputedVecBuilder, + pub dateindex: ComputedVecBuilder, + pub weekindex: ComputedVecBuilder, + pub difficultyepoch: ComputedVecBuilder, + pub monthindex: ComputedVecBuilder, + pub quarterindex: ComputedVecBuilder, + pub yearindex: ComputedVecBuilder, + // TODO: pub halvingepoch: StorableVecGeneator, + pub decadeindex: ComputedVecBuilder, +} + +impl ComputedVecsFromTxindex +where + T: ComputedType + Ord + From, + f64: From, +{ + pub fn forced_import( + path: &Path, + name: &str, + version: Version, + compressed: Compressed, + options: StorableVecGeneatorOptions, + ) -> color_eyre::Result { + let txindex = ComputedVec::forced_import( + &path.join(format!("txindex_to_{name}")), + version, + compressed, + )?; + + let height = ComputedVecBuilder::forced_import(path, name, compressed, options)?; + let dateindex = ComputedVecBuilder::forced_import(path, name, compressed, options)?; + + let options = options.remove_percentiles(); + + Ok(Self { + txindex, + height, + dateindex, + weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + }) + } + + pub fn compute( + &mut self, + mut compute: F, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> + where + F: FnMut(&mut ComputedVec) -> Result<()>, + { + compute(&mut self.txindex)?; + + self.height.compute( + starting_indexes.height, + &mut self.txindex, + indexer.mut_vecs().height_to_first_txindex.mut_vec(), + indexes.height_to_last_txindex.mut_vec(), + exit, + )?; + + self.dateindex.from_aligned( + starting_indexes.dateindex, + &mut self.height, + indexes.dateindex_to_first_height.mut_vec(), + indexes.dateindex_to_last_height.mut_vec(), + exit, + )?; + + self.weekindex.from_aligned( + starting_indexes.weekindex, + &mut self.dateindex, + indexes.weekindex_to_first_dateindex.mut_vec(), + indexes.weekindex_to_last_dateindex.mut_vec(), + exit, + )?; + + self.monthindex.from_aligned( + starting_indexes.monthindex, + &mut self.dateindex, + indexes.monthindex_to_first_dateindex.mut_vec(), + indexes.monthindex_to_last_dateindex.mut_vec(), + exit, + )?; + + self.quarterindex.from_aligned( + starting_indexes.quarterindex, + &mut self.monthindex, + indexes.quarterindex_to_first_monthindex.mut_vec(), + indexes.quarterindex_to_last_monthindex.mut_vec(), + exit, + )?; + + self.yearindex.from_aligned( + starting_indexes.yearindex, + &mut self.monthindex, + indexes.yearindex_to_first_monthindex.mut_vec(), + indexes.yearindex_to_last_monthindex.mut_vec(), + exit, + )?; + + self.decadeindex.from_aligned( + starting_indexes.decadeindex, + &mut self.yearindex, + indexes.decadeindex_to_first_yearindex.mut_vec(), + indexes.decadeindex_to_last_yearindex.mut_vec(), + exit, + )?; + + self.difficultyepoch.from_aligned( + starting_indexes.difficultyepoch, + &mut self.height, + indexes.difficultyepoch_to_first_height.mut_vec(), + indexes.difficultyepoch_to_last_height.mut_vec(), + exit, + )?; + + Ok(()) + } + + pub fn any_vecs(&self) -> Vec<&dyn AnyStorableVec> { + [ + vec![self.txindex.any_vec()], + self.height.any_vecs(), + self.dateindex.any_vecs(), + self.weekindex.any_vecs(), + self.difficultyepoch.any_vecs(), + self.monthindex.any_vecs(), + self.quarterindex.any_vecs(), + self.yearindex.any_vecs(), + // self.halvingepoch.as_any_vecs(), + self.decadeindex.any_vecs(), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/stats/mod.rs b/crates/brk_computer/src/storage/vecs/grouped/mod.rs similarity index 52% rename from crates/brk_computer/src/storage/vecs/stats/mod.rs rename to crates/brk_computer/src/storage/vecs/grouped/mod.rs index 359184006..3bf2f923b 100644 --- a/crates/brk_computer/src/storage/vecs/stats/mod.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/mod.rs @@ -1,11 +1,13 @@ -mod from_date; +mod builder; +mod from_dateindex; mod from_height; mod from_height_strict; -mod generic; +mod from_txindex; mod stored_type; -pub use from_date::*; +pub use builder::*; +pub use from_dateindex::*; pub use from_height::*; pub use from_height_strict::*; -pub use generic::*; +pub use from_txindex::*; pub use stored_type::*; diff --git a/crates/brk_computer/src/storage/vecs/stats/stored_type.rs b/crates/brk_computer/src/storage/vecs/grouped/stored_type.rs similarity index 100% rename from crates/brk_computer/src/storage/vecs/stats/stored_type.rs rename to crates/brk_computer/src/storage/vecs/grouped/stored_type.rs diff --git a/crates/brk_computer/src/storage/vecs/indexes.rs b/crates/brk_computer/src/storage/vecs/indexes.rs index c47303ede..b45c564e4 100644 --- a/crates/brk_computer/src/storage/vecs/indexes.rs +++ b/crates/brk_computer/src/storage/vecs/indexes.rs @@ -8,60 +8,60 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; -use super::StorableVec; +use super::ComputedVec; #[derive(Clone)] pub struct Vecs { // pub height_to_last_addressindex: StorableVec, // pub height_to_last_txoutindex: StorableVec, - pub dateindex_to_date: StorableVec, - pub dateindex_to_dateindex: StorableVec, - pub dateindex_to_first_height: StorableVec, - pub dateindex_to_last_height: StorableVec, - pub dateindex_to_monthindex: StorableVec, - pub dateindex_to_timestamp: StorableVec, - pub dateindex_to_weekindex: StorableVec, - pub decadeindex_to_decadeindex: StorableVec, - pub decadeindex_to_first_yearindex: StorableVec, - pub decadeindex_to_last_yearindex: StorableVec, - pub decadeindex_to_timestamp: StorableVec, - pub difficultyepoch_to_difficultyepoch: StorableVec, - pub difficultyepoch_to_first_height: StorableVec, - pub difficultyepoch_to_last_height: StorableVec, - pub difficultyepoch_to_timestamp: StorableVec, - pub halvingepoch_to_first_height: StorableVec, - pub halvingepoch_to_halvingepoch: StorableVec, - pub halvingepoch_to_last_height: StorableVec, - pub halvingepoch_to_timestamp: StorableVec, - pub height_to_dateindex: StorableVec, - pub height_to_difficultyepoch: StorableVec, - pub height_to_fixed_date: StorableVec, - pub height_to_fixed_timestamp: StorableVec, - pub height_to_halvingepoch: StorableVec, - pub height_to_height: StorableVec, - pub height_to_last_txindex: StorableVec, - pub height_to_real_date: StorableVec, - pub monthindex_to_first_dateindex: StorableVec, - pub monthindex_to_last_dateindex: StorableVec, - pub monthindex_to_monthindex: StorableVec, - pub monthindex_to_quarterindex: StorableVec, - pub monthindex_to_timestamp: StorableVec, - pub monthindex_to_yearindex: StorableVec, - pub quarterindex_to_first_monthindex: StorableVec, - pub quarterindex_to_last_monthindex: StorableVec, - pub quarterindex_to_quarterindex: StorableVec, - pub quarterindex_to_timestamp: StorableVec, - pub txindex_to_last_txinindex: StorableVec, - pub txindex_to_last_txoutindex: StorableVec, - pub weekindex_to_first_dateindex: StorableVec, - pub weekindex_to_last_dateindex: StorableVec, - pub weekindex_to_timestamp: StorableVec, - pub weekindex_to_weekindex: StorableVec, - pub yearindex_to_decadeindex: StorableVec, - pub yearindex_to_first_monthindex: StorableVec, - pub yearindex_to_last_monthindex: StorableVec, - pub yearindex_to_timestamp: StorableVec, - pub yearindex_to_yearindex: StorableVec, + pub dateindex_to_date: ComputedVec, + pub dateindex_to_dateindex: ComputedVec, + pub dateindex_to_first_height: ComputedVec, + pub dateindex_to_last_height: ComputedVec, + pub dateindex_to_monthindex: ComputedVec, + pub dateindex_to_timestamp: ComputedVec, + pub dateindex_to_weekindex: ComputedVec, + pub decadeindex_to_decadeindex: ComputedVec, + pub decadeindex_to_first_yearindex: ComputedVec, + pub decadeindex_to_last_yearindex: ComputedVec, + pub decadeindex_to_timestamp: ComputedVec, + pub difficultyepoch_to_difficultyepoch: ComputedVec, + pub difficultyepoch_to_first_height: ComputedVec, + pub difficultyepoch_to_last_height: ComputedVec, + pub difficultyepoch_to_timestamp: ComputedVec, + pub halvingepoch_to_first_height: ComputedVec, + pub halvingepoch_to_halvingepoch: ComputedVec, + pub halvingepoch_to_last_height: ComputedVec, + pub halvingepoch_to_timestamp: ComputedVec, + pub height_to_dateindex: ComputedVec, + pub height_to_difficultyepoch: ComputedVec, + pub height_to_fixed_date: ComputedVec, + pub height_to_fixed_timestamp: ComputedVec, + pub height_to_halvingepoch: ComputedVec, + pub height_to_height: ComputedVec, + pub height_to_last_txindex: ComputedVec, + pub height_to_real_date: ComputedVec, + pub monthindex_to_first_dateindex: ComputedVec, + pub monthindex_to_last_dateindex: ComputedVec, + pub monthindex_to_monthindex: ComputedVec, + pub monthindex_to_quarterindex: ComputedVec, + pub monthindex_to_timestamp: ComputedVec, + pub monthindex_to_yearindex: ComputedVec, + pub quarterindex_to_first_monthindex: ComputedVec, + pub quarterindex_to_last_monthindex: ComputedVec, + pub quarterindex_to_quarterindex: ComputedVec, + pub quarterindex_to_timestamp: ComputedVec, + pub txindex_to_last_txinindex: ComputedVec, + pub txindex_to_last_txoutindex: ComputedVec, + pub weekindex_to_first_dateindex: ComputedVec, + pub weekindex_to_last_dateindex: ComputedVec, + pub weekindex_to_timestamp: ComputedVec, + pub weekindex_to_weekindex: ComputedVec, + pub yearindex_to_decadeindex: ComputedVec, + pub yearindex_to_first_monthindex: ComputedVec, + pub yearindex_to_last_monthindex: ComputedVec, + pub yearindex_to_timestamp: ComputedVec, + pub yearindex_to_yearindex: ComputedVec, } impl Vecs { @@ -69,242 +69,242 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_date: StorableVec::forced_import( + dateindex_to_date: ComputedVec::forced_import( &path.join("dateindex_to_date"), Version::ONE, compressed, )?, - dateindex_to_dateindex: StorableVec::forced_import( + dateindex_to_dateindex: ComputedVec::forced_import( &path.join("dateindex_to_dateindex"), Version::ONE, compressed, )?, - dateindex_to_first_height: StorableVec::forced_import( + dateindex_to_first_height: ComputedVec::forced_import( &path.join("dateindex_to_first_height"), Version::ONE, compressed, )?, - dateindex_to_last_height: StorableVec::forced_import( + dateindex_to_last_height: ComputedVec::forced_import( &path.join("dateindex_to_last_height"), Version::ONE, compressed, )?, - height_to_real_date: StorableVec::forced_import( + height_to_real_date: ComputedVec::forced_import( &path.join("height_to_real_date"), Version::ONE, compressed, )?, - height_to_fixed_date: StorableVec::forced_import( + height_to_fixed_date: ComputedVec::forced_import( &path.join("height_to_fixed_date"), Version::ONE, compressed, )?, - height_to_dateindex: StorableVec::forced_import( + height_to_dateindex: ComputedVec::forced_import( &path.join("height_to_dateindex"), Version::ONE, compressed, )?, - height_to_height: StorableVec::forced_import( + height_to_height: ComputedVec::forced_import( &path.join("height_to_height"), Version::ONE, compressed, )?, - height_to_last_txindex: StorableVec::forced_import( + height_to_last_txindex: ComputedVec::forced_import( &path.join("height_to_last_txindex"), Version::ONE, compressed, )?, - txindex_to_last_txinindex: StorableVec::forced_import( + txindex_to_last_txinindex: ComputedVec::forced_import( &path.join("txindex_to_last_txinindex"), Version::ONE, compressed, )?, - txindex_to_last_txoutindex: StorableVec::forced_import( + txindex_to_last_txoutindex: ComputedVec::forced_import( &path.join("txindex_to_last_txoutindex"), Version::ONE, compressed, )?, - difficultyepoch_to_first_height: StorableVec::forced_import( + difficultyepoch_to_first_height: ComputedVec::forced_import( &path.join("difficultyepoch_to_first_height"), Version::ONE, compressed, )?, - difficultyepoch_to_last_height: StorableVec::forced_import( + difficultyepoch_to_last_height: ComputedVec::forced_import( &path.join("difficultyepoch_to_last_height"), Version::ONE, compressed, )?, - halvingepoch_to_first_height: StorableVec::forced_import( + halvingepoch_to_first_height: ComputedVec::forced_import( &path.join("halvingepoch_to_first_height"), Version::ONE, compressed, )?, - halvingepoch_to_last_height: StorableVec::forced_import( + halvingepoch_to_last_height: ComputedVec::forced_import( &path.join("halvingepoch_to_last_height"), Version::ONE, compressed, )?, - weekindex_to_first_dateindex: StorableVec::forced_import( + weekindex_to_first_dateindex: ComputedVec::forced_import( &path.join("weekindex_to_first_dateindex"), Version::ONE, compressed, )?, - weekindex_to_last_dateindex: StorableVec::forced_import( + weekindex_to_last_dateindex: ComputedVec::forced_import( &path.join("weekindex_to_last_dateindex"), Version::ONE, compressed, )?, - monthindex_to_first_dateindex: StorableVec::forced_import( + monthindex_to_first_dateindex: ComputedVec::forced_import( &path.join("monthindex_to_first_dateindex"), Version::ONE, compressed, )?, - monthindex_to_last_dateindex: StorableVec::forced_import( + monthindex_to_last_dateindex: ComputedVec::forced_import( &path.join("monthindex_to_last_dateindex"), Version::ONE, compressed, )?, - yearindex_to_first_monthindex: StorableVec::forced_import( + yearindex_to_first_monthindex: ComputedVec::forced_import( &path.join("yearindex_to_first_monthindex"), Version::ONE, compressed, )?, - yearindex_to_last_monthindex: StorableVec::forced_import( + yearindex_to_last_monthindex: ComputedVec::forced_import( &path.join("yearindex_to_last_monthindex"), Version::ONE, compressed, )?, - decadeindex_to_first_yearindex: StorableVec::forced_import( + decadeindex_to_first_yearindex: ComputedVec::forced_import( &path.join("decadeindex_to_first_yearindex"), Version::ONE, compressed, )?, - decadeindex_to_last_yearindex: StorableVec::forced_import( + decadeindex_to_last_yearindex: ComputedVec::forced_import( &path.join("decadeindex_to_last_yearindex"), Version::ONE, compressed, )?, - dateindex_to_weekindex: StorableVec::forced_import( + dateindex_to_weekindex: ComputedVec::forced_import( &path.join("dateindex_to_weekindex"), Version::ONE, compressed, )?, - dateindex_to_monthindex: StorableVec::forced_import( + dateindex_to_monthindex: ComputedVec::forced_import( &path.join("dateindex_to_monthindex"), Version::ONE, compressed, )?, - monthindex_to_yearindex: StorableVec::forced_import( + monthindex_to_yearindex: ComputedVec::forced_import( &path.join("monthindex_to_yearindex"), Version::ONE, compressed, )?, - yearindex_to_decadeindex: StorableVec::forced_import( + yearindex_to_decadeindex: ComputedVec::forced_import( &path.join("yearindex_to_decadeindex"), Version::ONE, compressed, )?, - height_to_difficultyepoch: StorableVec::forced_import( + height_to_difficultyepoch: ComputedVec::forced_import( &path.join("height_to_difficultyepoch"), Version::ONE, compressed, )?, - height_to_halvingepoch: StorableVec::forced_import( + height_to_halvingepoch: ComputedVec::forced_import( &path.join("height_to_halvingepoch"), Version::ONE, compressed, )?, - weekindex_to_weekindex: StorableVec::forced_import( + weekindex_to_weekindex: ComputedVec::forced_import( &path.join("weekindex_to_weekindex"), Version::ONE, compressed, )?, - monthindex_to_monthindex: StorableVec::forced_import( + monthindex_to_monthindex: ComputedVec::forced_import( &path.join("monthindex_to_monthindex"), Version::ONE, compressed, )?, - yearindex_to_yearindex: StorableVec::forced_import( + yearindex_to_yearindex: ComputedVec::forced_import( &path.join("yearindex_to_yearindex"), Version::ONE, compressed, )?, - decadeindex_to_decadeindex: StorableVec::forced_import( + decadeindex_to_decadeindex: ComputedVec::forced_import( &path.join("decadeindex_to_decadeindex"), Version::ONE, compressed, )?, - difficultyepoch_to_difficultyepoch: StorableVec::forced_import( + difficultyepoch_to_difficultyepoch: ComputedVec::forced_import( &path.join("difficultyepoch_to_difficultyepoch"), Version::ONE, compressed, )?, - halvingepoch_to_halvingepoch: StorableVec::forced_import( + halvingepoch_to_halvingepoch: ComputedVec::forced_import( &path.join("halvingepoch_to_halvingepoch"), Version::ONE, compressed, )?, - dateindex_to_timestamp: StorableVec::forced_import( + dateindex_to_timestamp: ComputedVec::forced_import( &path.join("dateindex_to_timestamp"), Version::ONE, compressed, )?, - decadeindex_to_timestamp: StorableVec::forced_import( + decadeindex_to_timestamp: ComputedVec::forced_import( &path.join("decadeindex_to_timestamp"), Version::ONE, compressed, )?, - difficultyepoch_to_timestamp: StorableVec::forced_import( + difficultyepoch_to_timestamp: ComputedVec::forced_import( &path.join("difficultyepoch_to_timestamp"), Version::ONE, compressed, )?, - halvingepoch_to_timestamp: StorableVec::forced_import( + halvingepoch_to_timestamp: ComputedVec::forced_import( &path.join("halvingepoch_to_timestamp"), Version::ONE, compressed, )?, - monthindex_to_timestamp: StorableVec::forced_import( + monthindex_to_timestamp: ComputedVec::forced_import( &path.join("monthindex_to_timestamp"), Version::ONE, compressed, )?, - weekindex_to_timestamp: StorableVec::forced_import( + weekindex_to_timestamp: ComputedVec::forced_import( &path.join("weekindex_to_timestamp"), Version::ONE, compressed, )?, - yearindex_to_timestamp: StorableVec::forced_import( + yearindex_to_timestamp: ComputedVec::forced_import( &path.join("yearindex_to_timestamp"), Version::ONE, compressed, )?, - height_to_fixed_timestamp: StorableVec::forced_import( + height_to_fixed_timestamp: ComputedVec::forced_import( &path.join("height_to_fixed_timestamp"), Version::ONE, compressed, )?, - monthindex_to_quarterindex: StorableVec::forced_import( + monthindex_to_quarterindex: ComputedVec::forced_import( &path.join("monthindex_to_quarterindex"), Version::ONE, compressed, )?, - quarterindex_to_first_monthindex: StorableVec::forced_import( + quarterindex_to_first_monthindex: ComputedVec::forced_import( &path.join("quarterindex_to_first_monthindex"), Version::ONE, compressed, )?, - quarterindex_to_last_monthindex: StorableVec::forced_import( + quarterindex_to_last_monthindex: ComputedVec::forced_import( &path.join("quarterindex_to_last_monthindex"), Version::ONE, compressed, )?, - quarterindex_to_quarterindex: StorableVec::forced_import( + quarterindex_to_quarterindex: ComputedVec::forced_import( &path.join("quarterindex_to_quarterindex"), Version::ONE, compressed, )?, - quarterindex_to_timestamp: StorableVec::forced_import( + quarterindex_to_timestamp: ComputedVec::forced_import( &path.join("quarterindex_to_timestamp"), Version::ONE, compressed, diff --git a/crates/brk_computer/src/storage/vecs/marketprice.rs b/crates/brk_computer/src/storage/vecs/marketprice.rs index 9d057b81c..2c88569dd 100644 --- a/crates/brk_computer/src/storage/vecs/marketprice.rs +++ b/crates/brk_computer/src/storage/vecs/marketprice.rs @@ -10,53 +10,45 @@ use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; use super::{ - Indexes, StorableVec, indexes, - stats::{ - StorableVecGeneatorOptions, StorableVecsStatsFromDate, StorableVecsStatsFromHeightStrict, + ComputedVec, Indexes, + grouped::{ + ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions, }, + indexes, }; #[derive(Clone)] pub struct Vecs { - pub dateindex_to_close: StorableVec>, - pub dateindex_to_close_in_cents: StorableVec>, - pub dateindex_to_high: StorableVec>, - pub dateindex_to_high_in_cents: StorableVec>, - pub dateindex_to_low: StorableVec>, - pub dateindex_to_low_in_cents: StorableVec>, - pub dateindex_to_ohlc: StorableVec, - pub dateindex_to_ohlc_in_cents: StorableVec, - pub dateindex_to_open: StorableVec>, - pub dateindex_to_open_in_cents: StorableVec>, - pub dateindex_to_sats_per_dollar: StorableVec>, - pub height_to_close: StorableVec>, - pub height_to_close_in_cents: StorableVec>, - pub height_to_high: StorableVec>, - pub height_to_high_in_cents: StorableVec>, - pub height_to_low: StorableVec>, - pub height_to_low_in_cents: StorableVec>, - pub height_to_ohlc: StorableVec, - pub height_to_ohlc_in_cents: StorableVec, - pub height_to_open: StorableVec>, - pub height_to_open_in_cents: StorableVec>, - pub height_to_sats_per_dollar: StorableVec>, - pub timeindexes_to_close: StorableVecsStatsFromDate>, - pub timeindexes_to_high: StorableVecsStatsFromDate>, - pub timeindexes_to_low: StorableVecsStatsFromDate>, - pub timeindexes_to_open: StorableVecsStatsFromDate>, - pub timeindexes_to_sats_per_dollar: StorableVecsStatsFromDate>, - pub chainindexes_to_close: StorableVecsStatsFromHeightStrict>, - pub chainindexes_to_high: StorableVecsStatsFromHeightStrict>, - pub chainindexes_to_low: StorableVecsStatsFromHeightStrict>, - pub chainindexes_to_open: StorableVecsStatsFromHeightStrict>, - pub chainindexes_to_sats_per_dollar: StorableVecsStatsFromHeightStrict>, - pub weekindex_to_ohlc: StorableVec, - pub difficultyepoch_to_ohlc: StorableVec, - pub monthindex_to_ohlc: StorableVec, - pub quarterindex_to_ohlc: StorableVec, - pub yearindex_to_ohlc: StorableVec, + // pub dateindex_to_close: ComputedVec>, + pub dateindex_to_close_in_cents: ComputedVec>, + pub dateindex_to_high_in_cents: ComputedVec>, + pub dateindex_to_low_in_cents: ComputedVec>, + pub dateindex_to_ohlc: ComputedVec, + pub dateindex_to_ohlc_in_cents: ComputedVec, + pub dateindex_to_open_in_cents: ComputedVec>, + pub height_to_close_in_cents: ComputedVec>, + pub height_to_high_in_cents: ComputedVec>, + pub height_to_low_in_cents: ComputedVec>, + pub height_to_ohlc: ComputedVec, + pub height_to_ohlc_in_cents: ComputedVec, + pub height_to_open_in_cents: ComputedVec>, + pub timeindexes_to_close: ComputedVecsFromDateindex>, + pub timeindexes_to_high: ComputedVecsFromDateindex>, + pub timeindexes_to_low: ComputedVecsFromDateindex>, + pub timeindexes_to_open: ComputedVecsFromDateindex>, + pub timeindexes_to_sats_per_dollar: ComputedVecsFromDateindex>, + pub chainindexes_to_close: ComputedVecsFromHeightStrict>, + pub chainindexes_to_high: ComputedVecsFromHeightStrict>, + pub chainindexes_to_low: ComputedVecsFromHeightStrict>, + pub chainindexes_to_open: ComputedVecsFromHeightStrict>, + pub chainindexes_to_sats_per_dollar: ComputedVecsFromHeightStrict>, + pub weekindex_to_ohlc: ComputedVec, + pub difficultyepoch_to_ohlc: ComputedVec, + pub monthindex_to_ohlc: ComputedVec, + pub quarterindex_to_ohlc: ComputedVec, + pub yearindex_to_ohlc: ComputedVec, // pub halvingepoch_to_ohlc: StorableVec, - pub decadeindex_to_ohlc: StorableVec, + pub decadeindex_to_ohlc: ComputedVec, } impl Vecs { @@ -64,193 +56,163 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - dateindex_to_ohlc_in_cents: StorableVec::forced_import( + dateindex_to_ohlc_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_ohlc_in_cents"), Version::ONE, compressed, )?, - dateindex_to_ohlc: StorableVec::forced_import( + dateindex_to_ohlc: ComputedVec::forced_import( &path.join("dateindex_to_ohlc"), Version::ONE, compressed, )?, - dateindex_to_close_in_cents: StorableVec::forced_import( + dateindex_to_close_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_close_in_cents"), Version::ONE, compressed, )?, - dateindex_to_close: StorableVec::forced_import( - &path.join("dateindex_to_close"), - Version::ONE, - compressed, - )?, - dateindex_to_high_in_cents: StorableVec::forced_import( + dateindex_to_high_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_high_in_cents"), Version::ONE, compressed, )?, - dateindex_to_high: StorableVec::forced_import( - &path.join("dateindex_to_high"), - Version::ONE, - compressed, - )?, - dateindex_to_low_in_cents: StorableVec::forced_import( + dateindex_to_low_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_low_in_cents"), Version::ONE, compressed, )?, - dateindex_to_low: StorableVec::forced_import( - &path.join("dateindex_to_low"), - Version::ONE, - compressed, - )?, - dateindex_to_open_in_cents: StorableVec::forced_import( + dateindex_to_open_in_cents: ComputedVec::forced_import( &path.join("dateindex_to_open_in_cents"), Version::ONE, compressed, )?, - dateindex_to_open: StorableVec::forced_import( - &path.join("dateindex_to_open"), - Version::ONE, - compressed, - )?, - dateindex_to_sats_per_dollar: StorableVec::forced_import( - &path.join("dateindex_to_sats_per_dollar"), - Version::ONE, - compressed, - )?, - height_to_ohlc_in_cents: StorableVec::forced_import( + height_to_ohlc_in_cents: ComputedVec::forced_import( &path.join("height_to_ohlc_in_cents"), Version::ONE, compressed, )?, - height_to_ohlc: StorableVec::forced_import( + height_to_ohlc: ComputedVec::forced_import( &path.join("height_to_ohlc"), Version::ONE, compressed, )?, - height_to_close_in_cents: StorableVec::forced_import( + height_to_close_in_cents: ComputedVec::forced_import( &path.join("height_to_close_in_cents"), Version::ONE, compressed, )?, - height_to_close: StorableVec::forced_import( - &path.join("height_to_close"), - Version::ONE, - compressed, - )?, - height_to_high_in_cents: StorableVec::forced_import( + height_to_high_in_cents: ComputedVec::forced_import( &path.join("height_to_high_in_cents"), Version::ONE, compressed, )?, - height_to_high: StorableVec::forced_import( - &path.join("height_to_high"), - Version::ONE, - compressed, - )?, - height_to_low_in_cents: StorableVec::forced_import( + height_to_low_in_cents: ComputedVec::forced_import( &path.join("height_to_low_in_cents"), Version::ONE, compressed, )?, - height_to_low: StorableVec::forced_import( - &path.join("height_to_low"), - Version::ONE, - compressed, - )?, - height_to_open_in_cents: StorableVec::forced_import( + height_to_open_in_cents: ComputedVec::forced_import( &path.join("height_to_open_in_cents"), Version::ONE, compressed, )?, - height_to_open: StorableVec::forced_import( - &path.join("height_to_open"), + timeindexes_to_open: ComputedVecsFromDateindex::forced_import( + path, + "open", Version::ONE, compressed, - )?, - height_to_sats_per_dollar: StorableVec::forced_import( - &path.join("height_to_sats_per_dollar"), - Version::ONE, - compressed, - )?, - timeindexes_to_open: StorableVecsStatsFromDate::forced_import( - &path.join("open"), - compressed, StorableVecGeneatorOptions::default().add_first(), )?, - timeindexes_to_high: StorableVecsStatsFromDate::forced_import( - &path.join("high"), + timeindexes_to_high: ComputedVecsFromDateindex::forced_import( + path, + "high", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_max(), )?, - timeindexes_to_low: StorableVecsStatsFromDate::forced_import( - &path.join("low"), + timeindexes_to_low: ComputedVecsFromDateindex::forced_import( + path, + "low", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_min(), )?, - timeindexes_to_close: StorableVecsStatsFromDate::forced_import( - &path.join("close"), + timeindexes_to_close: ComputedVecsFromDateindex::forced_import( + path, + "close", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_last(), )?, - timeindexes_to_sats_per_dollar: StorableVecsStatsFromDate::forced_import( - &path.join("sats_per_dollar"), + timeindexes_to_sats_per_dollar: ComputedVecsFromDateindex::forced_import( + path, + "sats_per_dollar", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_last(), )?, - chainindexes_to_open: StorableVecsStatsFromHeightStrict::forced_import( - &path.join("open"), + chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import( + path, + "open", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_first(), )?, - chainindexes_to_high: StorableVecsStatsFromHeightStrict::forced_import( - &path.join("high"), + chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import( + path, + "high", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_max(), )?, - chainindexes_to_low: StorableVecsStatsFromHeightStrict::forced_import( - &path.join("low"), + chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import( + path, + "low", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_min(), )?, - chainindexes_to_close: StorableVecsStatsFromHeightStrict::forced_import( - &path.join("close"), + chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import( + path, + "close", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_last(), )?, - chainindexes_to_sats_per_dollar: StorableVecsStatsFromHeightStrict::forced_import( - &path.join("sats_per_dollar"), + chainindexes_to_sats_per_dollar: ComputedVecsFromHeightStrict::forced_import( + path, + "sats_per_dollar", + Version::ONE, compressed, StorableVecGeneatorOptions::default().add_last(), )?, - weekindex_to_ohlc: StorableVec::forced_import( + weekindex_to_ohlc: ComputedVec::forced_import( &path.join("weekindex_to_ohlc"), Version::ONE, compressed, )?, - difficultyepoch_to_ohlc: StorableVec::forced_import( + difficultyepoch_to_ohlc: ComputedVec::forced_import( &path.join("difficultyepoch_to_ohlc"), Version::ONE, compressed, )?, - monthindex_to_ohlc: StorableVec::forced_import( + monthindex_to_ohlc: ComputedVec::forced_import( &path.join("monthindex_to_ohlc"), Version::ONE, compressed, )?, - quarterindex_to_ohlc: StorableVec::forced_import( + quarterindex_to_ohlc: ComputedVec::forced_import( &path.join("quarterindex_to_ohlc"), Version::ONE, compressed, )?, - yearindex_to_ohlc: StorableVec::forced_import( + yearindex_to_ohlc: ComputedVec::forced_import( &path.join("yearindex_to_ohlc"), Version::ONE, compressed, )?, // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ONE, compressed)?, - decadeindex_to_ohlc: StorableVec::forced_import( + decadeindex_to_ohlc: ComputedVec::forced_import( &path.join("decadeindex_to_ohlc"), Version::ONE, compressed, @@ -320,41 +282,6 @@ impl Vecs { exit, )?; - self.height_to_open.compute_transform( - starting_indexes.height, - self.height_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.open), - exit, - )?; - - self.height_to_high.compute_transform( - starting_indexes.height, - self.height_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.high), - exit, - )?; - - self.height_to_low.compute_transform( - starting_indexes.height, - self.height_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.low), - exit, - )?; - - self.height_to_close.compute_transform( - starting_indexes.height, - self.height_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.close), - exit, - )?; - - self.height_to_sats_per_dollar.compute_transform( - starting_indexes.height, - self.height_to_close.mut_vec(), - |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)), - exit, - )?; - self.dateindex_to_ohlc_in_cents.compute_transform( starting_indexes.dateindex, indexes.dateindex_to_date.mut_vec(), @@ -400,92 +327,113 @@ impl Vecs { exit, )?; - self.dateindex_to_open.compute_transform( - starting_indexes.dateindex, - self.dateindex_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.open), - exit, - )?; - - self.dateindex_to_high.compute_transform( - starting_indexes.dateindex, - self.dateindex_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.high), - exit, - )?; - - self.dateindex_to_low.compute_transform( - starting_indexes.dateindex, - self.dateindex_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.low), - exit, - )?; - - self.dateindex_to_close.compute_transform( - starting_indexes.dateindex, - self.dateindex_to_ohlc.mut_vec(), - |(di, ohlc, ..)| (di, ohlc.close), - exit, - )?; - - self.dateindex_to_sats_per_dollar.compute_transform( - starting_indexes.dateindex, - self.dateindex_to_close.mut_vec(), - |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)), - exit, - )?; - self.timeindexes_to_close.compute( - &mut self.dateindex_to_close, + |v| { + v.compute_transform( + starting_indexes.dateindex, + self.dateindex_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.close), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.timeindexes_to_high.compute( - &mut self.dateindex_to_high, + |v| { + v.compute_transform( + starting_indexes.dateindex, + self.dateindex_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.high), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.timeindexes_to_low.compute( - &mut self.dateindex_to_low, + |v| { + v.compute_transform( + starting_indexes.dateindex, + self.dateindex_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.low), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.timeindexes_to_open.compute( - &mut self.dateindex_to_open, + |v| { + v.compute_transform( + starting_indexes.dateindex, + self.dateindex_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.open), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.chainindexes_to_close.compute( - &mut self.height_to_close, + |v| { + v.compute_transform( + starting_indexes.height, + self.height_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.close), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.chainindexes_to_high.compute( - &mut self.height_to_high, + |v| { + v.compute_transform( + starting_indexes.height, + self.height_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.high), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.chainindexes_to_low.compute( - &mut self.height_to_low, + |v| { + v.compute_transform( + starting_indexes.height, + self.height_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.low), + exit, + ) + }, indexes, starting_indexes, exit, )?; self.chainindexes_to_open.compute( - &mut self.height_to_open, + |v| { + v.compute_transform( + starting_indexes.height, + self.height_to_ohlc.mut_vec(), + |(di, ohlc, ..)| (di, ohlc.open), + exit, + ) + }, indexes, starting_indexes, exit, @@ -771,34 +719,52 @@ impl Vecs { exit, )?; + self.chainindexes_to_sats_per_dollar.compute( + |v| { + v.compute_transform( + starting_indexes.height, + self.chainindexes_to_close.height.mut_vec(), + |(i, close, ..)| (i, Close::from(Sats::ONE_BTC / *close)), + exit, + ) + }, + indexes, + starting_indexes, + exit, + )?; + + self.timeindexes_to_sats_per_dollar.compute( + |v| { + v.compute_transform( + starting_indexes.dateindex, + self.timeindexes_to_close.dateindex.mut_vec(), + |(i, close, ..)| (i, Close::from(Sats::ONE_BTC / *close)), + exit, + ) + }, + indexes, + starting_indexes, + exit, + )?; + Ok(()) } pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { vec![ vec![ - self.dateindex_to_close.any_vec(), self.dateindex_to_close_in_cents.any_vec(), - self.dateindex_to_high.any_vec(), self.dateindex_to_high_in_cents.any_vec(), - self.dateindex_to_low.any_vec(), self.dateindex_to_low_in_cents.any_vec(), self.dateindex_to_ohlc.any_vec(), self.dateindex_to_ohlc_in_cents.any_vec(), - self.dateindex_to_open.any_vec(), self.dateindex_to_open_in_cents.any_vec(), - self.dateindex_to_sats_per_dollar.any_vec(), - self.height_to_close.any_vec(), self.height_to_close_in_cents.any_vec(), - self.height_to_high.any_vec(), self.height_to_high_in_cents.any_vec(), - self.height_to_low.any_vec(), self.height_to_low_in_cents.any_vec(), self.height_to_ohlc.any_vec(), self.height_to_ohlc_in_cents.any_vec(), - self.height_to_open.any_vec(), self.height_to_open_in_cents.any_vec(), - self.height_to_sats_per_dollar.any_vec(), self.weekindex_to_ohlc.any_vec(), self.difficultyepoch_to_ohlc.any_vec(), self.monthindex_to_ohlc.any_vec(), @@ -807,14 +773,16 @@ impl Vecs { // self.halvingepoch_to_ohlc.any_vec(), self.decadeindex_to_ohlc.any_vec(), ], - self.timeindexes_to_close.as_any_vecs(), - self.timeindexes_to_high.as_any_vecs(), - self.timeindexes_to_low.as_any_vecs(), - self.timeindexes_to_open.as_any_vecs(), - self.chainindexes_to_close.as_any_vecs(), - self.chainindexes_to_high.as_any_vecs(), - self.chainindexes_to_low.as_any_vecs(), - self.chainindexes_to_open.as_any_vecs(), + self.chainindexes_to_sats_per_dollar.any_vecs(), + self.timeindexes_to_sats_per_dollar.any_vecs(), + self.timeindexes_to_close.any_vecs(), + self.timeindexes_to_high.any_vecs(), + self.timeindexes_to_low.any_vecs(), + self.timeindexes_to_open.any_vecs(), + self.chainindexes_to_close.any_vecs(), + self.chainindexes_to_high.any_vecs(), + self.chainindexes_to_low.any_vecs(), + self.chainindexes_to_open.any_vecs(), ] .concat() } diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index e873110cf..ce8a1d064 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -7,9 +7,9 @@ use brk_vec::{AnyStorableVec, Compressed}; mod base; mod blocks; +mod grouped; mod indexes; mod marketprice; -mod stats; mod transactions; use base::*; diff --git a/crates/brk_computer/src/storage/vecs/stats/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/stats/from_height_strict.rs deleted file mode 100644 index 35d351e75..000000000 --- a/crates/brk_computer/src/storage/vecs/stats/from_height_strict.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::path::Path; - -use brk_core::{Difficultyepoch, Height}; -use brk_exit::Exit; -use brk_vec::{AnyStorableVec, Compressed}; - -use crate::storage::vecs::{Indexes, base::StorableVec, indexes}; - -use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions}; - -#[derive(Clone)] -pub struct StorableVecsStatsFromHeightStrict -where - T: ComputedType + PartialOrd, -{ - pub difficultyepoch: StorableVecBuilder, - // TODO: pub halvingepoch: StorableVecGeneator, -} - -impl StorableVecsStatsFromHeightStrict -where - T: ComputedType + Ord + From, - f64: From, -{ - pub fn forced_import( - path: &Path, - compressed: Compressed, - options: StorableVecGeneatorOptions, - ) -> color_eyre::Result { - let options = options.remove_percentiles(); - - Ok(Self { - difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?, - // halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?, - }) - } - - pub fn compute( - &mut self, - source: &mut StorableVec, - indexes: &mut indexes::Vecs, - starting_indexes: &Indexes, - exit: &Exit, - ) -> color_eyre::Result<()> { - self.difficultyepoch.compute( - starting_indexes.difficultyepoch, - source, - indexes.difficultyepoch_to_first_height.mut_vec(), - indexes.difficultyepoch_to_last_height.mut_vec(), - exit, - )?; - - Ok(()) - } - - pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> { - [ - self.difficultyepoch.as_any_vecs(), - // self.halvingepoch.as_any_vecs(), - ] - .concat() - } -} diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 7d425e5bb..10e0b8ae5 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -5,27 +5,27 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyStorableVec, Compressed, Version}; -use super::{Indexes, StorableVec, indexes}; +use super::{ComputedVec, Indexes, indexes}; #[derive(Clone)] pub struct Vecs { - // pub height_to_fee: StorableVec, - // pub height_to_inputcount: StorableVec, - // pub height_to_maxfeerate: StorableVec, - // pub height_to_medianfeerate: StorableVec, - // pub height_to_minfeerate: StorableVec, - // pub height_to_outputcount: StorableVec, - // pub height_to_subsidy: StorableVec, - // pub height_to_totalfees: StorableVec, - // pub height_to_txcount: StorableVec, - // pub txindex_to_fee: StorableVec, - pub txindex_to_is_coinbase: StorableVec, - // pub txindex_to_feerate: StorableVec, - pub txindex_to_inputs_count: StorableVec, - // pub txindex_to_inputs_sum: StorableVec, - pub txindex_to_outputs_count: StorableVec, - // pub txindex_to_outputs_sum: StorableVec, - // pub txinindex_to_value: StorableVec, + // pub height_to_fee: ComputedVec, + // pub height_to_inputcount: ComputedVec, + // pub height_to_maxfeerate: ComputedVec, + // pub height_to_medianfeerate: ComputedVec, + // pub height_to_minfeerate: ComputedVec, + // pub height_to_outputcount: ComputedVec, + // pub height_to_subsidy: ComputedVec, + // pub height_to_totalfees: ComputedVec, + // pub height_to_txcount: ComputedVec, + // pub txindex_to_fee: ComputedVec, + pub txindex_to_is_coinbase: ComputedVec, + // pub txindex_to_feerate: ComputedVec, + pub txindex_to_inputs_count: ComputedVec, + // pub txindex_to_inputs_sum: ComputedVec, + pub txindex_to_outputs_count: ComputedVec, + // pub txindex_to_outputs_sum: ComputedVec, + // pub txinindex_to_value: ComputedVec, } impl Vecs { @@ -52,13 +52,13 @@ impl Vecs { // &path.join("txindex_to_fee"), // Version::ONE, // )?, - txindex_to_is_coinbase: StorableVec::forced_import( + txindex_to_is_coinbase: ComputedVec::forced_import( &path.join("txindex_to_is_coinbase"), Version::ONE, compressed, )?, // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ONE)?, - txindex_to_inputs_count: StorableVec::forced_import( + txindex_to_inputs_count: ComputedVec::forced_import( &path.join("txindex_to_inputs_count"), Version::ONE, compressed, @@ -67,7 +67,7 @@ impl Vecs { // &path.join("txindex_to_inputs_sum"), // Version::ONE, // )?, - txindex_to_outputs_count: StorableVec::forced_import( + txindex_to_outputs_count: ComputedVec::forced_import( &path.join("txindex_to_outputs_count"), Version::ONE, compressed, diff --git a/crates/brk_indexer/src/vecs/base.rs b/crates/brk_indexer/src/vecs/base.rs index a05f4ed21..fd94b769f 100644 --- a/crates/brk_indexer/src/vecs/base.rs +++ b/crates/brk_indexer/src/vecs/base.rs @@ -6,19 +6,19 @@ use std::{ }; use brk_vec::{ - AnyStorableVec, Compressed, Error, MAX_CACHE_SIZE, MAX_PAGE_SIZE, Result, StoredIndex, - StoredType, Value, Version, + AnyStorableVec, Compressed, Error, MAX_CACHE_SIZE, MAX_PAGE_SIZE, Result, StorableVec, + StoredIndex, StoredType, Value, Version, }; use super::Height; #[derive(Debug)] -pub struct StorableVec { +pub struct IndexedVec { height: Option, - vec: brk_vec::StorableVec, + vec: StorableVec, } -impl StorableVec +impl IndexedVec where I: StoredIndex, T: StoredType, @@ -162,7 +162,7 @@ where } } -impl Clone for StorableVec +impl Clone for IndexedVec where I: StoredIndex, T: StoredType, @@ -180,7 +180,7 @@ pub trait AnyIndexedVec: Send + Sync { fn flush(&mut self, height: Height) -> io::Result<()>; } -impl AnyIndexedVec for StorableVec +impl AnyIndexedVec for IndexedVec where I: StoredIndex, T: StoredType, diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index c1591ae46..2791bf927 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -18,50 +18,50 @@ pub use base::*; #[derive(Clone)] pub struct Vecs { - pub addressindex_to_addresstype: StorableVec, - pub addressindex_to_addresstypeindex: StorableVec, - pub addressindex_to_height: StorableVec, - pub height_to_blockhash: StorableVec, - pub height_to_difficulty: StorableVec, - pub height_to_first_addressindex: StorableVec, - pub height_to_first_emptyindex: StorableVec, - pub height_to_first_multisigindex: StorableVec, - pub height_to_first_opreturnindex: StorableVec, - pub height_to_first_pushonlyindex: StorableVec, - pub height_to_first_txindex: StorableVec, - pub height_to_first_txinindex: StorableVec, - pub height_to_first_txoutindex: StorableVec, - pub height_to_first_unknownindex: StorableVec, - pub height_to_first_p2pk33index: StorableVec, - pub height_to_first_p2pk65index: StorableVec, - pub height_to_first_p2pkhindex: StorableVec, - pub height_to_first_p2shindex: StorableVec, - pub height_to_first_p2trindex: StorableVec, - pub height_to_first_p2wpkhindex: StorableVec, - pub height_to_first_p2wshindex: StorableVec, - pub height_to_size: StorableVec, - pub height_to_timestamp: StorableVec, - pub height_to_weight: StorableVec, - pub p2pk33index_to_p2pk33addressbytes: StorableVec, - pub p2pk65index_to_p2pk65addressbytes: StorableVec, - pub p2pkhindex_to_p2pkhaddressbytes: StorableVec, - pub p2shindex_to_p2shaddressbytes: StorableVec, - pub p2trindex_to_p2traddressbytes: StorableVec, - pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec, - pub p2wshindex_to_p2wshaddressbytes: StorableVec, - pub txindex_to_first_txinindex: StorableVec, - pub txindex_to_first_txoutindex: StorableVec, - pub txindex_to_height: StorableVec, - pub txindex_to_locktime: StorableVec, - pub txindex_to_txid: StorableVec, - pub txindex_to_base_size: StorableVec, - pub txindex_to_total_size: StorableVec, - pub txindex_to_is_explicitly_rbf: StorableVec, - pub txindex_to_txversion: StorableVec, + pub addressindex_to_addresstype: IndexedVec, + pub addressindex_to_addresstypeindex: IndexedVec, + pub addressindex_to_height: IndexedVec, + pub height_to_blockhash: IndexedVec, + pub height_to_difficulty: IndexedVec, + pub height_to_first_addressindex: IndexedVec, + pub height_to_first_emptyindex: IndexedVec, + pub height_to_first_multisigindex: IndexedVec, + pub height_to_first_opreturnindex: IndexedVec, + pub height_to_first_pushonlyindex: IndexedVec, + pub height_to_first_txindex: IndexedVec, + pub height_to_first_txinindex: IndexedVec, + pub height_to_first_txoutindex: IndexedVec, + pub height_to_first_unknownindex: IndexedVec, + pub height_to_first_p2pk33index: IndexedVec, + pub height_to_first_p2pk65index: IndexedVec, + pub height_to_first_p2pkhindex: IndexedVec, + pub height_to_first_p2shindex: IndexedVec, + pub height_to_first_p2trindex: IndexedVec, + pub height_to_first_p2wpkhindex: IndexedVec, + pub height_to_first_p2wshindex: IndexedVec, + pub height_to_size: IndexedVec, + pub height_to_timestamp: IndexedVec, + pub height_to_weight: IndexedVec, + pub p2pk33index_to_p2pk33addressbytes: IndexedVec, + pub p2pk65index_to_p2pk65addressbytes: IndexedVec, + pub p2pkhindex_to_p2pkhaddressbytes: IndexedVec, + pub p2shindex_to_p2shaddressbytes: IndexedVec, + pub p2trindex_to_p2traddressbytes: IndexedVec, + pub p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec, + pub p2wshindex_to_p2wshaddressbytes: IndexedVec, + pub txindex_to_first_txinindex: IndexedVec, + pub txindex_to_first_txoutindex: IndexedVec, + pub txindex_to_height: IndexedVec, + pub txindex_to_locktime: IndexedVec, + pub txindex_to_txid: IndexedVec, + pub txindex_to_base_size: IndexedVec, + pub txindex_to_total_size: IndexedVec, + pub txindex_to_is_explicitly_rbf: IndexedVec, + pub txindex_to_txversion: IndexedVec, /// If txoutindex == Txoutindex MAX then is it's coinbase - pub txinindex_to_txoutindex: StorableVec, - pub txoutindex_to_addressindex: StorableVec, - pub txoutindex_to_value: StorableVec, + pub txinindex_to_txoutindex: IndexedVec, + pub txoutindex_to_addressindex: IndexedVec, + pub txoutindex_to_value: IndexedVec, } impl Vecs { @@ -69,217 +69,217 @@ impl Vecs { fs::create_dir_all(path)?; Ok(Self { - addressindex_to_addresstype: StorableVec::forced_import( + addressindex_to_addresstype: IndexedVec::forced_import( &path.join("addressindex_to_addresstype"), Version::ONE, compressed, )?, - addressindex_to_addresstypeindex: StorableVec::forced_import( + addressindex_to_addresstypeindex: IndexedVec::forced_import( &path.join("addressindex_to_addresstypeindex"), Version::ONE, compressed, )?, - addressindex_to_height: StorableVec::forced_import( + addressindex_to_height: IndexedVec::forced_import( &path.join("addressindex_to_height"), Version::ONE, compressed, )?, - height_to_blockhash: StorableVec::forced_import( + height_to_blockhash: IndexedVec::forced_import( &path.join("height_to_blockhash"), Version::ONE, Compressed::NO, )?, - height_to_difficulty: StorableVec::forced_import( + height_to_difficulty: IndexedVec::forced_import( &path.join("height_to_difficulty"), Version::ONE, compressed, )?, - height_to_first_addressindex: StorableVec::forced_import( + height_to_first_addressindex: IndexedVec::forced_import( &path.join("height_to_first_addressindex"), Version::ONE, compressed, )?, - height_to_first_emptyindex: StorableVec::forced_import( + height_to_first_emptyindex: IndexedVec::forced_import( &path.join("height_to_first_emptyindex"), Version::ONE, compressed, )?, - height_to_first_multisigindex: StorableVec::forced_import( + height_to_first_multisigindex: IndexedVec::forced_import( &path.join("height_to_first_multisigindex"), Version::ONE, compressed, )?, - height_to_first_opreturnindex: StorableVec::forced_import( + height_to_first_opreturnindex: IndexedVec::forced_import( &path.join("height_to_first_opreturnindex"), Version::ONE, compressed, )?, - height_to_first_pushonlyindex: StorableVec::forced_import( + height_to_first_pushonlyindex: IndexedVec::forced_import( &path.join("height_to_first_pushonlyindex"), Version::ONE, compressed, )?, - height_to_first_txindex: StorableVec::forced_import( + height_to_first_txindex: IndexedVec::forced_import( &path.join("height_to_first_txindex"), Version::ONE, compressed, )?, - height_to_first_txinindex: StorableVec::forced_import( + height_to_first_txinindex: IndexedVec::forced_import( &path.join("height_to_first_txinindex"), Version::ONE, compressed, )?, - height_to_first_txoutindex: StorableVec::forced_import( + height_to_first_txoutindex: IndexedVec::forced_import( &path.join("height_to_first_txoutindex"), Version::ONE, compressed, )?, - height_to_first_unknownindex: StorableVec::forced_import( + height_to_first_unknownindex: IndexedVec::forced_import( &path.join("height_to_first_unkownindex"), Version::ONE, compressed, )?, - height_to_first_p2pk33index: StorableVec::forced_import( + height_to_first_p2pk33index: IndexedVec::forced_import( &path.join("height_to_first_p2pk33index"), Version::ONE, compressed, )?, - height_to_first_p2pk65index: StorableVec::forced_import( + height_to_first_p2pk65index: IndexedVec::forced_import( &path.join("height_to_first_p2pk65index"), Version::ONE, compressed, )?, - height_to_first_p2pkhindex: StorableVec::forced_import( + height_to_first_p2pkhindex: IndexedVec::forced_import( &path.join("height_to_first_p2pkhindex"), Version::ONE, compressed, )?, - height_to_first_p2shindex: StorableVec::forced_import( + height_to_first_p2shindex: IndexedVec::forced_import( &path.join("height_to_first_p2shindex"), Version::ONE, compressed, )?, - height_to_first_p2trindex: StorableVec::forced_import( + height_to_first_p2trindex: IndexedVec::forced_import( &path.join("height_to_first_p2trindex"), Version::ONE, compressed, )?, - height_to_first_p2wpkhindex: StorableVec::forced_import( + height_to_first_p2wpkhindex: IndexedVec::forced_import( &path.join("height_to_first_p2wpkhindex"), Version::ONE, compressed, )?, - height_to_first_p2wshindex: StorableVec::forced_import( + height_to_first_p2wshindex: IndexedVec::forced_import( &path.join("height_to_first_p2wshindex"), Version::ONE, compressed, )?, - height_to_size: StorableVec::forced_import( + height_to_size: IndexedVec::forced_import( &path.join("height_to_size"), Version::ONE, compressed, )?, - height_to_timestamp: StorableVec::forced_import( + height_to_timestamp: IndexedVec::forced_import( &path.join("height_to_timestamp"), Version::ONE, compressed, )?, - height_to_weight: StorableVec::forced_import( + height_to_weight: IndexedVec::forced_import( &path.join("height_to_weight"), Version::ONE, compressed, )?, - p2pk33index_to_p2pk33addressbytes: StorableVec::forced_import( + p2pk33index_to_p2pk33addressbytes: IndexedVec::forced_import( &path.join("p2pk33index_to_p2pk33addressbytes"), Version::ONE, Compressed::NO, )?, - p2pk65index_to_p2pk65addressbytes: StorableVec::forced_import( + p2pk65index_to_p2pk65addressbytes: IndexedVec::forced_import( &path.join("p2pk65index_to_p2pk65addressbytes"), Version::ONE, Compressed::NO, )?, - p2pkhindex_to_p2pkhaddressbytes: StorableVec::forced_import( + p2pkhindex_to_p2pkhaddressbytes: IndexedVec::forced_import( &path.join("p2pkhindex_to_p2pkhaddressbytes"), Version::ONE, Compressed::NO, )?, - p2shindex_to_p2shaddressbytes: StorableVec::forced_import( + p2shindex_to_p2shaddressbytes: IndexedVec::forced_import( &path.join("p2shindex_to_p2shaddressbytes"), Version::ONE, Compressed::NO, )?, - p2trindex_to_p2traddressbytes: StorableVec::forced_import( + p2trindex_to_p2traddressbytes: IndexedVec::forced_import( &path.join("p2trindex_to_p2traddressbytes"), Version::ONE, Compressed::NO, )?, - p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::forced_import( + p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec::forced_import( &path.join("p2wpkhindex_to_p2wpkhaddressbytes"), Version::ONE, Compressed::NO, )?, - p2wshindex_to_p2wshaddressbytes: StorableVec::forced_import( + p2wshindex_to_p2wshaddressbytes: IndexedVec::forced_import( &path.join("p2wshindex_to_p2wshaddressbytes"), Version::ONE, Compressed::NO, )?, - txindex_to_first_txinindex: StorableVec::forced_import( + txindex_to_first_txinindex: IndexedVec::forced_import( &path.join("txindex_to_first_txinindex"), Version::ONE, compressed, )?, - txindex_to_first_txoutindex: StorableVec::forced_import( + txindex_to_first_txoutindex: IndexedVec::forced_import( &path.join("txindex_to_first_txoutindex"), Version::ONE, Compressed::NO, )?, - txindex_to_height: StorableVec::forced_import( + txindex_to_height: IndexedVec::forced_import( &path.join("txindex_to_height"), Version::ONE, compressed, )?, - txindex_to_locktime: StorableVec::forced_import( + txindex_to_locktime: IndexedVec::forced_import( &path.join("txindex_to_locktime"), Version::ONE, compressed, )?, - txindex_to_txid: StorableVec::forced_import( + txindex_to_txid: IndexedVec::forced_import( &path.join("txindex_to_txid"), Version::ONE, Compressed::NO, )?, - txindex_to_base_size: StorableVec::forced_import( + txindex_to_base_size: IndexedVec::forced_import( &path.join("txindex_to_base_size"), Version::ONE, compressed, )?, - txindex_to_total_size: StorableVec::forced_import( + txindex_to_total_size: IndexedVec::forced_import( &path.join("txindex_to_total_size"), Version::ONE, compressed, )?, - txindex_to_is_explicitly_rbf: StorableVec::forced_import( + txindex_to_is_explicitly_rbf: IndexedVec::forced_import( &path.join("txindex_to_is_explicitly_rbf"), Version::ONE, compressed, )?, - txindex_to_txversion: StorableVec::forced_import( + txindex_to_txversion: IndexedVec::forced_import( &path.join("txindex_to_txversion"), Version::ONE, compressed, )?, - txinindex_to_txoutindex: StorableVec::forced_import( + txinindex_to_txoutindex: IndexedVec::forced_import( &path.join("txinindex_to_txoutindex"), Version::ONE, compressed, )?, - txoutindex_to_addressindex: StorableVec::forced_import( + txoutindex_to_addressindex: IndexedVec::forced_import( &path.join("txoutindex_to_addressindex"), Version::ONE, compressed, )?, - txoutindex_to_value: StorableVec::forced_import( + txoutindex_to_value: IndexedVec::forced_import( &path.join("txoutindex_to_value"), Version::ONE, compressed, diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 21dbe513e..6db305e2d 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -25,6 +25,5 @@ oxc = { version = "0.62.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.44.1", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full", "trace"] } -zip = "2.6.0" +zip = "2.6.1" tracing = "0.1.41" -tracing-subscriber = "0.3.19" diff --git a/publish.sh b/publish.sh deleted file mode 100755 index 75bbc856f..000000000 --- a/publish.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -cargo clean - -cargo build --all-targets - -cd crates/brk - -cd ../brk_core -cargo publish - -cd ../brk_exit -cargo publish - -cd ../brk_vec -cargo publish - -cd ../brk_logger -cargo publish - -cd ../brk_indexer -cargo publish - -cd ../brk_parser -cargo publish - -cd ../brk_fetcher -cargo publish - -cd ../brk_computer -cargo publish - -cd ../brk_query -cargo publish - -cd ../brk_server -cargo publish - -cd ../brk_cli -cargo publish - -cd ../brk -cargo publish - -cd ../.. diff --git a/update.sh b/update.sh index 7aaa44ceb..d6951dfc1 100755 --- a/update.sh +++ b/update.sh @@ -4,3 +4,4 @@ cargo clean rustup update cargo upgrade --incompatible cargo update +cargo build --all-targets diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 68afdbdfd..df1f3a48f 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -124,7 +124,7 @@ export function createVecIdToIndexes() { p2wshaddressbytes: [P2WSHindex], quarterindex: [Monthindex, Quarterindex], "real-date": [Height], - "sats-per-dollar": [Dateindex, Height], + "sats-per-dollar": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], size: [Height], timestamp: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch, Halvingepoch], "total-block-count": [Dateindex],