diff --git a/Cargo.lock b/Cargo.lock index 279e430ff..bcefebcd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2321,9 +2321,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] diff --git a/crates/brk_computer/src/storage/vecs/base.rs b/crates/brk_computer/src/storage/vecs/base.rs index 849956dc5..7557989d2 100644 --- a/crates/brk_computer/src/storage/vecs/base.rs +++ b/crates/brk_computer/src/storage/vecs/base.rs @@ -6,7 +6,7 @@ use std::{ path::{Path, PathBuf}, }; -use brk_core::CheckedSub; +use brk_core::{Bitcoin, CheckedSub, Close, Dollars, Height, Sats, Txindex}; use brk_exit::Exit; use brk_vec::{ Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version, @@ -419,6 +419,78 @@ where } } +impl ComputedVec +where + I: StoredIndex, +{ + pub fn compute_from_sats( + &mut self, + max_from: I, + sats: &mut StoredVec, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.version() + sats.version(), + )?; + + let index = max_from.min(I::from(self.len())); + sats.iter_from(index, |(i, sats, ..)| { + let (i, v) = (i, Bitcoin::from(sats)); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit) + } +} + +impl ComputedVec { + pub fn compute_from_bitcoin( + &mut self, + max_from: Height, + bitcoin: &mut StoredVec, + price: &mut StoredVec>, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.version() + bitcoin.version(), + )?; + + let index = max_from.min(Height::from(self.len())); + bitcoin.iter_from(index, |(i, bitcoin, ..)| { + let dollars = price.double_unwrap_cached_get(i); + let (i, v) = (i, *dollars * bitcoin); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit) + } +} + +impl ComputedVec { + pub fn compute_from_bitcoin( + &mut self, + max_from: Txindex, + bitcoin: &mut StoredVec, + i_to_height: &mut StoredVec, + price: &mut StoredVec>, + exit: &Exit, + ) -> Result<()> { + self.validate_computed_version_or_reset_file( + Version::ZERO + self.version() + bitcoin.version(), + )?; + + let index = max_from.min(Txindex::from(self.len())); + bitcoin.iter_from(index, |(i, bitcoin, ..)| { + let height = i_to_height.double_unwrap_cached_get(i); + let dollars = price.double_unwrap_cached_get(height); + let (i, v) = (i, *dollars * bitcoin); + self.forced_push_at(i, v, exit) + })?; + + self.safe_flush(exit) + } +} + impl Clone for ComputedVec where I: StoredIndex, diff --git a/crates/brk_computer/src/storage/vecs/grouped/builder.rs b/crates/brk_computer/src/storage/vecs/grouped/builder.rs index a411dcb76..84835ad53 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/builder.rs @@ -30,6 +30,8 @@ where total: Option>, } +const VERSION: Version = Version::ZERO; + impl ComputedVecBuilder where I: StoredIndex, @@ -38,6 +40,7 @@ where pub fn forced_import( path: &Path, name: &str, + version: Version, compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { @@ -67,50 +70,100 @@ where } }; + let version = VERSION + version; + let s = Self { first: options.first.then(|| { - ComputedVec::forced_import(&maybe_prefix("first"), Version::ZERO, compressed) - .unwrap() + ComputedVec::forced_import( + &maybe_prefix("first"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), last: options.last.then(|| { ComputedVec::forced_import( &path.join(format!("{key}_to_{name}")), - Version::ZERO, + version + Version::ZERO, compressed, ) .unwrap() }), min: options.min.then(|| { - ComputedVec::forced_import(&maybe_suffix("min"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("min"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), max: options.max.then(|| { - ComputedVec::forced_import(&maybe_suffix("max"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("max"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), median: options.median.then(|| { - ComputedVec::forced_import(&maybe_suffix("median"), Version::ZERO, compressed) - .unwrap() + ComputedVec::forced_import( + &maybe_suffix("median"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), average: options.average.then(|| { - ComputedVec::forced_import(&maybe_suffix("average"), Version::ZERO, compressed) - .unwrap() + ComputedVec::forced_import( + &maybe_suffix("average"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), sum: options.sum.then(|| { - ComputedVec::forced_import(&maybe_suffix("sum"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("sum"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), total: options.total.then(|| { - ComputedVec::forced_import(&prefix("total"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import(&prefix("total"), version + Version::ZERO, compressed) + .unwrap() }), _90p: options._90p.then(|| { - ComputedVec::forced_import(&maybe_suffix("90p"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("90p"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), _75p: options._75p.then(|| { - ComputedVec::forced_import(&maybe_suffix("75p"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("75p"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), _25p: options._25p.then(|| { - ComputedVec::forced_import(&maybe_suffix("25p"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("25p"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), _10p: options._10p.then(|| { - ComputedVec::forced_import(&maybe_suffix("10p"), Version::ZERO, compressed).unwrap() + ComputedVec::forced_import( + &maybe_suffix("10p"), + version + Version::ZERO, + compressed, + ) + .unwrap() }), }; diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs index 3bfaa121f..bf446fe01 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs @@ -23,6 +23,8 @@ where pub decadeindex: ComputedVecBuilder, } +const VERSION: Version = Version::ZERO; + impl ComputedVecsFromDateindex where T: ComputedType + Ord + From, @@ -35,8 +37,15 @@ where compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let dateindex_extra = - ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?; + let version = VERSION + version; + + let dateindex_extra = ComputedVecBuilder::forced_import( + path, + name, + version, + compressed, + options.copy_self_extra(), + )?; let options = options.remove_percentiles(); @@ -47,11 +56,17 @@ where compressed, )?, dateindex_extra, - weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + monthindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + quarterindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, }) } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs index a1e1662c0..99e442af5 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height.rs @@ -28,6 +28,8 @@ where pub decadeindex: ComputedVecBuilder, } +const VERSION: Version = Version::ZERO; + impl ComputedVecsFromHeight where T: ComputedType + Ord + From, @@ -41,15 +43,23 @@ where compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { + let version = VERSION + version; + let height = compute_source.then(|| { ComputedVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed) .unwrap() }); - let height_extra = - ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?; + let height_extra = ComputedVecBuilder::forced_import( + path, + name, + version, + compressed, + options.copy_self_extra(), + )?; - let dateindex = ComputedVecBuilder::forced_import(path, name, compressed, options)?; + let dateindex = + ComputedVecBuilder::forced_import(path, name, version, compressed, options)?; let options = options.remove_percentiles(); @@ -57,13 +67,21 @@ where height, height_extra, dateindex, - weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, - decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + difficultyepoch: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + monthindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + quarterindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, }) } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs index 6519a04cc..98251b12e 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs @@ -20,6 +20,8 @@ where // TODO: pub halvingepoch: StorableVecGeneator, } +const VERSION: Version = Version::ZERO; + impl ComputedVecsFromHeightStrict where T: ComputedType + Ord + From, @@ -32,22 +34,31 @@ where compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { + let version = VERSION + version; + let height = ComputedVec::forced_import( &path.join(format!("height_to_{name}")), version, compressed, )?; - let height_extra = - ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?; + let height_extra = ComputedVecBuilder::forced_import( + path, + name, + version, + compressed, + options.copy_self_extra(), + )?; let options = options.remove_percentiles(); Ok(Self { height, height_extra, - difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, + difficultyepoch: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?, }) } diff --git a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs index 9314ecf15..ad9a9d167 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs @@ -29,6 +29,8 @@ where pub decadeindex: ComputedVecBuilder, } +const VERSION: Version = Version::ZERO; + impl ComputedVecsFromTxindex where T: ComputedType + Ord + From, @@ -42,6 +44,8 @@ where compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { + let version = VERSION + version; + let txindex = compute_source.then(|| { ComputedVec::forced_import( &path.join(format!("txindex_to_{name}")), @@ -51,21 +55,29 @@ where .unwrap() }); - let height = ComputedVecBuilder::forced_import(path, name, compressed, options)?; + let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?; let options = options.remove_percentiles(); Ok(Self { txindex, height, - dateindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, - // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?, - decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?, + dateindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + difficultyepoch: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + monthindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + quarterindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, + yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, + // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?, + decadeindex: ComputedVecBuilder::forced_import( + path, name, version, compressed, options, + )?, }) } diff --git a/crates/brk_computer/src/storage/vecs/grouped/mod.rs b/crates/brk_computer/src/storage/vecs/grouped/mod.rs index 3bf2f923b..3ebd53122 100644 --- a/crates/brk_computer/src/storage/vecs/grouped/mod.rs +++ b/crates/brk_computer/src/storage/vecs/grouped/mod.rs @@ -4,6 +4,8 @@ mod from_height; mod from_height_strict; mod from_txindex; mod stored_type; +mod value_from_height; +mod value_from_txindex; pub use builder::*; pub use from_dateindex::*; @@ -11,3 +13,5 @@ pub use from_height::*; pub use from_height_strict::*; pub use from_txindex::*; pub use stored_type::*; +pub use value_from_height::*; +pub use value_from_txindex::*; diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs new file mode 100644 index 000000000..aabf3d38f --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_height.rs @@ -0,0 +1,157 @@ +use std::path::Path; + +use brk_core::{Bitcoin, Dollars, Height, Sats}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; + +use crate::storage::{ + base::ComputedVec, + marketprice, + vecs::{Indexes, indexes}, +}; + +use super::{ComputedVecsFromHeight, StorableVecGeneatorOptions}; + +#[derive(Clone)] +pub struct ComputedValueVecsFromHeight { + pub sats: ComputedVecsFromHeight, + pub bitcoin: ComputedVecsFromHeight, + pub dollars: Option>, +} + +const VERSION: Version = Version::ONE; + +impl ComputedValueVecsFromHeight { + pub fn forced_import( + path: &Path, + name: &str, + compute_source: bool, + version: Version, + compressed: Compressed, + options: StorableVecGeneatorOptions, + compute_dollars: bool, + ) -> color_eyre::Result { + Ok(Self { + sats: ComputedVecsFromHeight::forced_import( + path, + name, + compute_source, + VERSION + version, + compressed, + options, + )?, + bitcoin: ComputedVecsFromHeight::forced_import( + path, + &format!("{name}_in_btc"), + true, + VERSION + version, + compressed, + options, + )?, + dollars: compute_dollars.then(|| { + ComputedVecsFromHeight::forced_import( + path, + &format!("{name}_in_usd"), + true, + VERSION + version, + compressed, + options, + ) + .unwrap() + }), + }) + } + + pub fn compute_all( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + marketprices: &mut Option<&mut marketprice::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + mut compute: F, + ) -> color_eyre::Result<()> + where + F: FnMut( + &mut ComputedVec, + &mut Indexer, + &mut indexes::Vecs, + &Indexes, + &Exit, + ) -> Result<()>, + { + compute( + self.sats.height.as_mut().unwrap(), + indexer, + indexes, + starting_indexes, + exit, + )?; + + self.compute_rest(indexer, indexes, marketprices, starting_indexes, exit, None)?; + + Ok(()) + } + + pub fn compute_rest( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + marketprices: &mut Option<&mut marketprice::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + mut height: Option<&mut StoredVec>, + ) -> color_eyre::Result<()> { + if let Some(height) = height.as_mut() { + self.sats + .compute_rest(indexes, starting_indexes, exit, Some(height))?; + } else { + self.sats + .compute_rest(indexes, starting_indexes, exit, None)?; + } + + let height = height.unwrap_or_else(|| self.sats.height.as_mut().unwrap().mut_vec()); + + self.bitcoin.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_from_sats(starting_indexes.height, height, exit) + }, + )?; + + let txindex = self.bitcoin.height.as_mut().unwrap().mut_vec(); + let price = marketprices + .as_mut() + .unwrap() + .chainindexes_to_close + .height + .mut_vec(); + + if let Some(dollars) = self.dollars.as_mut() { + dollars.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_from_bitcoin(starting_indexes.height, txindex, price, exit) + }, + )?; + } + + Ok(()) + } + + pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + [ + self.sats.any_vecs(), + self.bitcoin.any_vecs(), + self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs new file mode 100644 index 000000000..d17ab2300 --- /dev/null +++ b/crates/brk_computer/src/storage/vecs/grouped/value_from_txindex.rs @@ -0,0 +1,163 @@ +use std::path::Path; + +use brk_core::{Bitcoin, Dollars, Sats, Txindex}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version}; + +use crate::storage::{ + base::ComputedVec, + marketprice, + vecs::{Indexes, indexes}, +}; + +use super::{ComputedVecsFromTxindex, StorableVecGeneatorOptions}; + +#[derive(Clone)] +pub struct ComputedValueVecsFromTxindex { + pub sats: ComputedVecsFromTxindex, + pub bitcoin: ComputedVecsFromTxindex, + pub dollars: Option>, +} + +const VERSION: Version = Version::ONE; + +impl ComputedValueVecsFromTxindex { + pub fn forced_import( + path: &Path, + name: &str, + compute_source: bool, + version: Version, + compressed: Compressed, + options: StorableVecGeneatorOptions, + compute_dollars: bool, + ) -> color_eyre::Result { + Ok(Self { + sats: ComputedVecsFromTxindex::forced_import( + path, + name, + compute_source, + VERSION + version, + compressed, + options, + )?, + bitcoin: ComputedVecsFromTxindex::forced_import( + path, + &format!("{name}_in_btc"), + true, + VERSION + version, + compressed, + options, + )?, + dollars: compute_dollars.then(|| { + ComputedVecsFromTxindex::forced_import( + path, + &format!("{name}_in_usd"), + true, + VERSION + version, + compressed, + options, + ) + .unwrap() + }), + }) + } + + pub fn compute_all( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + marketprices: &mut Option<&mut marketprice::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + mut compute: F, + ) -> color_eyre::Result<()> + where + F: FnMut( + &mut ComputedVec, + &mut Indexer, + &mut indexes::Vecs, + &Indexes, + &Exit, + ) -> Result<()>, + { + compute( + self.sats.txindex.as_mut().unwrap(), + indexer, + indexes, + starting_indexes, + exit, + )?; + + self.compute_rest(indexer, indexes, marketprices, starting_indexes, exit, None)?; + + Ok(()) + } + + pub fn compute_rest( + &mut self, + indexer: &mut Indexer, + indexes: &mut indexes::Vecs, + marketprices: &mut Option<&mut marketprice::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + mut txindex: Option<&mut StoredVec>, + ) -> color_eyre::Result<()> { + if let Some(txindex) = txindex.as_mut() { + self.sats + .compute_rest(indexer, indexes, starting_indexes, exit, Some(txindex))?; + } else { + self.sats + .compute_rest(indexer, indexes, starting_indexes, exit, None)?; + } + + let txindex = txindex.unwrap_or_else(|| self.sats.txindex.as_mut().unwrap().mut_vec()); + + self.bitcoin.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, _, _, starting_indexes, exit| { + v.compute_from_sats(starting_indexes.txindex, txindex, exit) + }, + )?; + + let txindex = self.bitcoin.txindex.as_mut().unwrap().mut_vec(); + let price = marketprices + .as_mut() + .unwrap() + .chainindexes_to_close + .height + .mut_vec(); + + if let Some(dollars) = self.dollars.as_mut() { + dollars.compute_all( + indexer, + indexes, + starting_indexes, + exit, + |v, indexer, _, starting_indexes, exit| { + v.compute_from_bitcoin( + starting_indexes.txindex, + txindex, + indexer.mut_vecs().txindex_to_height.mut_vec(), + price, + exit, + ) + }, + )?; + } + + Ok(()) + } + + pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> { + [ + self.sats.any_vecs(), + self.bitcoin.any_vecs(), + self.dollars.as_ref().map_or(vec![], |v| v.any_vecs()), + ] + .concat() + } +} diff --git a/crates/brk_computer/src/storage/vecs/mod.rs b/crates/brk_computer/src/storage/vecs/mod.rs index a5c133f88..1ba7b1802 100644 --- a/crates/brk_computer/src/storage/vecs/mod.rs +++ b/crates/brk_computer/src/storage/vecs/mod.rs @@ -5,12 +5,12 @@ use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::{AnyStoredVec, Compressed}; -mod base; -mod blocks; -mod grouped; -mod indexes; -mod marketprice; -mod transactions; +pub mod base; +pub mod blocks; +pub mod grouped; +pub mod indexes; +pub mod marketprice; +pub mod transactions; use base::*; use indexes::*; @@ -30,7 +30,7 @@ impl Vecs { Ok(Self { blocks: blocks::Vecs::forced_import(path, compressed)?, indexes: indexes::Vecs::forced_import(path, compressed)?, - transactions: transactions::Vecs::forced_import(path, compressed)?, + transactions: transactions::Vecs::forced_import(path, compressed, fetch)?, marketprice: fetch.then(|| marketprice::Vecs::forced_import(path, compressed).unwrap()), }) } @@ -47,9 +47,6 @@ impl Vecs { self.blocks .compute(indexer, &mut self.indexes, &starting_indexes, exit)?; - self.transactions - .compute(indexer, &mut self.indexes, &starting_indexes, exit)?; - if let Some(marketprice) = self.marketprice.as_mut() { marketprice.compute( indexer, @@ -60,6 +57,14 @@ impl Vecs { )?; } + self.transactions.compute( + indexer, + &mut self.indexes, + &starting_indexes, + &mut self.marketprice.as_mut(), + exit, + )?; + Ok(()) } diff --git a/crates/brk_computer/src/storage/vecs/transactions.rs b/crates/brk_computer/src/storage/vecs/transactions.rs index 927f3ad06..aec977184 100644 --- a/crates/brk_computer/src/storage/vecs/transactions.rs +++ b/crates/brk_computer/src/storage/vecs/transactions.rs @@ -11,14 +11,17 @@ use brk_vec::{Compressed, DynamicVec, StoredIndex, Version}; use super::{ ComputedVec, Indexes, - grouped::{ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions}, - indexes, + grouped::{ + ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight, + ComputedVecsFromTxindex, StorableVecGeneatorOptions, + }, + indexes, marketprice, }; #[derive(Clone)] pub struct Vecs { pub indexes_to_tx_count: ComputedVecsFromHeight, - pub indexes_to_fee: ComputedVecsFromTxindex, + pub indexes_to_fee: ComputedValueVecsFromTxindex, pub indexes_to_feerate: ComputedVecsFromTxindex, pub indexes_to_input_value: ComputedVecsFromTxindex, pub indexes_to_output_value: ComputedVecsFromTxindex, @@ -37,12 +40,16 @@ pub struct Vecs { pub txindex_to_weight: ComputedVec, /// Value == 0 when Coinbase pub txinindex_to_value: ComputedVec, - pub indexes_to_subsidy: ComputedVecsFromHeight, - pub indexes_to_coinbase: ComputedVecsFromHeight, + pub indexes_to_subsidy: ComputedValueVecsFromHeight, + pub indexes_to_coinbase: ComputedValueVecsFromHeight, } impl Vecs { - pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result { + pub fn forced_import( + path: &Path, + compressed: Compressed, + compute_dollars: bool, + ) -> color_eyre::Result { fs::create_dir_all(path)?; Ok(Self { @@ -142,7 +149,7 @@ impl Vecs { .add_sum() .add_total(), )?, - indexes_to_fee: ComputedVecsFromTxindex::forced_import( + indexes_to_fee: ComputedValueVecsFromTxindex::forced_import( path, "fee", true, @@ -154,6 +161,7 @@ impl Vecs { .add_percentiles() .add_minmax() .add_average(), + compute_dollars, )?, indexes_to_feerate: ComputedVecsFromTxindex::forced_import( path, @@ -198,7 +206,7 @@ impl Vecs { .add_minmax() .add_average(), )?, - indexes_to_subsidy: ComputedVecsFromHeight::forced_import( + indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import( path, "subsidy", true, @@ -210,8 +218,9 @@ impl Vecs { .add_total() .add_minmax() .add_average(), + compute_dollars, )?, - indexes_to_coinbase: ComputedVecsFromHeight::forced_import( + indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import( path, "coinbase", true, @@ -223,6 +232,7 @@ impl Vecs { .add_percentiles() .add_minmax() .add_average(), + compute_dollars, )?, }) } @@ -232,6 +242,7 @@ impl Vecs { indexer: &mut Indexer, indexes: &mut indexes::Vecs, starting_indexes: &Indexes, + marketprices: &mut Option<&mut marketprice::Vecs>, exit: &Exit, ) -> color_eyre::Result<()> { self.indexes_to_tx_count.compute_all( @@ -403,6 +414,7 @@ impl Vecs { self.indexes_to_fee.compute_all( indexer, indexes, + marketprices, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { @@ -441,7 +453,7 @@ impl Vecs { |vec, _, _, starting_indexes, exit| { vec.compute_transform( starting_indexes.txindex, - self.indexes_to_fee.txindex.as_mut().unwrap().mut_vec(), + self.indexes_to_fee.sats.txindex.as_mut().unwrap().mut_vec(), |(txindex, fee, ..)| { let vsize = self .txindex_to_vsize @@ -474,6 +486,7 @@ impl Vecs { self.indexes_to_subsidy.compute_all( indexer, indexes, + marketprices, starting_indexes, exit, |vec, indexer, indexes, starting_indexes, exit| { @@ -507,15 +520,22 @@ impl Vecs { self.indexes_to_coinbase.compute_all( indexer, indexes, + marketprices, starting_indexes, exit, |vec, _, _, starting_indexes, exit| { vec.compute_transform( starting_indexes.height, - self.indexes_to_subsidy.height.as_mut().unwrap().mut_vec(), + self.indexes_to_subsidy + .sats + .height + .as_mut() + .unwrap() + .mut_vec(), |(height, subsidy, ..)| { let fees = self .indexes_to_fee + .sats .height .unwrap_sum() .mut_vec() diff --git a/crates/brk_core/src/structs/bitcoin.rs b/crates/brk_core/src/structs/bitcoin.rs index 754d2e597..90ef59e44 100644 --- a/crates/brk_core/src/structs/bitcoin.rs +++ b/crates/brk_core/src/structs/bitcoin.rs @@ -1,10 +1,32 @@ -use std::ops::Mul; +use std::ops::{Add, Div, Mul}; + +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Sats; -#[derive(Debug, Default, Clone, Copy)] +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + PartialOrd, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] pub struct Bitcoin(f64); +impl Add for Bitcoin { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + impl Mul for Bitcoin { type Output = Self; fn mul(self, rhs: Self) -> Self::Output { @@ -12,6 +34,13 @@ impl Mul for Bitcoin { } } +impl Div for Bitcoin { + type Output = Self; + fn div(self, rhs: usize) -> Self::Output { + Self(self.0 / rhs as f64) + } +} + impl From for Bitcoin { fn from(value: Sats) -> Self { Self(u64::from(value) as f64 / (u64::from(Sats::ONE_BTC) as f64)) @@ -29,3 +58,18 @@ impl From for f64 { value.0 } } + +impl From for Bitcoin { + fn from(value: usize) -> Self { + Self(value as f64) + } +} + +impl Eq for Bitcoin {} + +#[allow(clippy::derive_ord_xor_partial_ord)] +impl Ord for Bitcoin { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.partial_cmp(&other.0).unwrap() + } +} diff --git a/crates/brk_core/src/structs/cents.rs b/crates/brk_core/src/structs/cents.rs index 97fb9460a..1ff3382e4 100644 --- a/crates/brk_core/src/structs/cents.rs +++ b/crates/brk_core/src/structs/cents.rs @@ -22,7 +22,7 @@ pub struct Cents(u64); impl From for Cents { fn from(value: Dollars) -> Self { - Self((*value * 100.0).floor() as u64) + Self((*value * 100.0).round() as u64) } } @@ -31,3 +31,15 @@ impl From for f64 { value.0 as f64 } } + +impl From for Cents { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl From for u64 { + fn from(value: Cents) -> Self { + value.0 + } +} diff --git a/crates/brk_core/src/structs/dollars.rs b/crates/brk_core/src/structs/dollars.rs index 993bf789a..53e8a7db2 100644 --- a/crates/brk_core/src/structs/dollars.rs +++ b/crates/brk_core/src/structs/dollars.rs @@ -1,10 +1,10 @@ -use std::ops::{Add, Div}; +use std::ops::{Add, Div, Mul}; use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Cents; +use super::{Bitcoin, Cents, Sats}; #[derive( Debug, @@ -68,3 +68,12 @@ impl Ord for Dollars { self.0.partial_cmp(&other.0).unwrap() } } + +impl Mul for Dollars { + type Output = Dollars; + fn mul(self, rhs: Bitcoin) -> Self::Output { + Self::from(Cents::from( + u64::from(Sats::from(rhs)) * u64::from(Cents::from(self)) / u64::from(Sats::ONE_BTC), + )) + } +} diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index e3f608560..7a3daaf6c 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -141,7 +141,7 @@ impl From for Amount { impl From for Sats { fn from(value: Bitcoin) -> Self { - Self((f64::from(value) * (u64::from(Sats::ONE_BTC) as f64)) as u64) + Self((f64::from(value) * (u64::from(Sats::ONE_BTC) as f64)).round() as u64) } } diff --git a/crates/brk_parser/README.md b/crates/brk_parser/README.md index 91eea57e5..ce85fc10e 100644 --- a/crates/brk_parser/README.md +++ b/crates/brk_parser/README.md @@ -64,6 +64,6 @@ A state of the local chain is saved in `{bitcoindir}/blocks/blk_index_to_blk_rec | `0..=855_000` | 4mn 10s | 4mn 45s | > 2h | | `800_000..=855_000` | 0mn 52s (4mn 10s if first run) | 0mn 55s | > 2h | -\* `blocks_iterator` is with the default config (and thus with `skip_prevout = false` which does a lot more than just iterate over blocks) so it isn't an apples to apples comparaison. Will update the benchmark with `skip_prevout = true` as soon as possible. +\* `blocks_iterator` is with the default config (and thus with `skip_prevout = false` which does a lot more than just iterate over blocks) so it isn't an apples to apples comparaison and the numbers are misleading. You should expect much closer times. Will update the benchmark with `skip_prevout = true` as soon as possible. *Benchmarked on a Macbook Pro M3 Pro* diff --git a/crates/brk_server/src/api/query/dts.rs b/crates/brk_server/src/api/query/dts.rs index c3613ebf9..ee46202fb 100644 --- a/crates/brk_server/src/api/query/dts.rs +++ b/crates/brk_server/src/api/query/dts.rs @@ -32,7 +32,7 @@ impl DTS for Query<'static> { let indexes = Index::all(); let mut contents = "// -// File auto-generated, any modification will be overwritten +// File auto-generated, any modifications will be overwritten //\n\n" .to_string(); diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index 10f75822c..e4a887d68 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -1,5 +1,5 @@ // -// File auto-generated, any modification will be overwritten +// File auto-generated, any modifications will be overwritten // /** @typedef {0} Height */ @@ -84,6 +84,26 @@ export function createVecIdToIndexes() { "coinbase-75p": [Dateindex], "coinbase-90p": [Dateindex], "coinbase-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-btc": [Height], + "coinbase-in-btc-10p": [Dateindex], + "coinbase-in-btc-25p": [Dateindex], + "coinbase-in-btc-75p": [Dateindex], + "coinbase-in-btc-90p": [Dateindex], + "coinbase-in-btc-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-btc-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-btc-median": [Dateindex], + "coinbase-in-btc-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-btc-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-usd": [Height], + "coinbase-in-usd-10p": [Dateindex], + "coinbase-in-usd-25p": [Dateindex], + "coinbase-in-usd-75p": [Dateindex], + "coinbase-in-usd-90p": [Dateindex], + "coinbase-in-usd-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-usd-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-usd-median": [Dateindex], + "coinbase-in-usd-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "coinbase-in-usd-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "coinbase-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "coinbase-median": [Dateindex], "coinbase-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], @@ -99,6 +119,26 @@ export function createVecIdToIndexes() { "fee-75p": [Height], "fee-90p": [Height], "fee-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-btc": [Txindex], + "fee-in-btc-10p": [Height], + "fee-in-btc-25p": [Height], + "fee-in-btc-75p": [Height], + "fee-in-btc-90p": [Height], + "fee-in-btc-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-btc-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-btc-median": [Height], + "fee-in-btc-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-btc-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-usd": [Txindex], + "fee-in-usd-10p": [Height], + "fee-in-usd-25p": [Height], + "fee-in-usd-75p": [Height], + "fee-in-usd-90p": [Height], + "fee-in-usd-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-usd-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-usd-median": [Height], + "fee-in-usd-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "fee-in-usd-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "fee-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "fee-median": [Height], "fee-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], @@ -198,6 +238,26 @@ export function createVecIdToIndexes() { "subsidy-75p": [Dateindex], "subsidy-90p": [Dateindex], "subsidy-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-btc": [Height], + "subsidy-in-btc-10p": [Dateindex], + "subsidy-in-btc-25p": [Dateindex], + "subsidy-in-btc-75p": [Dateindex], + "subsidy-in-btc-90p": [Dateindex], + "subsidy-in-btc-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-btc-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-btc-median": [Dateindex], + "subsidy-in-btc-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-btc-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-usd": [Height], + "subsidy-in-usd-10p": [Dateindex], + "subsidy-in-usd-25p": [Dateindex], + "subsidy-in-usd-75p": [Dateindex], + "subsidy-in-usd-90p": [Dateindex], + "subsidy-in-usd-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-usd-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-usd-median": [Dateindex], + "subsidy-in-usd-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "subsidy-in-usd-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "subsidy-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "subsidy-median": [Dateindex], "subsidy-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], @@ -208,13 +268,19 @@ export function createVecIdToIndexes() { "total-block-vbytes": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-block-weight": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-coinbase": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-coinbase-in-btc": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-coinbase-in-usd": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-fee": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-fee-in-btc": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-fee-in-usd": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-input-value": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-output-value": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-size": [Height, Txindex], "total-subsidy": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-subsidy-in-btc": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], + "total-subsidy-in-usd": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-tx-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-tx-v1": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch], "total-tx-v2": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],