diff --git a/app/src/scripts/presets/blocks/index.ts b/app/src/scripts/presets/blocks/index.ts index 07ab30861..b1dc02ef8 100644 --- a/app/src/scripts/presets/blocks/index.ts +++ b/app/src/scripts/presets/blocks/index.ts @@ -139,6 +139,45 @@ export function createPresets(scale: ResourceScale) { }, ], }, + { + scale, + name: "Mined", + tree: [ + { + scale, + icon: IconTablerCube, + name: "Daily Sum", + title: "Daily Sum Of Blocks Mined", + description: "", + bottom: [ + { + title: "Target", + color: colors.white, + datasetPath: `/date-to-blocks-mined-1d-target`, + options: { + lineStyle: 3, + }, + }, + { + title: "1W Avg.", + color: colors.momentumYellow, + datasetPath: `/date-to-blocks-mined-1w-sma`, + defaultVisible: false, + }, + { + title: "1M Avg.", + color: colors.bitcoin, + datasetPath: `/date-to-blocks-mined-1m-sma`, + }, + { + title: "Mined", + color: colors.darkBitcoin, + datasetPath: `/date-to-blocks-mined`, + }, + ], + }, + ], + }, ] : [ { diff --git a/parser/Cargo.lock b/parser/Cargo.lock index b693bd112..8b50e4404 100644 --- a/parser/Cargo.lock +++ b/parser/Cargo.lock @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" dependencies = [ "serde", "serde_spanned", @@ -1881,9 +1881,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" dependencies = [ "indexmap", "serde", diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 5f7cfc6f1..116c5381b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -30,4 +30,4 @@ reqwest = { version = "0.12.5", features = ["blocking", "json"] } sanakirja = "1.4.2" serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.120" -toml = "0.8.14" +toml = "0.8.15" diff --git a/parser/src/databases/address_index_to_address_data.rs b/parser/src/databases/address_index_to_address_data.rs index a516f3246..c2422d21b 100644 --- a/parser/src/databases/address_index_to_address_data.rs +++ b/parser/src/databases/address_index_to_address_data.rs @@ -12,7 +12,7 @@ use crate::{ utils::time, }; -use super::{databases_folder_path, AnyDatabaseGroup, Metadata, SizedDatabase}; +use super::{AnyDatabaseGroup, Metadata, SizedDatabase}; type Key = u32; type Value = AddressData; diff --git a/parser/src/datasets/mining.rs b/parser/src/datasets/mining.rs index e49203657..bad7c8bfb 100644 --- a/parser/src/datasets/mining.rs +++ b/parser/src/datasets/mining.rs @@ -1,15 +1,20 @@ use allocative::Allocative; +use itertools::Itertools; +use ordered_float::OrderedFloat; use crate::{ bitcoin::TARGET_BLOCKS_PER_DAY, datasets::AnyDataset, structs::{ - Amount, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey, + date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap, + AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey, }, utils::{BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, }; -use super::{ComputeData, InsertData, MinInitialStates}; +use super::{ + ComputeData, DateRecapDataset, InsertData, MinInitialStates, RecapDataset, RecapOptions, +}; #[derive(Allocative)] pub struct MiningDataset { @@ -50,10 +55,14 @@ pub struct MiningDataset { pub last_subsidy: DateMap, pub last_subsidy_in_dollars: DateMap, pub difficulty: BiMap, - pub block_size: HeightMap, // in MB - pub block_weight: HeightMap, // in MB + pub block_size: HeightMap, // in MB + pub block_size_recap: DateRecapDataset, // in MB + pub block_weight: HeightMap, // in MB + pub block_weight_recap: DateRecapDataset, // in MB pub block_vbytes: HeightMap, - pub block_interval: HeightMap, // in s + pub block_vbytes_recap: DateRecapDataset, + pub block_interval: HeightMap, // in s + pub block_interval_recap: DateRecapDataset, // in s // Computed pub annualized_issuance: BiMap, // Same as subsidy_1y_sum @@ -173,19 +182,59 @@ impl MiningDataset { difficulty: BiMap::new_bin(1, &f("difficulty")), difficulty_adjustment: DateMap::new_bin(1, &f("difficulty_adjustment")), block_size: HeightMap::new_bin(1, &f("block_size")), - // - // block_size_1d_sma: HeightMap::new_bin(1, &f("block_size")), - // block_size_1d_median: HeightMap::new_bin(1, &f("block_size")), - // + block_size_recap: RecapDataset::import( + &f("block_size_1d"), + RecapOptions::default() + .add_average() + .add_max() + .add_90p() + .add_75p() + .add_median() + .add_25p() + .add_10p() + .add_min(), + )?, cumulative_block_size: BiMap::new_bin(1, &f("cumulative_block_size")), block_weight: HeightMap::new_bin(1, &f("block_weight")), - // - // block_weight_1d_sma: HeightMap::new_bin(1, &f("block_weight")), + block_weight_recap: RecapDataset::import( + &f("block_weight_1d"), + RecapOptions::default() + .add_average() + .add_max() + .add_90p() + .add_75p() + .add_median() + .add_25p() + .add_10p() + .add_min(), + )?, block_vbytes: HeightMap::new_bin(1, &f("block_vbytes")), + block_vbytes_recap: RecapDataset::import( + &f("block_vbytes_1d"), + RecapOptions::default() + .add_average() + .add_max() + .add_90p() + .add_75p() + .add_median() + .add_25p() + .add_10p() + .add_min(), + )?, // block_vbytes_1d_sma: HeightMap::new_bin(1, &f("block_vbytes")), block_interval: HeightMap::new_bin(2, &f("block_interval")), - // block_interval_1d_sma: HeightMap::new_bin(2, &f("block_interval")), - // + block_interval_recap: RecapDataset::import( + &f("block_interval_1d"), + RecapOptions::default() + .add_average() + .add_max() + .add_90p() + .add_75p() + .add_median() + .add_25p() + .add_10p() + .add_min(), + )?, hash_rate: DateMap::new_bin(1, &f("hash_rate")), hash_rate_1w_sma: DateMap::new_bin(1, &f("hash_rate_1w_sma")), hash_rate_1m_sma: DateMap::new_bin(1, &f("hash_rate_1m_sma")), @@ -295,6 +344,7 @@ impl MiningDataset { pub fn compute( &mut self, &ComputeData { heights, dates, .. }: &ComputeData, + first_height: &mut DateMap, last_height: &mut DateMap, ) { self.blocks_mined_1w_sum.multi_insert_last_x_sum( @@ -491,6 +541,38 @@ impl MiningDataset { &mut self.difficulty.date, ONE_DAY_IN_DAYS, ); + + dates.iter().for_each(|date| { + let first = first_height.get_or_import(date).unwrap(); + let last = last_height.get_or_import(date).unwrap(); + + self.block_size_recap.compute( + *date, + &mut self.block_vbytes.get_or_import_range_inclusive(first, last), + ); + + self.block_weight_recap.compute( + *date, + &mut self + .block_weight + .get_or_import_range_inclusive(first, last) + .into_iter() + .map(OrderedFloat) + .collect_vec(), + ); + + self.block_vbytes_recap.compute( + *date, + &mut self.block_vbytes.get_or_import_range_inclusive(first, last), + ); + + self.block_interval_recap.compute( + *date, + &mut self + .block_interval + .get_or_import_range_inclusive(first, last), + ); + }) } } @@ -600,8 +682,8 @@ impl AnyDataset for MiningDataset { } fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { - vec![ - &self.blocks_mined_1d_target, + [ + &self.blocks_mined_1d_target as &(dyn AnyDateMap + Send + Sync), &self.blocks_mined_1w_sma, &self.blocks_mined_1m_sma, &self.blocks_mined_1w_sum, @@ -625,11 +707,25 @@ impl AnyDataset for MiningDataset { &self.puell_multiple, &self.difficulty_adjustment, ] + .into_iter() + .chain(date_map_vec_to_any_date_map_vec( + self.block_size_recap.as_vec(), + )) + .chain(date_map_vec_to_any_date_map_vec( + self.block_vbytes_recap.as_vec(), + )) + .chain(date_map_vec_to_any_date_map_vec( + self.block_weight_recap.as_vec(), + )) + .chain(date_map_vec_to_any_date_map_vec( + self.block_interval_recap.as_vec(), + )) + .collect_vec() } fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { - vec![ - &mut self.blocks_mined_1d_target, + [ + &mut self.blocks_mined_1d_target as &mut dyn AnyDateMap, &mut self.blocks_mined_1w_sma, &mut self.blocks_mined_1m_sma, &mut self.blocks_mined_1w_sum, @@ -653,5 +749,19 @@ impl AnyDataset for MiningDataset { &mut self.puell_multiple, &mut self.difficulty_adjustment, ] + .into_iter() + .chain(date_map_vec_to_mut_any_date_map_vec( + self.block_size_recap.as_mut_vec(), + )) + .chain(date_map_vec_to_mut_any_date_map_vec( + self.block_vbytes_recap.as_mut_vec(), + )) + .chain(date_map_vec_to_mut_any_date_map_vec( + self.block_weight_recap.as_mut_vec(), + )) + .chain(date_map_vec_to_mut_any_date_map_vec( + self.block_interval_recap.as_mut_vec(), + )) + .collect_vec() } } diff --git a/parser/src/datasets/mod.rs b/parser/src/datasets/mod.rs index c432b6dc7..12a23bd57 100644 --- a/parser/src/datasets/mod.rs +++ b/parser/src/datasets/mod.rs @@ -199,8 +199,11 @@ impl AllDatasets { } if self.mining.should_compute(&compute_data) { - self.mining - .compute(&compute_data, &mut self.date_metadata.last_height); + self.mining.compute( + &compute_data, + &mut self.date_metadata.first_height, + &mut self.date_metadata.last_height, + ); } // No compute needed for now diff --git a/parser/src/datasets/subs/recap.rs b/parser/src/datasets/subs/recap.rs index 1b877574e..d902b2d88 100644 --- a/parser/src/datasets/subs/recap.rs +++ b/parser/src/datasets/subs/recap.rs @@ -1,32 +1,26 @@ +use std::{iter::Sum, ops::Add}; + use allocative::Allocative; use crate::{ - datasets::{AnyDataset, ComputeData, MinInitialStates}, - structs::{AnyDateMap, MapValue}, - DateMap, HeightMap, + structs::{DateMapChunkId, GenericMap, MapKey, MapSerialized, MapValue}, + utils::{get_percentile, LossyFrom}, + Date, MapChunkId, SerializedBTreeMap, }; -#[derive(Allocative)] -pub enum RecapTime { - Insert, - Compute, -} +pub type DateRecapDataset = RecapDataset>; #[derive(Allocative)] -pub struct RecapDataset { - min_initial_states: MinInitialStates, - time: RecapTime, - - // Computed - average: Option>, - sum: Option>, - max: Option>, - _90p: Option>, - _75p: Option>, - median: Option>, - _25p: Option>, - _10p: Option>, - min: Option>, +pub struct RecapDataset { + average: Option>, + sum: Option>, + max: Option>, + _90p: Option>, + _75p: Option>, + median: Option>, + _25p: Option>, + _10p: Option>, + min: Option>, } #[derive(Default)] @@ -43,132 +37,141 @@ pub struct RecapOptions { } impl RecapOptions { - pub fn add_min(&mut self) { + pub fn add_min(mut self) -> Self { self.min = true; + self } - pub fn add_max(&mut self) { + pub fn add_max(mut self) -> Self { self.max = true; + self } - pub fn add_median(&mut self) { + pub fn add_median(mut self) -> Self { self.median = true; + self } - pub fn add_average(&mut self) { + pub fn add_average(mut self) -> Self { self.average = true; + self } - pub fn add_sum(&mut self) { + #[allow(unused)] + pub fn add_sum(mut self) -> Self { self.sum = true; + self } - pub fn add_90p(&mut self) { + pub fn add_90p(mut self) -> Self { self._90p = true; + self } - pub fn add_75p(&mut self) { + pub fn add_75p(mut self) -> Self { self._75p = true; + self } - pub fn add_25p(&mut self) { + pub fn add_25p(mut self) -> Self { self._25p = true; + self } - pub fn add_10p(&mut self) { + pub fn add_10p(mut self) -> Self { self._10p = true; + self } } -impl RecapDataset +impl RecapDataset where - T: MapValue, + Value: MapValue, + ChunkId: MapChunkId, + Key: MapKey, + Serialized: MapSerialized, { - pub fn import( - parent_path: &str, - time: RecapTime, - options: RecapOptions, - ) -> color_eyre::Result { + pub fn import(parent_path: &str, options: RecapOptions) -> color_eyre::Result { let f = |s: &str| format!("{parent_path}/{s}"); - let mut s = Self { - min_initial_states: MinInitialStates::default(), - time, - - min: options.min.then(|| DateMap::new_bin(1, &f("min"))), - max: options.max.then(|| DateMap::new_bin(1, &f("max"))), - median: options.median.then(|| DateMap::new_bin(1, &f("median"))), - average: options.average.then(|| DateMap::new_bin(1, &f("average"))), - sum: options.sum.then(|| DateMap::new_bin(1, &f("sum"))), - _90p: options._90p.then(|| DateMap::new_bin(1, &f("90p"))), - _75p: options._75p.then(|| DateMap::new_bin(1, &f("75p"))), - _25p: options._25p.then(|| DateMap::new_bin(1, &f("25p"))), - _10p: options._10p.then(|| DateMap::new_bin(1, &f("10p"))), + let s = Self { + min: options.min.then(|| GenericMap::new_bin(1, &f("min"))), + max: options.max.then(|| GenericMap::new_bin(1, &f("max"))), + median: options.median.then(|| GenericMap::new_bin(1, &f("median"))), + average: options + .average + .then(|| GenericMap::new_bin(1, &f("average"))), + sum: options.sum.then(|| GenericMap::new_bin(1, &f("sum"))), + _90p: options._90p.then(|| GenericMap::new_bin(1, &f("90p"))), + _75p: options._75p.then(|| GenericMap::new_bin(1, &f("75p"))), + _25p: options._25p.then(|| GenericMap::new_bin(1, &f("25p"))), + _10p: options._10p.then(|| GenericMap::new_bin(1, &f("10p"))), }; - s.min_initial_states - .consume(MinInitialStates::compute_from_dataset(&s)); - Ok(s) } - pub fn compute( - &mut self, - &ComputeData { heights, dates, .. }: &ComputeData, - source: &mut HeightMap, - ) { - dates.iter().enumerate().for_each(|(index, date)| { - // let heights = heights_by_date.get(index).unwrap(); + pub fn compute<'a, Value2>(&mut self, key: Key, values: &'a mut [Value2]) + where + Value: LossyFrom + LossyFrom, + Value2: Sum<&'a Value2> + Ord + Add + Clone + Copy + LossyFrom, + f32: LossyFrom + LossyFrom, + { + if self.max.is_some() + || self._90p.is_some() + || self._75p.is_some() + || self.median.is_some() + || self._25p.is_some() + || self._10p.is_some() + || self.min.is_some() + { + values.sort_unstable(); - if let Some(sum) = self.sum.as_ref() { - // v.push(sum); + if let Some(max) = self.max.as_mut() { + max.insert(key, Value::lossy_from(*values.last().unwrap())); } - if let Some(average) = self.average.as_ref() { - // v.push(average); + if let Some(_90p) = self._90p.as_mut() { + _90p.insert(key, Value::lossy_from(get_percentile(values, 0.90))); } - if let Some(max) = self.max.as_ref() { - // v.push(max); + if let Some(_75p) = self._75p.as_mut() { + _75p.insert(key, Value::lossy_from(get_percentile(values, 0.75))); } - if let Some(_90p) = self._90p.as_ref() { - // v.push(_90p); + if let Some(median) = self.median.as_mut() { + median.insert(key, Value::lossy_from(get_percentile(values, 0.50))); } - if let Some(_75p) = self._75p.as_ref() { - // v.push(_75p); + if let Some(_25p) = self._25p.as_mut() { + _25p.insert(key, Value::lossy_from(get_percentile(values, 0.25))); } - if let Some(median) = self.median.as_ref() { - // v.push(median); + if let Some(_10p) = self._10p.as_mut() { + _10p.insert(key, Value::lossy_from(get_percentile(values, 0.10))); } - if let Some(_25p) = self._25p.as_ref() { - // v.push(_25p); + if let Some(min) = self.min.as_mut() { + min.insert(key, Value::lossy_from(*values.first().unwrap())); + } + } + + if self.sum.is_some() || self.average.is_some() { + let sum = Value::lossy_from(values.iter().sum::()); + + if let Some(sum_map) = self.sum.as_mut() { + sum_map.insert(key, sum); } - if let Some(_10p) = self._10p.as_ref() { - // v.push(_10p); + if let Some(average) = self.average.as_mut() { + let len = values.len() as f32; + average.insert(key, Value::lossy_from(f32::lossy_from(sum) / len)); } - - if let Some(min) = self.min.as_ref() { - // v.push(min); - } - }); - } -} - -impl AnyDataset for RecapDataset -where - T: MapValue, -{ - fn get_min_initial_states(&self) -> &MinInitialStates { - &self.min_initial_states + } } - fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { - let mut v: Vec<&(dyn AnyDateMap + Send + Sync)> = vec![]; + pub fn as_vec(&self) -> Vec<&GenericMap> { + let mut v = vec![]; if let Some(min) = self.min.as_ref() { v.push(min); @@ -209,8 +212,8 @@ where v } - fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { - let mut v: Vec<&mut dyn AnyDateMap> = vec![]; + pub fn as_mut_vec(&mut self) -> Vec<&mut GenericMap> { + let mut v = vec![]; if let Some(min) = self.min.as_mut() { v.push(min); @@ -251,3 +254,99 @@ where v } } + +// impl AnyDataset for RecapDataset +// where +// Value: MapValue, +// ChunkId: MapChunkId, +// Key: MapKey, +// Serialized: MapSerialized, +// { +// fn get_min_initial_states(&self) -> &MinInitialStates { +// &self.min_initial_states +// } + +// fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { +// let mut v: Vec<&(dyn AnyDateMap + Send + Sync)> = vec![]; + +// if let Some(min) = self.min.as_ref() { +// v.push(min); +// } + +// if let Some(max) = self.max.as_ref() { +// v.push(max); +// } + +// if let Some(median) = self.median.as_ref() { +// v.push(median); +// } + +// if let Some(average) = self.average.as_ref() { +// v.push(average); +// } + +// if let Some(sum) = self.sum.as_ref() { +// v.push(sum); +// } + +// if let Some(_90p) = self._90p.as_ref() { +// v.push(_90p); +// } + +// if let Some(_75p) = self._75p.as_ref() { +// v.push(_75p); +// } + +// if let Some(_25p) = self._25p.as_ref() { +// v.push(_25p); +// } + +// if let Some(_10p) = self._10p.as_ref() { +// v.push(_10p); +// } + +// v +// } + +// fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { +// let mut v: Vec<&mut dyn AnyDateMap> = vec![]; + +// if let Some(min) = self.min.as_mut() { +// v.push(min); +// } + +// if let Some(max) = self.max.as_mut() { +// v.push(max); +// } + +// if let Some(median) = self.median.as_mut() { +// v.push(median); +// } + +// if let Some(average) = self.average.as_mut() { +// v.push(average); +// } + +// if let Some(sum) = self.sum.as_mut() { +// v.push(sum); +// } + +// if let Some(_90p) = self._90p.as_mut() { +// v.push(_90p); +// } + +// if let Some(_75p) = self._75p.as_mut() { +// v.push(_75p); +// } + +// if let Some(_25p) = self._25p.as_mut() { +// v.push(_25p); +// } + +// if let Some(_10p) = self._10p.as_mut() { +// v.push(_10p); +// } + +// v +// } +// } diff --git a/parser/src/structs/bi_map.rs b/parser/src/structs/bi_map.rs index f9cd9a191..01d626f23 100644 --- a/parser/src/structs/bi_map.rs +++ b/parser/src/structs/bi_map.rs @@ -4,24 +4,23 @@ use std::{ }; use allocative::Allocative; -use ordered_float::FloatCore; use crate::{bitcoin::TARGET_BLOCKS_PER_DAY, utils::LossyFrom}; use super::{AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapValue}; #[derive(Default, Allocative)] -pub struct BiMap +pub struct BiMap where - T: MapValue, + Value: MapValue, { - pub height: HeightMap, - pub date: DateMap, + pub height: HeightMap, + pub date: DateMap, } -impl BiMap +impl BiMap where - T: MapValue, + Value: MapValue, { pub fn new_bin(version: u32, path: &str) -> Self { Self { @@ -39,7 +38,7 @@ where pub fn date_insert_sum_range(&mut self, date: Date, date_blocks_range: &RangeInclusive) where - T: Sum, + Value: Sum, { self.date .insert(date, self.height.sum_range(date_blocks_range)); @@ -51,7 +50,7 @@ where first_height: &mut DateMap, last_height: &mut DateMap, ) where - T: Sum, + Value: Sum, { dates.iter().for_each(|date| { let first_height = first_height.get_or_import(date).unwrap(); @@ -62,7 +61,7 @@ where }) } - pub fn multi_insert_const(&mut self, heights: &[Height], dates: &[Date], constant: T) { + pub fn multi_insert_const(&mut self, heights: &[Height], dates: &[Date], constant: Value) { self.height.multi_insert_const(heights, constant); self.date.multi_insert_const(dates, constant); @@ -75,8 +74,8 @@ where source: &mut BiMap, transform: &F, ) where - T: Div, - F: Fn(K) -> T, + Value: Div, + F: Fn(K) -> Value, K: MapValue, { self.height @@ -95,8 +94,8 @@ where ) where A: MapValue, B: MapValue, - T: LossyFrom + LossyFrom, - T: Add, + Value: LossyFrom + LossyFrom, + Value: Add, { self.height .multi_insert_add(heights, &mut added.height, &mut adder.height); @@ -113,8 +112,8 @@ where ) where A: MapValue, B: MapValue, - T: LossyFrom + LossyFrom, - T: Sub, + Value: LossyFrom + LossyFrom, + Value: Sub, { self.height .multi_insert_subtract(heights, &mut subtracted.height, &mut subtracter.height); @@ -132,8 +131,8 @@ where ) where A: MapValue, B: MapValue, - T: LossyFrom + LossyFrom, - T: Mul, + Value: LossyFrom + LossyFrom, + Value: Mul, { self.height .multi_insert_multiply(heights, &mut multiplied.height, &mut multiplier.height); @@ -150,8 +149,8 @@ where ) where A: MapValue, B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, + Value: LossyFrom + LossyFrom, + Value: Div + Mul + From, { self.height .multi_insert_divide(heights, &mut divided.height, &mut divider.height); @@ -168,8 +167,8 @@ where ) where A: MapValue, B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, + Value: LossyFrom + LossyFrom, + Value: Div + Mul + From, { self.height .multi_insert_percentage(heights, &mut divided.height, &mut divider.height); @@ -184,8 +183,8 @@ where source: &mut BiMap, ) where K: MapValue, - T: LossyFrom, - T: Add + Sub, + Value: LossyFrom, + Value: Add + Sub, { self.height .multi_insert_cumulative(heights, &mut source.height); @@ -201,8 +200,8 @@ where days: usize, ) where K: MapValue, - T: LossyFrom, - T: Add + Sub, + Value: LossyFrom, + Value: Add + Sub, { self.height.multi_insert_last_x_sum( heights, @@ -221,7 +220,7 @@ where source: &mut BiMap, days: usize, ) where - T: Into + From, + Value: Into + From, K: MapValue + Sum, f32: LossyFrom, { @@ -238,10 +237,10 @@ where &mut self, heights: &[Height], dates: &[Date], - source: &mut BiMap, + source: &mut BiMap, days: usize, ) where - T: Sub, + Value: Sub, { self.height.multi_insert_net_change( heights, @@ -256,10 +255,11 @@ where &mut self, heights: &[Height], dates: &[Date], - source: &mut BiMap, + source: &mut BiMap, days: Option, ) where - T: FloatCore, + Value: LossyFrom, + f32: LossyFrom, { self.height.multi_insert_median( heights, @@ -274,10 +274,11 @@ where &mut self, heights: &[Height], dates: &[Date], - mut map_and_percentiles: Vec<(&mut BiMap, f32)>, + mut map_and_percentiles: Vec<(&mut BiMap, f32)>, days: Option, ) where - T: FloatCore, + Value: LossyFrom, + f32: LossyFrom, { let mut date_map_and_percentiles = vec![]; let mut height_map_and_percentiles = vec![]; diff --git a/parser/src/structs/date_map.rs b/parser/src/structs/date_map.rs index b3e754cfc..d4e903808 100644 --- a/parser/src/structs/date_map.rs +++ b/parser/src/structs/date_map.rs @@ -35,6 +35,27 @@ pub trait AnyDateMap: AnyMap { fn as_any_mut_map(&mut self) -> &mut dyn AnyMap; } +#[inline(always)] +pub fn date_map_vec_to_any_date_map_vec( + vec: Vec<&DateMap>, +) -> impl Iterator +where + T: MapValue, +{ + vec.into_iter() + .map(|map| map as &(dyn AnyDateMap + Send + Sync)) +} + +#[inline(always)] +pub fn date_map_vec_to_mut_any_date_map_vec( + vec: Vec<&mut DateMap>, +) -> impl Iterator +where + T: MapValue, +{ + vec.into_iter().map(|map| map as &mut dyn AnyDateMap) +} + impl AnyDateMap for DateMap where T: MapValue, diff --git a/parser/src/structs/generic_map.rs b/parser/src/structs/generic_map.rs index 3f38d0903..10716952b 100644 --- a/parser/src/structs/generic_map.rs +++ b/parser/src/structs/generic_map.rs @@ -11,10 +11,14 @@ use std::{ use allocative::Allocative; use bincode::{Decode, Encode}; use itertools::Itertools; -use ordered_float::{FloatCore, OrderedFloat}; +use ordered_float::OrderedFloat; use serde::{de::DeserializeOwned, Serialize}; -use crate::{log, utils::LossyFrom, Serialization}; +use crate::{ + log, + utils::{get_percentile, LossyFrom}, + Serialization, +}; use super::{AnyMap, MapValue}; @@ -693,28 +697,31 @@ where pub fn multi_insert_percentage_change(&mut self, keys: &[Key], source: &mut Self, len: usize) where - Value: Sub + FloatCore, + Value: Sub + LossyFrom, + f32: LossyFrom, { - let one = Value::from(1.0).unwrap(); - let hundred = Value::from(100.0).unwrap(); + let one = 1.0; + let hundred = 100.0; keys.iter().for_each(|key| { - let previous_value = key - .checked_sub(len) - .and_then(|previous_key| source.get_or_import(&previous_key)) - .unwrap_or_default(); + let previous_value = f32::lossy_from( + key.checked_sub(len) + .and_then(|previous_key| source.get_or_import(&previous_key)) + .unwrap_or_default(), + ); - let last_value = source.get_or_import(key).unwrap(); + let last_value = f32::lossy_from(source.get_or_import(key).unwrap()); let percentage_change = ((last_value / previous_value) - one) * hundred; - self.insert(*key, percentage_change); + self.insert(*key, Value::lossy_from(percentage_change)); }); } pub fn multi_insert_median(&mut self, keys: &[Key], source: &mut Self, len: Option) where - Value: FloatCore, + Value: LossyFrom, + f32: LossyFrom, { source.multi_insert_percentile(keys, vec![(self, 0.5)], len); } @@ -725,7 +732,8 @@ where mut map_and_percentiles: Vec<(&mut Self, f32)>, len: Option, ) where - Value: FloatCore, + Value: LossyFrom, + f32: LossyFrom, { if len.map_or(false, |size| size < 3) { panic!("Computing a percentile for a size lower than 3 is useless"); @@ -736,8 +744,7 @@ where let min_percentile_key = Key::min_percentile_key(); - let nan = Value::from(f32::NAN).unwrap(); - let two = Value::from(2.0).unwrap(); + let nan = Value::lossy_from(f32::NAN); keys.iter().cloned().try_for_each(|key| { if key < min_percentile_key { @@ -753,8 +760,9 @@ where let mut vec = start .iter_up_to(&key) .flat_map(|key| self.get_or_import(&key)) + .map(|v| f32::lossy_from(v)) .filter(|f| !f.is_nan()) - .map(|f| OrderedFloat(f)) + .map(OrderedFloat) .collect_vec(); if len.is_some() { @@ -765,7 +773,7 @@ where sorted_vec.replace(vec); } else { - let float_value = self.get_or_import(&key).unwrap(); + let float_value = f32::lossy_from(self.get_or_import(&key).unwrap()); if !float_value.is_nan() { let float_value = OrderedFloat(float_value); @@ -797,8 +805,6 @@ where let vec = sorted_vec.as_ref().unwrap(); - let len = vec.len(); - map_and_percentiles .iter_mut() .for_each(|(map, percentile)| { @@ -806,47 +812,9 @@ where panic!("The percentile should be between 0.0 and 1.0"); } - let value = { - if len < 2 { - nan - } else { - let index = (len - 1) as f32 * *percentile; + let float_value = get_percentile::>(vec, *percentile).0; - let fract = index.fract(); - - if fract != 0.0 { - (vec.get(index.ceil() as usize) - .unwrap_or_else(|| { - dbg!(vec, index, &self.path_all, &self.path_all, len); - panic!() - }) - .0 - + vec - .get(index as usize) - .unwrap_or_else(|| { - dbg!( - vec, - index, - &self.path_all, - &self.path_all, - len - ); - panic!() - }) - .0) - / two - } else { - vec.get(index as usize) - .unwrap_or_else(|| { - dbg!(vec, index); - panic!(); - }) - .0 - } - } - }; - - (*map).insert(key, value); + (*map).insert(key, Value::lossy_from(float_value)); }); } else { map_and_percentiles.iter_mut().for_each(|(map, _)| { diff --git a/parser/src/structs/height.rs b/parser/src/structs/height.rs index 0c46d2d9a..79dcf9b07 100644 --- a/parser/src/structs/height.rs +++ b/parser/src/structs/height.rs @@ -45,6 +45,12 @@ impl Height { pub fn is_safe(&self, block_count: usize) -> bool { **self < (block_count - NUMBER_OF_UNSAFE_BLOCKS) as u32 } + + pub fn iter_range_inclusive(first: Height, last: Height) -> impl Iterator { + let range = (*first)..=(*last); + + range.into_iter().map(Height::new) + } } impl PartialEq for Height { diff --git a/parser/src/structs/height_map.rs b/parser/src/structs/height_map.rs index 965ef847f..bd2f36a73 100644 --- a/parser/src/structs/height_map.rs +++ b/parser/src/structs/height_map.rs @@ -1,5 +1,7 @@ use std::{iter::Sum, ops::RangeInclusive}; +use itertools::Itertools; + use crate::SerializedVec; use super::{AnyMap, GenericMap, Height, HeightMapChunkId, MapValue}; @@ -8,18 +10,25 @@ pub const HEIGHT_MAP_CHUNK_SIZE: u32 = 10_000; pub type HeightMap = GenericMap>; -impl HeightMap +impl HeightMap where - T: MapValue, + Value: MapValue, { - pub fn sum_range(&self, range: &RangeInclusive) -> T + pub fn sum_range(&self, range: &RangeInclusive) -> Value where - T: Sum, + Value: Sum, { range .to_owned() .flat_map(|height| self.get(&Height::new(height))) - .sum::() + .sum::() + } + + pub fn get_or_import_range_inclusive(&mut self, first: Height, last: Height) -> Vec { + ((*first)..=(*last)) + .map(Height::new) + .map(|h| self.get_or_import(&h).unwrap()) + .collect_vec() } } @@ -55,990 +64,3 @@ where self } } - -// use std::{ -// cmp::Ordering, -// collections::{BTreeMap, VecDeque}, -// fmt::Debug, -// fs, -// iter::Sum, -// mem, -// ops::{Add, ControlFlow, Div, Mul, RangeInclusive, Sub}, -// path::{Path, PathBuf}, -// }; - -// use allocative::Allocative; -// use bincode::{Decode, Encode}; -// use itertools::Itertools; -// use ordered_float::{FloatCore, OrderedFloat}; -// use serde::{Deserialize, Serialize}; - -// use crate::{ -// bitcoin::NUMBER_OF_UNSAFE_BLOCKS, -// io::{format_path, Serialization}, -// utils::{log, LossyFrom}, -// }; - -// use super::{AnyMap, MapValue}; - -// pub const HEIGHT_MAP_CHUNK_SIZE: usize = 10_000; - -// #[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)] -// pub struct SerializedHeightMap { -// version: u32, -// map: Vec, -// } - -// #[derive(Default, Allocative)] -// pub struct HeightMap -// where -// T: MapValue, -// { -// version: u32, - -// path_all: String, -// path_last: Option, - -// chunks_in_memory: usize, - -// serialization: Serialization, - -// initial_last_height: Option, -// initial_first_unsafe_height: Option, - -// imported: BTreeMap>, -// to_insert: BTreeMap>, -// } - -// impl HeightMap -// where -// T: MapValue, -// { -// pub fn new_bin(version: u32, path: &str) -> Self { -// Self::new(version, path, Serialization::Binary, 1, true) -// } - -// pub fn _new_bin(version: u32, path: &str, export_last: bool) -> Self { -// Self::new(version, path, Serialization::Binary, 1, export_last) -// } - -// pub fn new_json(version: u32, path: &str, export_last: bool) -> Self { -// Self::new(version, path, Serialization::Json, usize::MAX, export_last) -// } - -// fn new( -// version: u32, -// path: &str, -// serialization: Serialization, -// chunks_in_memory: usize, -// export_last: bool, -// ) -> Self { -// if chunks_in_memory < 1 { -// panic!("Should always have at least the latest chunk in memory"); -// } - -// let path = format_path(path); - -// let path_all = format!("{path}/height"); - -// fs::create_dir_all(&path_all).unwrap(); - -// let path_last = { -// if export_last { -// Some(serialization.append_extension(&format!("{path}/last"))) -// } else { -// None -// } -// }; - -// let mut s = Self { -// version, - -// path_all, -// path_last, - -// chunks_in_memory, - -// serialization, - -// initial_first_unsafe_height: None, -// initial_last_height: None, - -// to_insert: BTreeMap::default(), -// imported: BTreeMap::default(), -// }; - -// s.read_dir() -// .into_iter() -// .rev() -// .take(chunks_in_memory) -// .for_each(|(chunk_start, path)| { -// if let Ok(serialized) = s.import(&path) { -// if serialized.version == s.version { -// s.imported.insert(chunk_start, serialized); -// } else { -// s.read_dir() -// .iter() -// .for_each(|(_, path)| fs::remove_file(path).unwrap()) -// } -// } -// }); - -// s.initial_last_height = s -// .imported -// .iter() -// .last() -// .map(|(chunk_start, serialized)| chunk_start + serialized.map.len()); - -// s.initial_first_unsafe_height = s.initial_last_height.and_then(|last_height| { -// let offset = NUMBER_OF_UNSAFE_BLOCKS - 1; -// last_height.checked_sub(offset) -// }); - -// if s.initial_first_unsafe_height.is_none() { -// log(&format!("New {path}")); -// } - -// s -// } - -// fn height_to_chunk_name(height: Height) -> String { -// let start = Self::height_to_chunk_start(height); -// let end = start + HEIGHT_MAP_CHUNK_SIZE; - -// format!("{start}..{end}") -// } - -// fn height_to_chunk_start(height: Height) -> usize { -// height / HEIGHT_MAP_CHUNK_SIZE * HEIGHT_MAP_CHUNK_SIZE -// } - -// pub fn insert(&mut self, height: Height, value: T) -> T { -// if !self.is_height_safe(height) { -// self.to_insert -// .entry(Self::height_to_chunk_start(height)) -// .or_default() -// .insert(height % HEIGHT_MAP_CHUNK_SIZE, value); -// } - -// value -// } - -// pub fn insert_default(&mut self, height: Height) -> T { -// self.insert(height, T::default()) -// } - -// pub fn get(&self, height: &usize) -> Option { -// let chunk_start = Self::height_to_chunk_start(*height); - -// self.to_insert -// .get(&chunk_start) -// .and_then(|map| map.get(&(height - chunk_start)).cloned()) -// .or_else(|| { -// self.imported -// .get(&chunk_start) -// .and_then(|serialized| serialized.map.get(height - chunk_start)) -// .cloned() -// }) -// } - -// pub fn get_or_import(&mut self, height: &usize) -> T { -// let chunk_start = Self::height_to_chunk_start(*height); - -// self.to_insert -// .get(&chunk_start) -// .and_then(|map| map.get(&(height - chunk_start)).cloned()) -// .or_else(|| { -// #[allow(clippy::map_entry)] // Can't be mut and then use read_dir() -// if !self.imported.contains_key(&chunk_start) { -// let dir_content = self.read_dir(); - -// let path = dir_content.get(&chunk_start).unwrap_or_else(|| { -// dbg!(self.path(), chunk_start, &dir_content); -// panic!(); -// }); - -// let serialized = self.import(path).unwrap(); - -// self.imported.insert(chunk_start, serialized); -// } - -// self.imported -// .get(&chunk_start) -// .and_then(|serialized| serialized.map.get(height - chunk_start)) -// .cloned() -// }) -// .unwrap_or_else(|| { -// dbg!(height, self.path()); -// panic!(); -// }) -// } - -// #[inline(always)] -// pub fn is_height_safe(&self, height: Height) -> bool { -// self.initial_first_unsafe_height.unwrap_or(0) > height -// } - -// fn read_dir(&self) -> BTreeMap { -// Self::_read_dir(&self.path_all, &self.serialization) -// } - -// pub fn _read_dir(path: &str, serialization: &Serialization) -> BTreeMap { -// fs::read_dir(path) -// .unwrap() -// .map(|entry| entry.unwrap().path()) -// .filter(|path| { -// let extension = path.extension().unwrap().to_str().unwrap(); - -// path.is_file() && extension == serialization.to_extension() -// }) -// .map(|path| { -// ( -// path.file_stem() -// .unwrap() -// .to_str() -// .unwrap() -// .split("..") -// .next() -// .unwrap() -// .parse::() -// .unwrap(), -// path, -// ) -// }) -// .collect() -// } - -// fn import(&self, path: &Path) -> color_eyre::Result> { -// self.serialization -// .import::>(path.to_str().unwrap()) -// } -// } - -// impl AnyMap for HeightMap -// where -// T: MapValue, -// { -// fn path(&self) -> &str { -// &self.path_all -// } - -// fn path_last(&self) -> &Option { -// &self.path_last -// } - -// fn t_name(&self) -> &str { -// std::any::type_name::() -// } - -// fn pre_export(&mut self) { -// let to_insert = &mut self.to_insert; - -// to_insert.iter_mut().for_each(|(chunk_start, map)| { -// if let Some((key, _)) = map.first_key_value() { -// if *key > 0 && !self.imported.contains_key(chunk_start) { -// // Had to copy paste many lines from functions as calling a function from self isn't allowed because of the &mut - -// let dir_content = Self::_read_dir(&self.path_all, &self.serialization); - -// let path = dir_content.get(chunk_start).unwrap_or_else(|| { -// dbg!(&self.path_all, chunk_start, &dir_content); -// panic!(); -// }); - -// let serialized = self -// .serialization -// .import::>(path.to_str().unwrap()) -// .unwrap(); - -// self.imported.insert(*chunk_start, serialized); -// } -// } - -// let serialized = self -// .imported -// .entry(*chunk_start) -// .or_insert(SerializedHeightMap { -// version: self.version, -// map: vec![], -// }); - -// mem::take(map) -// .into_iter() -// .for_each( -// |(chunk_height, value)| match serialized.map.len().cmp(&chunk_height) { -// Ordering::Greater => serialized.map[chunk_height] = value, -// Ordering::Equal => serialized.map.push(value), -// Ordering::Less => { -// dbg!(&self.path_all, &serialized.map, chunk_height, value); -// panic!() -// } -// }, -// ); -// }); -// } - -// fn export(&self) -> color_eyre::Result<()> { -// let len = self.imported.len(); - -// self.to_insert.iter().enumerate().try_for_each( -// |(index, (chunk_start, map))| -> color_eyre::Result<()> { -// if !map.is_empty() { -// unreachable!() -// } - -// let chunk_name = Self::height_to_chunk_name(*chunk_start); - -// let path = self -// .serialization -// .append_extension(&format!("{}/{}", self.path_all, chunk_name)); - -// let serialized = self.imported.get(chunk_start).unwrap_or_else(|| { -// dbg!(&self.path_all, chunk_start, &self.imported); -// panic!(); -// }); - -// self.serialization.export(&path, serialized)?; - -// if index == len - 1 { -// if let Some(path_last) = self.path_last.as_ref() { -// self.serialization -// .export(path_last, serialized.map.last().unwrap())?; -// } -// } - -// Ok(()) -// }, -// ) -// } - -// fn post_export(&mut self) { -// self.imported -// .keys() -// .rev() -// .enumerate() -// .filter(|(index, _)| *index + 1 > self.chunks_in_memory) -// .map(|(_, key)| *key) -// .collect_vec() -// .iter() -// .for_each(|key| { -// self.imported.remove(key); -// }); - -// self.to_insert.clear(); -// } -// } - -// pub trait AnyHeightMap: AnyMap { -// fn get_initial_first_unsafe_height(&self) -> Option; - -// fn get_initial_last_height(&self) -> Option; - -// fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync); - -// fn as_any_mut_map(&mut self) -> &mut dyn AnyMap; -// } - -// impl AnyHeightMap for HeightMap -// where -// T: MapValue, -// { -// #[inline(always)] -// fn get_initial_first_unsafe_height(&self) -> Option { -// self.initial_first_unsafe_height -// } - -// #[inline(always)] -// fn get_initial_last_height(&self) -> Option { -// self.initial_last_height -// } - -// fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync) { -// self -// } - -// fn as_any_mut_map(&mut self) -> &mut dyn AnyMap { -// self -// } -// } - -// impl HeightMap -// where -// T: MapValue, -// { -// pub fn sum_range(&self, range: &RangeInclusive) -> T -// where -// T: Sum, -// { -// range -// .to_owned() -// .flat_map(|height| self.get(&height)) -// .sum::() -// } - -// pub fn multi_insert_const(&mut self, heights: &[Height], constant: T) { -// heights.iter().for_each(|height| { -// let height = *height; - -// self.insert(height, constant); -// }); -// } - -// pub fn multi_insert_simple_transform( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// transform: F, -// ) where -// K: MapValue, -// F: Fn(K) -> T, -// { -// heights.iter().for_each(|height| { -// self.insert(*height, transform(source.get_or_import(height))); -// }); -// } - -// pub fn multi_insert_complex_transform( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// mut transform: F, -// ) where -// K: MapValue, -// F: FnMut((K, &usize)) -> T, -// { -// heights.iter().for_each(|height| { -// self.insert(*height, transform((source.get_or_import(height), height))); -// }); -// } - -// pub fn multi_insert_add( -// &mut self, -// heights: &[Height], -// added: &mut HeightMap, -// adder: &mut HeightMap, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Add, -// { -// heights.iter().for_each(|height| { -// self.insert( -// *height, -// T::lossy_from(added.get_or_import(height)) -// + T::lossy_from(adder.get_or_import(height)), -// ); -// }); -// } - -// pub fn multi_insert_subtract( -// &mut self, -// heights: &[Height], -// subtracted: &mut HeightMap, -// subtracter: &mut HeightMap, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Sub, -// { -// heights.iter().for_each(|height| { -// self.insert( -// *height, -// T::lossy_from(subtracted.get_or_import(height)) -// - T::lossy_from(subtracter.get_or_import(height)), -// ); -// }); -// } - -// pub fn multi_insert_multiply( -// &mut self, -// heights: &[Height], -// multiplied: &mut HeightMap, -// multiplier: &mut HeightMap, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Mul, -// { -// heights.iter().for_each(|height| { -// self.insert( -// *height, -// T::lossy_from(multiplied.get_or_import(height)) -// * T::lossy_from(multiplier.get_or_import(height)), -// ); -// }); -// } - -// pub fn multi_insert_divide( -// &mut self, -// heights: &[Height], -// divided: &mut HeightMap, -// divider: &mut HeightMap, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Div + Mul + From, -// { -// self._multi_insert_divide(heights, divided, divider, false) -// } - -// pub fn multi_insert_percentage( -// &mut self, -// heights: &[Height], -// divided: &mut HeightMap, -// divider: &mut HeightMap, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Div + Mul + From, -// { -// self._multi_insert_divide(heights, divided, divider, true) -// } - -// pub fn _multi_insert_divide( -// &mut self, -// heights: &[Height], -// divided: &mut HeightMap, -// divider: &mut HeightMap, -// as_percentage: bool, -// ) where -// A: MapValue, -// B: MapValue, -// T: LossyFrom + LossyFrom, -// T: Div + Mul + From, -// { -// let multiplier = T::from(if as_percentage { 100 } else { 1 }); - -// heights.iter().for_each(|height| { -// self.insert( -// *height, -// T::lossy_from(divided.get_or_import(height)) -// / T::lossy_from(divider.get_or_import(height)) -// * multiplier, -// ); -// }); -// } - -// pub fn multi_insert_cumulative(&mut self, heights: &[Height], source: &mut HeightMap) -// where -// K: MapValue, -// T: LossyFrom, -// T: Add + Sub, -// { -// self._multi_insert_last_x_sum(heights, source, None) -// } - -// pub fn multi_insert_last_x_sum( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// block_time: usize, -// ) where -// K: MapValue, -// T: LossyFrom, -// T: Add + Sub, -// { -// self._multi_insert_last_x_sum(heights, source, Some(block_time)) -// } - -// fn _multi_insert_last_x_sum( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// block_time: Option, -// ) where -// K: MapValue, -// T: LossyFrom, -// T: Add + Sub, -// { -// let mut sum = None; - -// heights.iter().for_each(|height| { -// let to_subtract = block_time -// .and_then(|x| { -// (height + 1) -// .checked_sub(x) -// .map(|previous_height| source.get_or_import(&previous_height)) -// }) -// .unwrap_or_default(); - -// let previous_sum = sum.unwrap_or_else(|| { -// height -// .checked_sub(1) -// .map(|previous_sum_height| self.get_or_import(&previous_sum_height)) -// .unwrap_or_default() -// }); - -// let last_value = source.get_or_import(height); - -// sum.replace(previous_sum + T::lossy_from(last_value) - T::lossy_from(to_subtract)); - -// self.insert(*height, sum.unwrap()); -// }); -// } - -// pub fn multi_insert_simple_average( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// block_time: usize, -// ) where -// T: Into + From, -// K: MapValue + Sum, -// f32: LossyFrom, -// { -// if block_time <= 1 { -// panic!("Average of 1 or less is not useful"); -// } - -// let mut average = None; - -// heights.iter().for_each(|height| { -// let height = *height; - -// let previous_average: f32 = average -// .unwrap_or_else(|| { -// height -// .checked_sub(block_time) -// .and_then(|previous_average_height| self.get(&previous_average_height)) -// .unwrap_or_default() -// }) -// .into(); - -// let mut last_value = f32::lossy_from(source.get_or_import(&height)); - -// if last_value.is_nan() { -// last_value = 0.0; -// } - -// average.replace( -// ((previous_average * (block_time as f32 - 1.0) + last_value) / block_time as f32) -// .into(), -// ); - -// self.insert(height, average.unwrap()); -// }); -// } - -// pub fn multi_insert_net_change( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// block_time: usize, -// ) where -// T: Sub, -// { -// heights.iter().for_each(|height| { -// let height = *height; - -// let previous_value = height -// .checked_sub(block_time) -// .map(|height| source.get_or_import(&height)) -// .unwrap_or_default(); - -// let last_value = source.get_or_import(&height); - -// let net = last_value - previous_value; - -// self.insert(height, net); -// }); -// } - -// pub fn multi_insert_median( -// &mut self, -// heights: &[Height], -// source: &mut HeightMap, -// block_time: Option, -// ) where -// T: FloatCore, -// { -// source.multi_insert_percentile(heights, vec![(self, 0.5)], block_time); -// } - -// pub fn multi_insert_percentile( -// &mut self, -// heights: &[Height], -// mut map_and_percentiles: Vec<(&mut HeightMap, f32)>, -// block_time: Option, -// ) where -// T: FloatCore, -// { -// if block_time.map_or(false, |size| size < 3) { -// panic!("Computing a percentile for a size lower than 3 is useless"); -// } - -// let mut ordered_vec = None; -// let mut sorted_vec = None; - -// let min_percentile_height = 160_000; - -// let nan = T::from(f32::NAN).unwrap(); -// let two = T::from(2.0).unwrap(); - -// if min_percentile_height % HEIGHT_MAP_CHUNK_SIZE != 0 { -// panic!("Should be 0"); -// } - -// heights.iter().cloned().try_for_each(|height| { -// if height < min_percentile_height { -// map_and_percentiles.iter_mut().for_each(|(map, _)| { -// (*map).insert(height, nan); -// }); -// return ControlFlow::Continue::<()>(()); -// } - -// if let Some(start) = -// block_time.map_or(Some(min_percentile_height), |size| height.checked_sub(size)) -// { -// if sorted_vec.is_none() { -// let mut vec = (start..=height) -// .map(|height| self.get_or_import(&height)) -// .filter(|f| !f.is_nan()) -// .map(|f| OrderedFloat(f)) -// .collect_vec(); - -// if block_time.is_some() { -// ordered_vec.replace(VecDeque::from(vec.clone())); -// } - -// vec.sort_unstable(); - -// sorted_vec.replace(vec); -// } else { -// let float_value = self.get_or_import(&height); - -// if !float_value.is_nan() { -// let float_value = OrderedFloat(float_value); - -// if block_time.is_some() { -// let first = ordered_vec.as_mut().unwrap().pop_front().unwrap(); -// let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); -// sorted_vec.as_mut().unwrap().remove(pos); - -// ordered_vec.as_mut().unwrap().push_back(float_value); -// } - -// let pos = sorted_vec -// .as_ref() -// .unwrap() -// .binary_search(&float_value) -// .unwrap_or_else(|pos| pos); - -// sorted_vec.as_mut().unwrap().insert(pos, float_value); -// } -// } - -// let vec = sorted_vec.as_ref().unwrap(); - -// let len = vec.len(); - -// map_and_percentiles -// .iter_mut() -// .for_each(|(map, percentile)| { -// if !(0.0..=1.0).contains(percentile) { -// panic!("The percentile should be between 0.0 and 1.0"); -// } - -// let value = { -// if len < 2 { -// nan -// } else { -// let index = (len - 1) as f32 * *percentile; - -// let fract = index.fract(); - -// if fract != 0.0 { -// (vec.get(index.ceil() as usize) -// .unwrap_or_else(|| { -// dbg!( -// index, -// &self.path_all, -// &self.path_all, -// &self.to_insert, -// block_time, -// vec -// ); -// panic!() -// }) -// .0 -// + vec -// .get(index.floor() as usize) -// .unwrap_or_else(|| { -// dbg!( -// index, -// &self.path_all, -// &self.path_all, -// block_time -// ); -// panic!() -// }) -// .0) -// / two -// } else { -// vec.get(index as usize).unwrap().0 -// } -// } -// }; - -// (*map).insert(height, value); -// }); -// } else { -// map_and_percentiles.iter_mut().for_each(|(map, _)| { -// (*map).insert(height, nan); -// }); -// } - -// ControlFlow::Continue(()) -// }); -// } - -// // pub fn insert_cumulative(&mut self, height: Height, source: &HeightMap) -> T -// // where -// // T: Add + Sub, -// // { -// // let previous_cum = height -// // .checked_sub(1) -// // .map(|previous_sum_height| { -// // self.get(&previous_sum_height).unwrap_or_else(|| { -// // dbg!(previous_sum_height); -// // panic!() -// // }) -// // }) -// // .unwrap_or_default(); - -// // let last_value = source.get(&height).unwrap(); - -// // let cum_value = previous_cum + last_value; - -// // self.insert(height, cum_value); - -// // cum_value -// // } - -// // pub fn insert_last_x_sum(&mut self, height: Height, source: &HeightMap, x: usize) -> T -// // where -// // T: Add + Sub, -// // { -// // let to_subtract = (height + 1) -// // .checked_sub(x) -// // .map(|previous_height| { -// // source.get(&previous_height).unwrap_or_else(|| { -// // dbg!(&self.path_all, &source.path_all, previous_height); -// // panic!() -// // }) -// // }) -// // .unwrap_or_default(); - -// // let previous_sum = height -// // .checked_sub(1) -// // .map(|previous_sum_height| self.get(&previous_sum_height).unwrap()) -// // .unwrap_or_default(); - -// // let last_value = source.get(&height).unwrap(); - -// // let sum = previous_sum + last_value - to_subtract; - -// // self.insert(height, sum); - -// // sum -// // } - -// // pub fn insert_simple_average(&mut self, height: Height, source: &HeightMap, block_time: usize) -// // where -// // T: Into + From, -// // { -// // let to_subtract: f32 = (height + 1) -// // .checked_sub(block_time) -// // .map(|previous_height| source.get(&previous_height).unwrap()) -// // .unwrap_or_default() -// // .into(); - -// // let previous_average: f32 = height -// // .checked_sub(1) -// // .map(|previous_average_height| self.get(&previous_average_height).unwrap()) -// // .unwrap_or_default() -// // .into(); - -// // let last_value: f32 = source.get(&height).unwrap().into(); - -// // let sum = previous_average * block_time as f32 - to_subtract + last_value; - -// // let average: T = (sum / block_time as f32).into(); - -// // self.insert(height, average); -// // } - -// // pub fn insert_net_change(&mut self, height: Height, source: &HeightMap, offset: usize) -> T -// // where -// // T: Sub, -// // { -// // let previous_value = height -// // .checked_sub(offset) -// // .map(|height| { -// // source.get(&height).unwrap_or_else(|| { -// // dbg!(&self.path_all, &source.path_all, offset); -// // panic!(); -// // }) -// // }) -// // .unwrap_or_default(); - -// // let last_value = source.get(&height).unwrap(); - -// // let net = last_value - previous_value; - -// // self.insert(height, net); - -// // net -// // } - -// // pub fn insert_median(&mut self, height: Height, source: &HeightMap, size: usize) -> T -// // where -// // T: FloatCore, -// // { -// // if size < 3 { -// // panic!("Computing a median for a size lower than 3 is useless"); -// // } - -// // let median = { -// // if let Some(start) = height.checked_sub(size - 1) { -// // let even = size % 2 == 0; -// // let median_index = size / 2; - -// // let mut vec = (start..=height) -// // .map(|height| { -// // OrderedFloat(source.get(&height).unwrap_or_else(|| { -// // dbg!(height, &source.path_all, size); -// // panic!() -// // })) -// // }) -// // .collect_vec(); - -// // vec.sort_unstable(); - -// // if even { -// // (vec.get(median_index) -// // .unwrap_or_else(|| { -// // dbg!(median_index, &self.path_all, &source.path_all, size); -// // panic!() -// // }) -// // .0 -// // + vec.get(median_index - 1).unwrap().0) -// // / T::from(2.0).unwrap() -// // } else { -// // vec.get(median_index).unwrap().0 -// // } -// // } else { -// // T::default() -// // } -// // }; - -// // self.insert(height, median); - -// // median -// // } -// } diff --git a/parser/src/utils/lossy.rs b/parser/src/utils/lossy.rs index d5849e680..27defd55a 100644 --- a/parser/src/utils/lossy.rs +++ b/parser/src/utils/lossy.rs @@ -1,11 +1,59 @@ +use ordered_float::OrderedFloat; + pub trait LossyFrom { fn lossy_from(x: T) -> Self; } +// --- +// u32 +// --- + +impl LossyFrom for u32 { + #[inline(always)] + fn lossy_from(x: u32) -> Self { + x + } +} + +impl LossyFrom for u32 { + #[inline(always)] + fn lossy_from(x: u64) -> Self { + x as u32 + } +} + +impl LossyFrom for u32 { + #[inline(always)] + fn lossy_from(x: usize) -> Self { + x as u32 + } +} + +impl LossyFrom for u32 { + #[inline(always)] + fn lossy_from(x: f32) -> Self { + x as u32 + } +} + +impl LossyFrom> for u32 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 as u32 + } +} + // --- // u64 // --- +impl LossyFrom for u64 { + #[inline(always)] + fn lossy_from(x: u32) -> Self { + x as u64 + } +} + impl LossyFrom for u64 { #[inline(always)] fn lossy_from(x: u64) -> Self { @@ -20,6 +68,20 @@ impl LossyFrom for u64 { } } +impl LossyFrom for u64 { + #[inline(always)] + fn lossy_from(x: f32) -> Self { + x as u64 + } +} + +impl LossyFrom> for u64 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 as u64 + } +} + // --- // usize // --- @@ -70,6 +132,13 @@ impl LossyFrom for f32 { } } +impl LossyFrom> for f32 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 + } +} + impl LossyFrom for f32 { #[inline(always)] fn lossy_from(x: f64) -> Self { @@ -77,6 +146,66 @@ impl LossyFrom for f32 { } } +impl LossyFrom> for f32 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 as f32 + } +} + +// --- +// OrderedFloat +// --- + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: u32) -> Self { + OrderedFloat(x as f32) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: u64) -> Self { + OrderedFloat(x as f32) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: usize) -> Self { + OrderedFloat(x as f32) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: f32) -> Self { + OrderedFloat(x) + } +} + +impl LossyFrom> for OrderedFloat { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: f64) -> Self { + OrderedFloat(x as f32) + } +} + +impl LossyFrom> for OrderedFloat { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + OrderedFloat(x.0 as f32) + } +} + // --- // f64 // --- @@ -102,9 +231,69 @@ impl LossyFrom for f64 { } } +impl LossyFrom> for f64 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 as f64 + } +} + impl LossyFrom for f64 { #[inline(always)] fn lossy_from(x: f64) -> Self { x } } + +impl LossyFrom> for f64 { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x.0 + } +} + +// --- +// OrderedFloat +// --- + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: u64) -> Self { + OrderedFloat(x as f64) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: usize) -> Self { + OrderedFloat(x as f64) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: f32) -> Self { + OrderedFloat(x as f64) + } +} + +impl LossyFrom> for OrderedFloat { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + OrderedFloat(x.0 as f64) + } +} + +impl LossyFrom for OrderedFloat { + #[inline(always)] + fn lossy_from(x: f64) -> Self { + OrderedFloat(x) + } +} + +impl LossyFrom> for OrderedFloat { + #[inline(always)] + fn lossy_from(x: OrderedFloat) -> Self { + x + } +} diff --git a/parser/src/utils/mod.rs b/parser/src/utils/mod.rs index 6ca7775cf..0a9f8a113 100644 --- a/parser/src/utils/mod.rs +++ b/parser/src/utils/mod.rs @@ -3,6 +3,7 @@ mod date; mod flamegraph; mod log; mod lossy; +mod percentile; mod retry; mod time; @@ -11,5 +12,6 @@ pub use date::*; pub use flamegraph::*; pub use log::*; pub use lossy::*; +pub use percentile::*; pub use retry::*; pub use time::*; diff --git a/parser/src/utils/percentile.rs b/parser/src/utils/percentile.rs new file mode 100644 index 000000000..c8e41f461 --- /dev/null +++ b/parser/src/utils/percentile.rs @@ -0,0 +1,28 @@ +use std::ops::Add; + +use super::LossyFrom; + +pub fn get_percentile(sorted: &[T], percentile: f32) -> T +where + T: Clone + Copy + LossyFrom + Add, + f32: LossyFrom, +{ + let len = sorted.len(); + + if len < 2 { + T::lossy_from(f32::NAN) + } else { + let index = (len - 1) as f32 * percentile; + + let fract = index.fract(); + + if fract != 0.0 { + let left = *sorted.get(index as usize).unwrap(); + let right = *sorted.get(index.ceil() as usize).unwrap(); + + T::lossy_from(f32::lossy_from(left + right) / 2.0) + } else { + *sorted.get(index as usize).unwrap() + } + } +} diff --git a/server/Cargo.lock b/server/Cargo.lock index 29f9df7c0..1bd077d73 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2135,9 +2135,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" dependencies = [ "serde", "serde_spanned", @@ -2156,9 +2156,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" dependencies = [ "indexmap", "serde",