diff --git a/parser/.gitignore b/parser/.gitignore index 4199ebbb3..7ff0b82bf 100644 --- a/parser/.gitignore +++ b/parser/.gitignore @@ -16,4 +16,4 @@ benches parser.log config.toml -*\ copy +*\ copy* diff --git a/parser/src/structs/date_map copy.rs b/parser/src/structs/date_map copy.rs deleted file mode 100644 index 3428528f9..000000000 --- a/parser/src/structs/date_map copy.rs +++ /dev/null @@ -1,1287 +0,0 @@ -use std::{ - collections::{BTreeMap, VecDeque}, - fmt::Debug, - fs, - iter::Sum, - mem, - ops::{Add, ControlFlow, Div, Mul, Sub}, - path::{Path, PathBuf}, -}; - -use allocative::Allocative; -use bincode::{Decode, Encode}; -use chrono::{Datelike, Days}; -use itertools::Itertools; -use ordered_float::{FloatCore, OrderedFloat}; -use serde::{Deserialize, Serialize}; - -use crate::{ - io::{format_path, Serialization}, - utils::{log, LossyFrom}, -}; - -use super::{AnyMap, Date, HeightMap, MapValue}; - -const NUMBER_OF_UNSAFE_DATES: usize = 2; -const MIN_YEAR: usize = 2009; - -#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)] -pub struct SerializedDateMap { - version: u32, - map: BTreeMap, -} - -#[derive(Default, Allocative)] -pub struct DateMap { - version: u32, - - path_all: String, - path_last: Option, - - chunks_in_memory: usize, - - serialization: Serialization, - - pub initial_last_date: Option, - pub initial_first_unsafe_date: Option, - - imported: BTreeMap>, - to_insert: BTreeMap>, -} - -impl DateMap -where - T: MapValue, -{ - pub fn new_bin(version: u32, path: &str) -> Self { - Self::new(version, path, Serialization::Binary, 1, true) - } - - pub fn _new_bin(version: u32, path: &str, export_last: bool) -> Self { - Self::new(version, path, Serialization::Binary, 1, export_last) - } - - pub fn new_json(version: u32, path: &str, export_last: bool) -> Self { - Self::new(version, path, Serialization::Json, usize::MAX, export_last) - } - - fn new( - version: u32, - path: &str, - serialization: Serialization, - chunks_in_memory: usize, - export_last: bool, - ) -> Self { - if chunks_in_memory < 1 { - panic!("Should always have at least the latest chunk in memory"); - } - - let path = format_path(path); - - let path_all = format!("{path}/date"); - - fs::create_dir_all(&path_all).unwrap(); - - let path_last = { - if export_last { - Some(serialization.append_extension(&format!("{path}/last"))) - } else { - None - } - }; - - let mut s = Self { - version, - - path_all, - path_last, - - chunks_in_memory, - - serialization, - - initial_last_date: None, - initial_first_unsafe_date: None, - - to_insert: BTreeMap::default(), - imported: BTreeMap::default(), - }; - - s.read_dir() - .into_iter() - .rev() - .take(chunks_in_memory) - .for_each(|(chunk_start, path)| { - if let Ok(serialized) = s.import(&path) { - if serialized.version == s.version { - s.imported.insert(chunk_start, serialized); - } else { - s.read_dir() - .iter() - .for_each(|(_, path)| fs::remove_file(path).unwrap()) - } - } - }); - - s.initial_last_date = s - .imported - .values() - .last() - .and_then(|serialized| serialized.map.keys().copied().max()); - - s.initial_first_unsafe_date = s.initial_last_date.and_then(|last_date| { - let offset = NUMBER_OF_UNSAFE_DATES - 1; - last_date - .checked_sub_days(Days::new(offset as u64)) - .map(Date::wrap) - }); - - if s.initial_first_unsafe_date.is_none() { - log(&format!("New {path}")); - } - - s - } - - pub fn insert(&mut self, date: Date, value: T) -> T { - if !self.is_date_safe(date) { - self.to_insert - .entry(date.year() as usize) - .or_default() - .insert(date, value); - } - - value - } - - pub fn insert_default(&mut self, date: Date) -> T { - self.insert(date, T::default()) - } - - pub fn get(&self, date: &Date) -> Option { - let year = date.year() as usize; - - self.to_insert - .get(&year) - .and_then(|tree| tree.get(date).cloned()) - .or_else(|| { - self.imported - .get(&year) - .and_then(|serialized| serialized.map.get(date)) - .cloned() - }) - } - - pub fn get_or_import(&mut self, date: &Date) -> Option { - let year = date.year() as usize; - - if year < MIN_YEAR { - return None; - } - - self.to_insert - .get(&year) - .and_then(|tree| tree.get(date).cloned()) - .or_else(|| { - #[allow(clippy::map_entry)] // Can't be mut and then use read_dir() - if !self.imported.contains_key(&year) { - let dir_content = self.read_dir(); - - if let Some(path) = dir_content.get(&year) { - let serialized = self.import(path).unwrap(); - // .unwrap_or(SerializedDateMap { - // version: self.version, - // map: BTreeMap::default(), - // }); - - self.imported.insert(year, serialized); - } - } - - self.imported - .get(&year) - .and_then(|serialized| serialized.map.get(date)) - .cloned() - }) - } - - #[inline(always)] - pub fn is_date_safe(&self, date: Date) -> bool { - self.initial_first_unsafe_date - .map_or(false, |initial_first_unsafe_date| { - initial_first_unsafe_date > date - }) - } - - fn read_dir(&self) -> BTreeMap { - Self::_read_dir(&self.path_all, &self.serialization) - } - - pub fn _read_dir(path: &str, serialization: &Serialization) -> BTreeMap { - fs::read_dir(path) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| { - let file_stem = path.file_stem().unwrap().to_str().unwrap(); - let extension = path.extension().unwrap().to_str().unwrap(); - - path.is_file() - && file_stem.len() == 4 - && file_stem.starts_with("20") - && extension == serialization.to_extension() - }) - .map(|path| { - let year = path - .file_stem() - .unwrap() - .to_str() - .unwrap() - .parse::() - .unwrap(); - - (year, path) - }) - .collect() - } - - fn import(&self, path: &Path) -> color_eyre::Result> { - self.serialization - .import::>(path.to_str().unwrap()) - } -} - -impl AnyMap for DateMap -where - T: MapValue, -{ - fn path(&self) -> &str { - &self.path_all - } - - fn path_last(&self) -> &Option { - &self.path_last - } - - fn t_name(&self) -> &str { - std::any::type_name::() - } - - // fn reset(&mut self) -> color_eyre::Result<()> { - // fs::remove_dir(&self.path_all)?; - - // self.initial_last_date = None; - // self.initial_first_unsafe_date = None; - - // self.imported.clear(); - // self.to_insert.clear(); - - // Ok(()) - // } - - fn pre_export(&mut self) { - self.to_insert.iter_mut().for_each(|(chunk_start, map)| { - self.imported - .entry(*chunk_start) - .or_insert(SerializedDateMap { - version: self.version, - map: BTreeMap::default(), - }) - .map - .extend(mem::take(map)); - }); - } - - fn export(&self) -> color_eyre::Result<()> { - let len = self.imported.len(); - - self.to_insert.iter().enumerate().try_for_each( - |(index, (year, map))| -> color_eyre::Result<()> { - if !map.is_empty() { - unreachable!() - } - - let path = self - .serialization - .append_extension(&format!("{}/{}", self.path_all, year)); - - let serialized = self.imported.get(year).unwrap(); - - self.serialization.export(&path, serialized)?; - - if index == len - 1 { - if let Some(path_last) = self.path_last.as_ref() { - self.serialization - .export(path_last, serialized.map.values().last().unwrap())?; - } - } - - Ok(()) - }, - ) - } - - fn post_export(&mut self) { - self.imported - .keys() - .rev() - .enumerate() - .filter(|(index, _)| *index + 1 > self.chunks_in_memory) - .map(|(_, key)| *key) - .collect_vec() - .iter() - .for_each(|key| { - self.imported.remove(key); - }); - - self.to_insert.clear(); - } -} - -pub trait AnyDateMap: AnyMap { - fn get_initial_first_unsafe_date(&self) -> Option; - - fn get_initial_last_date(&self) -> Option; - - fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync); - - fn as_any_mut_map(&mut self) -> &mut dyn AnyMap; -} - -impl AnyDateMap for DateMap -where - T: MapValue, -{ - #[inline(always)] - fn get_initial_first_unsafe_date(&self) -> Option { - self.initial_first_unsafe_date - } - - #[inline(always)] - fn get_initial_last_date(&self) -> Option { - self.initial_last_date - } - - fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync) { - self - } - - fn as_any_mut_map(&mut self) -> &mut dyn AnyMap { - self - } -} - -impl DateMap -where - T: MapValue, -{ - pub fn multi_insert(&mut self, dates: &[Date], mut callback: F) - where - F: FnMut(&Date) -> T, - { - dates.iter().for_each(|date| { - self.insert(*date, callback(date)); - }); - } - - // KEEEEEP - pub fn multi_insert_last( - &mut self, - dates: &[Date], - source: &mut HeightMap, - last_height: &mut DateMap, - ) { - dates.iter().for_each(|date| { - self.insert( - *date, - source.get_or_import(&last_height.get_or_import(date).unwrap()), - ); - }); - } - - pub fn multi_insert_const(&mut self, dates: &[Date], constant: T) { - dates.iter().for_each(|date| { - self.insert(*date, constant); - }); - } - - pub fn multi_insert_simple_transform( - &mut self, - dates: &[Date], - source: &mut DateMap, - transform: F, - ) where - F: Fn(K) -> T, - K: MapValue, - { - dates.iter().for_each(|date| { - self.insert(*date, transform(source.get_or_import(date).unwrap())); - }); - } - - pub fn multi_insert_complex_transform( - &mut self, - dates: &[Date], - source: &mut DateMap, - mut transform: F, - ) where - K: MapValue, - F: FnMut((K, &Date, &mut DateMap)) -> T, - { - dates.iter().for_each(|date| { - self.insert( - *date, - transform((source.get_or_import(date).unwrap(), date, source)), - ); - }); - } - - pub fn multi_insert_add( - &mut self, - dates: &[Date], - added: &mut DateMap, - adder: &mut DateMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Add, - { - dates.iter().for_each(|date| { - self.insert( - *date, - T::lossy_from(added.get_or_import(date).unwrap()) - + T::lossy_from(adder.get_or_import(date).unwrap()), - ); - }); - } - - pub fn multi_insert_subtract( - &mut self, - dates: &[Date], - subtracted: &mut DateMap, - subtracter: &mut DateMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Sub, - { - dates.iter().for_each(|date| { - self.insert( - *date, - T::lossy_from(subtracted.get_or_import(date).unwrap()) - - T::lossy_from(subtracter.get_or_import(date).unwrap()), - ); - }); - } - - pub fn multi_insert_multiply( - &mut self, - dates: &[Date], - multiplied: &mut DateMap, - multiplier: &mut DateMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Mul, - { - dates.iter().for_each(|date| { - self.insert( - *date, - T::lossy_from(multiplied.get_or_import(date).unwrap()) - * T::lossy_from(multiplier.get_or_import(date).unwrap()), - ); - }); - } - - pub fn multi_insert_divide( - &mut self, - dates: &[Date], - divided: &mut DateMap, - divider: &mut DateMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - self._multi_insert_divide(dates, divided, divider, false) - } - - pub fn multi_insert_percentage( - &mut self, - dates: &[Date], - divided: &mut DateMap, - divider: &mut DateMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - self._multi_insert_divide(dates, divided, divider, true) - } - - fn _multi_insert_divide( - &mut self, - dates: &[Date], - divided: &mut DateMap, - divider: &mut DateMap, - as_percentage: bool, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - let multiplier = T::from(if as_percentage { 100 } else { 1 }); - - dates.iter().for_each(|date| { - self.insert( - *date, - T::lossy_from(divided.get_or_import(date).unwrap()) - / T::lossy_from(divider.get_or_import(date).unwrap()) - * multiplier, - ); - }); - } - - pub fn multi_insert_cumulative(&mut self, dates: &[Date], source: &mut DateMap) - where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - self._multi_insert_last_x_sum(dates, source, None) - } - - pub fn multi_insert_last_x_sum( - &mut self, - dates: &[Date], - source: &mut DateMap, - days: usize, - ) where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - self._multi_insert_last_x_sum(dates, source, Some(days)) - } - - fn _multi_insert_last_x_sum( - &mut self, - dates: &[Date], - source: &mut DateMap, - days: Option, - ) where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - let mut sum = None; - - dates.iter().for_each(|date| { - let to_subtract = days - .and_then(|x| { - date.checked_sub_days(Days::new(x as u64)) - .and_then(|previous_date| source.get_or_import(&Date::wrap(previous_date))) - }) - .unwrap_or_default(); - - let previous_sum = sum.unwrap_or_else(|| { - date.checked_sub_days(Days::new(1)) - .and_then(|previous_sum_date| { - self.get_or_import(&Date::wrap(previous_sum_date)) - }) - .unwrap_or_default() - }); - - let last_value = source.get_or_import(date).unwrap_or_else(|| { - dbg!(date); - panic!(); - }); - - sum.replace(previous_sum - T::lossy_from(to_subtract) + T::lossy_from(last_value)); - - self.insert(*date, sum.unwrap()); - }); - } - - pub fn multi_insert_simple_average( - &mut self, - dates: &[Date], - source: &mut DateMap, - days: usize, - ) where - T: Into + From, - K: MapValue + Sum, - f32: LossyFrom, - { - if days <= 1 { - panic!("Average of 1 or less is not useful"); - } - - let days = days as f32; - - let mut average = None; - - dates.iter().for_each(|date| { - let previous_average: f32 = average - .unwrap_or_else(|| { - date.checked_sub_days(Days::new(1)) - .and_then(|previous_average_date| { - self.get(&Date::wrap(previous_average_date)) - }) - .unwrap_or_default() - }) - .into(); - - let mut last_value = f32::lossy_from(source.get_or_import(date).unwrap_or_else(|| { - dbg!(date); - panic!() - })); - - if last_value.is_nan() { - last_value = 0.0; - } - - average.replace(((previous_average * (days - 1.0) + last_value) / days).into()); - - self.insert(*date, average.unwrap()); - }); - } - - pub fn multi_insert_net_change(&mut self, dates: &[Date], source: &mut DateMap, days: usize) - where - T: Sub, - { - dates.iter().for_each(|date| { - let previous_value = date - .checked_sub_days(Days::new(days as u64)) - .and_then(|date| source.get_or_import(&Date::wrap(date))) - .unwrap_or_default(); - - let last_value = source.get_or_import(date).unwrap(); - - let net_change = last_value - previous_value; - - self.insert(*date, net_change); - }); - } - - pub fn multi_insert_percentage_change( - &mut self, - dates: &[Date], - source: &mut DateMap, - days: usize, - ) where - T: Sub + FloatCore, - { - let one = T::from(1.0).unwrap(); - let hundred = T::from(100.0).unwrap(); - - dates.iter().for_each(|date| { - let previous_value = date - .checked_sub_days(Days::new(days as u64)) - .and_then(|date| source.get_or_import(&Date::wrap(date))) - .unwrap_or_default(); - - let last_value = source.get_or_import(date).unwrap(); - - let percentage_change = ((last_value / previous_value) - one) * hundred; - - self.insert(*date, percentage_change); - }); - } - - pub fn multi_insert_median( - &mut self, - dates: &[Date], - source: &mut DateMap, - days: Option, - ) where - T: FloatCore, - { - source.multi_insert_percentile(dates, vec![(self, 0.5)], days); - } - - pub fn multi_insert_percentile( - &mut self, - dates: &[Date], - mut map_and_percentiles: Vec<(&mut DateMap, f32)>, - days: Option, - ) where - T: FloatCore, - { - if days.map_or(false, |size| size < 3) { - panic!("Computing a percentile for a size lower than 3 is useless"); - } - - let mut ordered_vec = None; - let mut sorted_vec = None; - - let min_percentile_date = chrono::NaiveDate::from_ymd_opt(2012, 1, 1).unwrap(); - let min_percentile_wdate = Date::wrap(min_percentile_date); - - let nan = T::from(f32::NAN).unwrap(); - let two = T::from(2.0).unwrap(); - - dates.iter().cloned().try_for_each(|date| { - if date < min_percentile_wdate { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(date, nan); - }); - return ControlFlow::Continue::<()>(()); - } - - if let Some(start) = days.map_or(Some(min_percentile_date), |size| { - date.checked_sub_days(Days::new(size as u64)) - }) { - if sorted_vec.is_none() { - let mut vec = start - .iter_days() - .take_while(|d| *d <= *date) - .flat_map(|date| self.get_or_import(&Date::wrap(date))) - .filter(|f| !f.is_nan()) - .map(|f| OrderedFloat(f)) - .collect_vec(); - - if days.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } - - vec.sort_unstable(); - sorted_vec.replace(vec); - } else { - let float_value = self.get_or_import(&date).unwrap(); - - if !float_value.is_nan() { - let float_value = OrderedFloat(float_value); - - if let Some(days) = days { - if let Some(ordered_vec) = ordered_vec.as_mut() { - if ordered_vec.len() == days { - let first = ordered_vec.pop_front().unwrap(); - - let pos = - sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); - - sorted_vec.as_mut().unwrap().remove(pos); - } - - ordered_vec.push_back(float_value); - } - } - - let pos = sorted_vec - .as_ref() - .unwrap() - .binary_search(&float_value) - .unwrap_or_else(|pos| pos); - - sorted_vec.as_mut().unwrap().insert(pos, float_value); - } - } - - let vec = sorted_vec.as_ref().unwrap(); - - let len = vec.len(); - - map_and_percentiles - .iter_mut() - .for_each(|(map, percentile)| { - if !(0.0..=1.0).contains(percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } - - let value = { - if len < 2 { - nan - } else { - let index = (len - 1) as f32 * *percentile; - - let fract = index.fract(); - - if fract != 0.0 { - (vec.get(index.ceil() as usize) - .unwrap_or_else(|| { - dbg!(vec, index, &self.path_all, &self.path_all, days); - panic!() - }) - .0 - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!( - vec, - index, - &self.path_all, - &self.path_all, - days - ); - panic!() - }) - .0) - / two - } else { - vec.get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!(vec, index); - panic!(); - }) - .0 - } - } - }; - - (*map).insert(date, value); - }); - } else { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(date, nan); - }); - } - - ControlFlow::Continue(()) - }); - } - - // - // pub fn transform(&self, transform: F) -> BTreeMap - // where - // T: Copy + Default, - // F: Fn((&WNaiveDate, &T, &BTreeMap, usize)) -> T, - // { - // Self::_transform(self.imported.lock().as_ref().unwrap(), transform) - // } - - // pub fn _transform(map: &BTreeMap, transform: F) -> BTreeMap - // where - // T: Copy + Default, - // F: Fn((&WNaiveDate, &T, &BTreeMap, usize)) -> T, - // { - // map.iter() - // .enumerate() - // .map(|(index, (date, value))| (date.to_owned(), transform((date, value, map, index)))) - // .collect() - // } - - // - // pub fn add(&self, other: &Self) -> BTreeMap - // where - // T: Add + Copy + Default, - // { - // Self::_add( - // self.imported.lock().as_ref().unwrap(), - // other.imported.lock().as_ref().unwrap(), - // ) - // } - - // pub fn _add( - // map1: &BTreeMap, - // map2: &BTreeMap, - // ) -> BTreeMap - // where - // T: Add + Copy + Default, - // { - // Self::_transform(map1, |(date, value, ..)| { - // map2.get(date) - // .map(|value2| *value + *value2) - // .unwrap_or_default() - // }) - // } - - // - // pub fn subtract(&self, other: &Self) -> BTreeMap - // where - // T: Sub + Copy + Default, - // { - // Self::_subtract( - // self.imported.lock().as_ref().unwrap(), - // other.imported.lock().as_ref().unwrap(), - // ) - // } - - // pub fn _subtract( - // map1: &BTreeMap, - // map2: &BTreeMap, - // ) -> BTreeMap - // where - // T: Sub + Copy + Default, - // { - // if map1.len() != map2.len() { - // panic!("Can't subtract two arrays with a different length"); - // } - - // Self::_transform(map1, |(date, value, ..)| { - // map2.get(date) - // .map(|value2| *value - *value2) - // .unwrap_or_default() - // }) - // } - - // - // pub fn multiply(&self, other: &Self) -> BTreeMap - // where - // T: Mul + Copy + Default, - // { - // Self::_multiply( - // self.imported.lock().as_ref().unwrap(), - // other.imported.lock().as_ref().unwrap(), - // ) - // } - - // - // pub fn _multiply( - // map1: &BTreeMap, - // map2: &BTreeMap, - // ) -> BTreeMap - // where - // T: Mul + Copy + Default, - // { - // Self::_transform(map1, |(date, value, ..)| { - // map2.get(date) - // .map(|value2| *value * *value2) - // .unwrap_or_default() - // }) - // } - - // - // pub fn divide(&self, other: &Self) -> BTreeMap - // where - // T: Div + Copy + Default, - // { - // Self::_divide( - // self.imported.lock().as_ref().unwrap(), - // other.imported.lock().as_ref().unwrap(), - // ) - // } - - // - // pub fn _divide( - // map1: &BTreeMap, - // map2: &BTreeMap, - // ) -> BTreeMap - // where - // T: Div + Copy + Default, - // { - // Self::_transform(map1, |(date, value, ..)| { - // map2.get(date) - // .map(|value2| *value / *value2) - // .unwrap_or_default() - // }) - // } - - // - // pub fn cumulate(&self) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign, - // { - // Self::_cumulate(self.imported.lock().as_ref().unwrap()) - // } - - // - // pub fn _cumulate(map: &BTreeMap) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign, - // { - // let mut sum = T::default(); - - // map.iter() - // .map(|(date, value)| { - // sum += *value; - // (date.to_owned(), sum) - // }) - // .collect() - // } - - // pub fn insert_cumulative(&mut self, date: NaiveDate, source: &DateMap) -> T - // where - // T: Add + Sub, - // { - // let previous_cum = date - // .checked_sub_days(Days::new(1)) - // .map(|previous_date| { - // self.get(previous_date).unwrap_or_else(|| { - // if previous_date.year() == 2009 && previous_date.month() == 1 { - // let day = previous_date.day(); - - // if day == 8 { - // self.get(NaiveDate::from_str("2009-01-03").unwrap()) - // .unwrap() - // } else if day == 2 { - // T::default() - // } else { - // panic!() - // } - // } else { - // dbg!(previous_date, &self.path_all); - // panic!() - // } - // }) - // }) - // .unwrap_or_default(); - - // let last_value = source.get(date).unwrap(); - - // let cum_value = previous_cum + last_value; - - // self.insert(date, cum_value); - - // cum_value - // } - - // - // pub fn insert_last_x_sum(&mut self, date: NaiveDate, source: &DateMap, x: usize) -> T - // where - // T: Add + Sub, - // { - // let to_subtract = date - // .checked_sub_days(Days::new(x as u64 - 1)) - // .and_then(|previous_date| source.get(previous_date)) - // .unwrap_or_default(); - - // let previous_sum = date - // .checked_sub_days(Days::new(1)) - // .and_then(|previous_sum_date| self.get(previous_sum_date)) - // .unwrap_or_default(); - - // let last_value = source.get(date).unwrap(); - - // let sum = previous_sum - to_subtract + last_value; - - // self.insert(date, sum); - - // sum - // } - - // - // pub fn last_x_sum(&self, x: usize) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign + SubAssign, - // { - // Self::_last_x_sum(self.imported.lock().as_ref().unwrap(), x) - // } - - // pub fn _last_x_sum(map: &BTreeMap, days: usize) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign + SubAssign, - // { - // let mut sum = T::default(); - - // map.iter() - // .enumerate() - // .map(|(index, (date, value))| { - // sum += *value; - - // if index >= days - 1 { - // let previous_index = index + 1 - days; - - // sum -= *map.values().nth(previous_index).unwrap() - // } - - // (date.to_owned(), sum) - // }) - // .collect() - // } - - // - // pub fn simple_moving_average(&self, x: usize) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign + SubAssign + ToF32, - // { - // Self::_simple_moving_average(self.imported.lock().as_ref().unwrap(), x) - // } - - // pub fn insert_simple_average(&mut self, date: NaiveDate, source: &DateMap, x: usize) - // where - // T: Into + From, - // K: Clone - // + Copy - // + Default - // + Debug - // + Serialize - // + DeserializeOwned - // + Sum - // + savefile::Serialize - // + savefile::Deserialize - // + savefile::ReprC - // + ToF32, - // { - // let previous_average: f32 = date - // .checked_sub_days(Days::new(1)) - // .and_then(|previous_average_date| self.get(previous_average_date)) - // .unwrap_or_default() - // .into(); - - // let last_value: f32 = source.get(date).unwrap().to_f32(); - - // let sum = previous_average * x as f32 - 1.0 + last_value; - - // let average: T = (sum / x as f32).into(); - - // self.insert(date, average); - // } - - // - // pub fn _simple_moving_average( - // map: &BTreeMap, - // x: usize, - // ) -> BTreeMap - // where - // T: Sum + Copy + Default + AddAssign + SubAssign + Into, - // { - // let mut sum = T::default(); - - // map.iter() - // .enumerate() - // .map(|(index, (date, value))| { - // sum += *value; - - // if index >= x - 1 { - // sum -= *map.values().nth(index + 1 - x).unwrap() - // } - - // let float_sum: f32 = sum.into(); - - // (date.to_owned(), float_sum / x as f32) - // }) - // .collect() - // } - - // - // pub fn net_change(&self, offset: usize) -> BTreeMap - // where - // T: Copy + Default + Sub, - // { - // Self::_net_change(self.imported.lock().as_ref().unwrap(), offset) - // } - // - // - // pub fn insert_net_change(&mut self, date: NaiveDate, source: &DateMap, offset: usize) -> T - // where - // T: Sub, - // { - // let previous_value = date - // .checked_sub_days(Days::new(offset as u64)) - // .and_then(|date| source.get(date)) - // .unwrap_or_default(); - - // let last_value = source.get(date).unwrap_or_else(|| { - // dbg!(date); - // panic!(); - // }); - - // let net = last_value - previous_value; - - // self.insert(date, net); - - // net - // } - - // - // pub fn _net_change(map: &BTreeMap, offset: usize) -> BTreeMap - // where - // T: Copy + Default + Sub, - // { - // Self::_transform(map, |(_, value, map, index)| { - // let previous = { - // if let Some(previous_index) = index.checked_sub(offset) { - // *map.values().nth(previous_index).unwrap() - // } else { - // T::default() - // } - // }; - - // *value - previous - // }) - // } - - // - // pub fn _median(map: &BTreeMap, size: usize) -> BTreeMap> - // where - // T: FloatCore, - // { - // let even = size % 2 == 0; - // let median_index = size / 2; - - // if size < 3 { - // panic!("Computing a median for a size lower than 3 is useless"); - // } - - // map.iter() - // .enumerate() - // .map(|(index, (date, _))| { - // let value = { - // if index >= size - 1 { - // let mut vec = map - // .values() - // .rev() - // .take(size) - // .map(|v| OrderedFloat(*v)) - // .collect_vec(); - - // vec.sort_unstable(); - - // if even { - // Some( - // (**vec.get(median_index).unwrap() - // + **vec.get(median_index - 1).unwrap()) - // / T::from(2.0).unwrap(), - // ) - // } else { - // Some(**vec.get(median_index).unwrap()) - // } - // } else { - // None - // } - // }; - - // (date.to_owned(), value) - // }) - // .collect() - // } - // - // pub fn insert_median(&mut self, date: NaiveDate, source: &DateMap, size: usize) -> T - // where - // T: FloatCore, - // { - // if size < 3 { - // panic!("Computing a median for a size lower than 3 is useless"); - // } - - // let median = { - // if let Some(start) = date.checked_sub_days(Days::new(size as u64 - 1)) { - // let even = size % 2 == 0; - // let median_index = size / 2; - - // let mut vec = start - // .iter_days() - // .take(size) - // .flat_map(|date| source.get(date)) - // .map(|f| OrderedFloat(f)) - // .collect_vec(); - - // if vec.len() != size { - // return T::default(); - // } - - // vec.sort_unstable(); - - // if even { - // (vec.get(median_index).unwrap().0 + vec.get(median_index - 1).unwrap().0) - // / T::from(2.0).unwrap() - // } else { - // vec.get(median_index).unwrap().0 - // } - // } else { - // T::default() - // } - // }; - - // self.insert(date, median); - - // median - // } -} diff --git a/parser/src/structs/height_map copy.rs b/parser/src/structs/height_map copy.rs deleted file mode 100644 index d5722168e..000000000 --- a/parser/src/structs/height_map copy.rs +++ /dev/null @@ -1,986 +0,0 @@ -use std::{ - cmp::Ordering, - collections::{BTreeMap, VecDeque}, - fmt::Debug, - fs, - iter::Sum, - mem, - ops::{Add, ControlFlow, Div, Mul, RangeInclusive, Sub}, - path::{Path, PathBuf}, -}; - -use allocative::Allocative; -use bincode::{Decode, Encode}; -use itertools::Itertools; -use ordered_float::{FloatCore, OrderedFloat}; -use serde::{Deserialize, Serialize}; - -use crate::{ - bitcoin::NUMBER_OF_UNSAFE_BLOCKS, - io::{format_path, Serialization}, - utils::{log, LossyFrom}, -}; - -use super::{AnyMap, MapValue}; - -pub const HEIGHT_MAP_CHUNK_SIZE: usize = 10_000; - -#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)] -pub struct SerializedHeightMap { - version: u32, - map: Vec, -} - -#[derive(Default, Allocative)] -pub struct HeightMap -where - T: MapValue, -{ - version: u32, - - path_all: String, - path_last: Option, - - chunks_in_memory: usize, - - serialization: Serialization, - - initial_last_height: Option, - initial_first_unsafe_height: Option, - - imported: BTreeMap>, - to_insert: BTreeMap>, -} - -impl HeightMap -where - T: MapValue, -{ - pub fn new_bin(version: u32, path: &str) -> Self { - Self::new(version, path, Serialization::Binary, 1, true) - } - - pub fn _new_bin(version: u32, path: &str, export_last: bool) -> Self { - Self::new(version, path, Serialization::Binary, 1, export_last) - } - - pub fn new_json(version: u32, path: &str, export_last: bool) -> Self { - Self::new(version, path, Serialization::Json, usize::MAX, export_last) - } - - fn new( - version: u32, - path: &str, - serialization: Serialization, - chunks_in_memory: usize, - export_last: bool, - ) -> Self { - if chunks_in_memory < 1 { - panic!("Should always have at least the latest chunk in memory"); - } - - let path = format_path(path); - - let path_all = format!("{path}/height"); - - fs::create_dir_all(&path_all).unwrap(); - - let path_last = { - if export_last { - Some(serialization.append_extension(&format!("{path}/last"))) - } else { - None - } - }; - - let mut s = Self { - version, - - path_all, - path_last, - - chunks_in_memory, - - serialization, - - initial_first_unsafe_height: None, - initial_last_height: None, - - to_insert: BTreeMap::default(), - imported: BTreeMap::default(), - }; - - s.read_dir() - .into_iter() - .rev() - .take(chunks_in_memory) - .for_each(|(chunk_start, path)| { - if let Ok(serialized) = s.import(&path) { - if serialized.version == s.version { - s.imported.insert(chunk_start, serialized); - } else { - s.read_dir() - .iter() - .for_each(|(_, path)| fs::remove_file(path).unwrap()) - } - } - }); - - s.initial_last_height = s - .imported - .iter() - .last() - .map(|(chunk_start, serialized)| chunk_start + serialized.map.len()); - - s.initial_first_unsafe_height = s.initial_last_height.and_then(|last_height| { - let offset = NUMBER_OF_UNSAFE_BLOCKS - 1; - last_height.checked_sub(offset) - }); - - if s.initial_first_unsafe_height.is_none() { - log(&format!("New {path}")); - } - - s - } - - fn height_to_chunk_name(height: Height) -> String { - let start = Self::height_to_chunk_start(height); - let end = start + HEIGHT_MAP_CHUNK_SIZE; - - format!("{start}..{end}") - } - - fn height_to_chunk_start(height: Height) -> usize { - height / HEIGHT_MAP_CHUNK_SIZE * HEIGHT_MAP_CHUNK_SIZE - } - - pub fn insert(&mut self, height: Height, value: T) -> T { - if !self.is_height_safe(height) { - self.to_insert - .entry(Self::height_to_chunk_start(height)) - .or_default() - .insert(height % HEIGHT_MAP_CHUNK_SIZE, value); - } - - value - } - - pub fn insert_default(&mut self, height: Height) -> T { - self.insert(height, T::default()) - } - - pub fn get(&self, height: &usize) -> Option { - let chunk_start = Self::height_to_chunk_start(*height); - - self.to_insert - .get(&chunk_start) - .and_then(|map| map.get(&(height - chunk_start)).cloned()) - .or_else(|| { - self.imported - .get(&chunk_start) - .and_then(|serialized| serialized.map.get(height - chunk_start)) - .cloned() - }) - } - - pub fn get_or_import(&mut self, height: &usize) -> T { - let chunk_start = Self::height_to_chunk_start(*height); - - self.to_insert - .get(&chunk_start) - .and_then(|map| map.get(&(height - chunk_start)).cloned()) - .or_else(|| { - #[allow(clippy::map_entry)] // Can't be mut and then use read_dir() - if !self.imported.contains_key(&chunk_start) { - let dir_content = self.read_dir(); - - let path = dir_content.get(&chunk_start).unwrap_or_else(|| { - dbg!(self.path(), chunk_start, &dir_content); - panic!(); - }); - - let serialized = self.import(path).unwrap(); - - self.imported.insert(chunk_start, serialized); - } - - self.imported - .get(&chunk_start) - .and_then(|serialized| serialized.map.get(height - chunk_start)) - .cloned() - }) - .unwrap_or_else(|| { - dbg!(height, self.path()); - panic!(); - }) - } - - #[inline(always)] - pub fn is_height_safe(&self, height: Height) -> bool { - self.initial_first_unsafe_height.unwrap_or(0) > height - } - - fn read_dir(&self) -> BTreeMap { - Self::_read_dir(&self.path_all, &self.serialization) - } - - pub fn _read_dir(path: &str, serialization: &Serialization) -> BTreeMap { - fs::read_dir(path) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| { - let extension = path.extension().unwrap().to_str().unwrap(); - - path.is_file() && extension == serialization.to_extension() - }) - .map(|path| { - ( - path.file_stem() - .unwrap() - .to_str() - .unwrap() - .split("..") - .next() - .unwrap() - .parse::() - .unwrap(), - path, - ) - }) - .collect() - } - - fn import(&self, path: &Path) -> color_eyre::Result> { - self.serialization - .import::>(path.to_str().unwrap()) - } -} - -impl AnyMap for HeightMap -where - T: MapValue, -{ - fn path(&self) -> &str { - &self.path_all - } - - fn path_last(&self) -> &Option { - &self.path_last - } - - fn t_name(&self) -> &str { - std::any::type_name::() - } - - fn pre_export(&mut self) { - let to_insert = &mut self.to_insert; - - to_insert.iter_mut().for_each(|(chunk_start, map)| { - if let Some((key, _)) = map.first_key_value() { - if *key > 0 && !self.imported.contains_key(chunk_start) { - // Had to copy paste many lines from functions as calling a function from self isn't allowed because of the &mut - - let dir_content = Self::_read_dir(&self.path_all, &self.serialization); - - let path = dir_content.get(chunk_start).unwrap_or_else(|| { - dbg!(&self.path_all, chunk_start, &dir_content); - panic!(); - }); - - let serialized = self - .serialization - .import::>(path.to_str().unwrap()) - .unwrap(); - - self.imported.insert(*chunk_start, serialized); - } - } - - let serialized = self - .imported - .entry(*chunk_start) - .or_insert(SerializedHeightMap { - version: self.version, - map: vec![], - }); - - mem::take(map) - .into_iter() - .for_each( - |(chunk_height, value)| match serialized.map.len().cmp(&chunk_height) { - Ordering::Greater => serialized.map[chunk_height] = value, - Ordering::Equal => serialized.map.push(value), - Ordering::Less => { - dbg!(&self.path_all, &serialized.map, chunk_height, value); - panic!() - } - }, - ); - }); - } - - fn export(&self) -> color_eyre::Result<()> { - let len = self.imported.len(); - - self.to_insert.iter().enumerate().try_for_each( - |(index, (chunk_start, map))| -> color_eyre::Result<()> { - if !map.is_empty() { - unreachable!() - } - - let chunk_name = Self::height_to_chunk_name(*chunk_start); - - let path = self - .serialization - .append_extension(&format!("{}/{}", self.path_all, chunk_name)); - - let serialized = self.imported.get(chunk_start).unwrap_or_else(|| { - dbg!(&self.path_all, chunk_start, &self.imported); - panic!(); - }); - - self.serialization.export(&path, serialized)?; - - if index == len - 1 { - if let Some(path_last) = self.path_last.as_ref() { - self.serialization - .export(path_last, serialized.map.last().unwrap())?; - } - } - - Ok(()) - }, - ) - } - - fn post_export(&mut self) { - self.imported - .keys() - .rev() - .enumerate() - .filter(|(index, _)| *index + 1 > self.chunks_in_memory) - .map(|(_, key)| *key) - .collect_vec() - .iter() - .for_each(|key| { - self.imported.remove(key); - }); - - self.to_insert.clear(); - } -} - -pub trait AnyHeightMap: AnyMap { - fn get_initial_first_unsafe_height(&self) -> Option; - - fn get_initial_last_height(&self) -> Option; - - fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync); - - fn as_any_mut_map(&mut self) -> &mut dyn AnyMap; -} - -impl AnyHeightMap for HeightMap -where - T: MapValue, -{ - #[inline(always)] - fn get_initial_first_unsafe_height(&self) -> Option { - self.initial_first_unsafe_height - } - - #[inline(always)] - fn get_initial_last_height(&self) -> Option { - self.initial_last_height - } - - fn as_any_map(&self) -> &(dyn AnyMap + Send + Sync) { - self - } - - fn as_any_mut_map(&mut self) -> &mut dyn AnyMap { - self - } -} - -impl HeightMap -where - T: MapValue, -{ - pub fn sum_range(&self, range: &RangeInclusive) -> T - where - T: Sum, - { - range - .to_owned() - .flat_map(|height| self.get(&height)) - .sum::() - } - - pub fn multi_insert_const(&mut self, heights: &[Height], constant: T) { - heights.iter().for_each(|height| { - let height = *height; - - self.insert(height, constant); - }); - } - - pub fn multi_insert_simple_transform( - &mut self, - heights: &[Height], - source: &mut HeightMap, - transform: F, - ) where - K: MapValue, - F: Fn(K) -> T, - { - heights.iter().for_each(|height| { - self.insert(*height, transform(source.get_or_import(height))); - }); - } - - pub fn multi_insert_complex_transform( - &mut self, - heights: &[Height], - source: &mut HeightMap, - mut transform: F, - ) where - K: MapValue, - F: FnMut((K, &usize)) -> T, - { - heights.iter().for_each(|height| { - self.insert(*height, transform((source.get_or_import(height), height))); - }); - } - - pub fn multi_insert_add( - &mut self, - heights: &[Height], - added: &mut HeightMap, - adder: &mut HeightMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Add, - { - heights.iter().for_each(|height| { - self.insert( - *height, - T::lossy_from(added.get_or_import(height)) - + T::lossy_from(adder.get_or_import(height)), - ); - }); - } - - pub fn multi_insert_subtract( - &mut self, - heights: &[Height], - subtracted: &mut HeightMap, - subtracter: &mut HeightMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Sub, - { - heights.iter().for_each(|height| { - self.insert( - *height, - T::lossy_from(subtracted.get_or_import(height)) - - T::lossy_from(subtracter.get_or_import(height)), - ); - }); - } - - pub fn multi_insert_multiply( - &mut self, - heights: &[Height], - multiplied: &mut HeightMap, - multiplier: &mut HeightMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Mul, - { - heights.iter().for_each(|height| { - self.insert( - *height, - T::lossy_from(multiplied.get_or_import(height)) - * T::lossy_from(multiplier.get_or_import(height)), - ); - }); - } - - pub fn multi_insert_divide( - &mut self, - heights: &[Height], - divided: &mut HeightMap, - divider: &mut HeightMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - self._multi_insert_divide(heights, divided, divider, false) - } - - pub fn multi_insert_percentage( - &mut self, - heights: &[Height], - divided: &mut HeightMap, - divider: &mut HeightMap, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - self._multi_insert_divide(heights, divided, divider, true) - } - - pub fn _multi_insert_divide( - &mut self, - heights: &[Height], - divided: &mut HeightMap, - divider: &mut HeightMap, - as_percentage: bool, - ) where - A: MapValue, - B: MapValue, - T: LossyFrom + LossyFrom, - T: Div + Mul + From, - { - let multiplier = T::from(if as_percentage { 100 } else { 1 }); - - heights.iter().for_each(|height| { - self.insert( - *height, - T::lossy_from(divided.get_or_import(height)) - / T::lossy_from(divider.get_or_import(height)) - * multiplier, - ); - }); - } - - pub fn multi_insert_cumulative(&mut self, heights: &[Height], source: &mut HeightMap) - where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - self._multi_insert_last_x_sum(heights, source, None) - } - - pub fn multi_insert_last_x_sum( - &mut self, - heights: &[Height], - source: &mut HeightMap, - block_time: usize, - ) where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - self._multi_insert_last_x_sum(heights, source, Some(block_time)) - } - - fn _multi_insert_last_x_sum( - &mut self, - heights: &[Height], - source: &mut HeightMap, - block_time: Option, - ) where - K: MapValue, - T: LossyFrom, - T: Add + Sub, - { - let mut sum = None; - - heights.iter().for_each(|height| { - let to_subtract = block_time - .and_then(|x| { - (height + 1) - .checked_sub(x) - .map(|previous_height| source.get_or_import(&previous_height)) - }) - .unwrap_or_default(); - - let previous_sum = sum.unwrap_or_else(|| { - height - .checked_sub(1) - .map(|previous_sum_height| self.get_or_import(&previous_sum_height)) - .unwrap_or_default() - }); - - let last_value = source.get_or_import(height); - - sum.replace(previous_sum + T::lossy_from(last_value) - T::lossy_from(to_subtract)); - - self.insert(*height, sum.unwrap()); - }); - } - - pub fn multi_insert_simple_average( - &mut self, - heights: &[Height], - source: &mut HeightMap, - block_time: usize, - ) where - T: Into + From, - K: MapValue + Sum, - f32: LossyFrom, - { - if block_time <= 1 { - panic!("Average of 1 or less is not useful"); - } - - let mut average = None; - - heights.iter().for_each(|height| { - let height = *height; - - let previous_average: f32 = average - .unwrap_or_else(|| { - height - .checked_sub(block_time) - .and_then(|previous_average_height| self.get(&previous_average_height)) - .unwrap_or_default() - }) - .into(); - - let mut last_value = f32::lossy_from(source.get_or_import(&height)); - - if last_value.is_nan() { - last_value = 0.0; - } - - average.replace( - ((previous_average * (block_time as f32 - 1.0) + last_value) / block_time as f32) - .into(), - ); - - self.insert(height, average.unwrap()); - }); - } - - pub fn multi_insert_net_change( - &mut self, - heights: &[Height], - source: &mut HeightMap, - block_time: usize, - ) where - T: Sub, - { - heights.iter().for_each(|height| { - let height = *height; - - let previous_value = height - .checked_sub(block_time) - .map(|height| source.get_or_import(&height)) - .unwrap_or_default(); - - let last_value = source.get_or_import(&height); - - let net = last_value - previous_value; - - self.insert(height, net); - }); - } - - pub fn multi_insert_median( - &mut self, - heights: &[Height], - source: &mut HeightMap, - block_time: Option, - ) where - T: FloatCore, - { - source.multi_insert_percentile(heights, vec![(self, 0.5)], block_time); - } - - pub fn multi_insert_percentile( - &mut self, - heights: &[Height], - mut map_and_percentiles: Vec<(&mut HeightMap, f32)>, - block_time: Option, - ) where - T: FloatCore, - { - if block_time.map_or(false, |size| size < 3) { - panic!("Computing a percentile for a size lower than 3 is useless"); - } - - let mut ordered_vec = None; - let mut sorted_vec = None; - - let min_percentile_height = 160_000; - - let nan = T::from(f32::NAN).unwrap(); - let two = T::from(2.0).unwrap(); - - if min_percentile_height % HEIGHT_MAP_CHUNK_SIZE != 0 { - panic!("Should be 0"); - } - - heights.iter().cloned().try_for_each(|height| { - if height < min_percentile_height { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(height, nan); - }); - return ControlFlow::Continue::<()>(()); - } - - if let Some(start) = - block_time.map_or(Some(min_percentile_height), |size| height.checked_sub(size)) - { - if sorted_vec.is_none() { - let mut vec = (start..=height) - .map(|height| self.get_or_import(&height)) - .filter(|f| !f.is_nan()) - .map(|f| OrderedFloat(f)) - .collect_vec(); - - if block_time.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } - - vec.sort_unstable(); - - sorted_vec.replace(vec); - } else { - let float_value = self.get_or_import(&height); - - if !float_value.is_nan() { - let float_value = OrderedFloat(float_value); - - if block_time.is_some() { - let first = ordered_vec.as_mut().unwrap().pop_front().unwrap(); - let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); - sorted_vec.as_mut().unwrap().remove(pos); - - ordered_vec.as_mut().unwrap().push_back(float_value); - } - - let pos = sorted_vec - .as_ref() - .unwrap() - .binary_search(&float_value) - .unwrap_or_else(|pos| pos); - - sorted_vec.as_mut().unwrap().insert(pos, float_value); - } - } - - let vec = sorted_vec.as_ref().unwrap(); - - let len = vec.len(); - - map_and_percentiles - .iter_mut() - .for_each(|(map, percentile)| { - if !(0.0..=1.0).contains(percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } - - let value = { - if len < 2 { - nan - } else { - let index = (len - 1) as f32 * *percentile; - - let fract = index.fract(); - - if fract != 0.0 { - (vec.get(index.ceil() as usize) - .unwrap_or_else(|| { - dbg!( - index, - &self.path_all, - &self.path_all, - &self.to_insert, - block_time, - vec - ); - panic!() - }) - .0 - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!( - index, - &self.path_all, - &self.path_all, - block_time - ); - panic!() - }) - .0) - / two - } else { - vec.get(index as usize).unwrap().0 - } - } - }; - - (*map).insert(height, value); - }); - } else { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(height, nan); - }); - } - - ControlFlow::Continue(()) - }); - } - - // pub fn insert_cumulative(&mut self, height: Height, source: &HeightMap) -> T - // where - // T: Add + Sub, - // { - // let previous_cum = height - // .checked_sub(1) - // .map(|previous_sum_height| { - // self.get(&previous_sum_height).unwrap_or_else(|| { - // dbg!(previous_sum_height); - // panic!() - // }) - // }) - // .unwrap_or_default(); - - // let last_value = source.get(&height).unwrap(); - - // let cum_value = previous_cum + last_value; - - // self.insert(height, cum_value); - - // cum_value - // } - - // pub fn insert_last_x_sum(&mut self, height: Height, source: &HeightMap, x: usize) -> T - // where - // T: Add + Sub, - // { - // let to_subtract = (height + 1) - // .checked_sub(x) - // .map(|previous_height| { - // source.get(&previous_height).unwrap_or_else(|| { - // dbg!(&self.path_all, &source.path_all, previous_height); - // panic!() - // }) - // }) - // .unwrap_or_default(); - - // let previous_sum = height - // .checked_sub(1) - // .map(|previous_sum_height| self.get(&previous_sum_height).unwrap()) - // .unwrap_or_default(); - - // let last_value = source.get(&height).unwrap(); - - // let sum = previous_sum + last_value - to_subtract; - - // self.insert(height, sum); - - // sum - // } - - // pub fn insert_simple_average(&mut self, height: Height, source: &HeightMap, block_time: usize) - // where - // T: Into + From, - // { - // let to_subtract: f32 = (height + 1) - // .checked_sub(block_time) - // .map(|previous_height| source.get(&previous_height).unwrap()) - // .unwrap_or_default() - // .into(); - - // let previous_average: f32 = height - // .checked_sub(1) - // .map(|previous_average_height| self.get(&previous_average_height).unwrap()) - // .unwrap_or_default() - // .into(); - - // let last_value: f32 = source.get(&height).unwrap().into(); - - // let sum = previous_average * block_time as f32 - to_subtract + last_value; - - // let average: T = (sum / block_time as f32).into(); - - // self.insert(height, average); - // } - - // pub fn insert_net_change(&mut self, height: Height, source: &HeightMap, offset: usize) -> T - // where - // T: Sub, - // { - // let previous_value = height - // .checked_sub(offset) - // .map(|height| { - // source.get(&height).unwrap_or_else(|| { - // dbg!(&self.path_all, &source.path_all, offset); - // panic!(); - // }) - // }) - // .unwrap_or_default(); - - // let last_value = source.get(&height).unwrap(); - - // let net = last_value - previous_value; - - // self.insert(height, net); - - // net - // } - - // pub fn insert_median(&mut self, height: Height, source: &HeightMap, size: usize) -> T - // where - // T: FloatCore, - // { - // if size < 3 { - // panic!("Computing a median for a size lower than 3 is useless"); - // } - - // let median = { - // if let Some(start) = height.checked_sub(size - 1) { - // let even = size % 2 == 0; - // let median_index = size / 2; - - // let mut vec = (start..=height) - // .map(|height| { - // OrderedFloat(source.get(&height).unwrap_or_else(|| { - // dbg!(height, &source.path_all, size); - // panic!() - // })) - // }) - // .collect_vec(); - - // vec.sort_unstable(); - - // if even { - // (vec.get(median_index) - // .unwrap_or_else(|| { - // dbg!(median_index, &self.path_all, &source.path_all, size); - // panic!() - // }) - // .0 - // + vec.get(median_index - 1).unwrap().0) - // / T::from(2.0).unwrap() - // } else { - // vec.get(median_index).unwrap().0 - // } - // } else { - // T::default() - // } - // }; - - // self.insert(height, median); - - // median - // } -}