diff --git a/parser/src/datasets/cointime.rs b/parser/src/datasets/cointime.rs index 9f247a73d..cafc63c75 100644 --- a/parser/src/datasets/cointime.rs +++ b/parser/src/datasets/cointime.rs @@ -86,7 +86,7 @@ impl CointimeDataset { cointime_value_stored: BiMap::new_bin(1, &f("cointime_value_stored")), concurrent_liveliness: BiMap::new_bin(1, &f("concurrent_liveliness")), concurrent_liveliness_2w_median: BiMap::new_bin( - 1, + 2, &f("concurrent_liveliness_2w_median"), ), cumulative_coinblocks_created: BiMap::new_bin(1, &f("cumulative_coinblocks_created")), @@ -100,7 +100,7 @@ impl CointimeDataset { liveliness: BiMap::new_bin(1, &f("liveliness")), liveliness_net_change: BiMap::new_bin(1, &f("liveliness_net_change")), liveliness_net_change_2w_median: BiMap::new_bin( - 2, + 3, &f("liveliness_net_change_2w_median"), ), producerness: BiMap::new_bin(1, &f("producerness")), diff --git a/parser/src/datasets/subs/ratio.rs b/parser/src/datasets/subs/ratio.rs index acdd321fb..a1aefb407 100644 --- a/parser/src/datasets/subs/ratio.rs +++ b/parser/src/datasets/subs/ratio.rs @@ -46,18 +46,18 @@ impl RatioDataset { 2, &f_ratio("ratio_1y_sma_momentum_oscillator"), ), - ratio_99p: BiMap::new_bin(1, &f_ratio("ratio_99p")), - ratio_99_5p: BiMap::new_bin(1, &f_ratio("ratio_99_5p")), - ratio_99_9p: BiMap::new_bin(1, &f_ratio("ratio_99_9p")), - ratio_1p: BiMap::new_bin(1, &f_ratio("ratio_1p")), - ratio_0_5p: BiMap::new_bin(1, &f_ratio("ratio_0_5p")), - ratio_0_1p: BiMap::new_bin(1, &f_ratio("ratio_0_1p")), - price_99p: BiMap::new_bin(1, &f_price("99p")), - price_99_5p: BiMap::new_bin(1, &f_price("99_5p")), - price_99_9p: BiMap::new_bin(1, &f_price("99_9p")), - price_1p: BiMap::new_bin(1, &f_price("1p")), - price_0_5p: BiMap::new_bin(1, &f_price("0_5p")), - price_0_1p: BiMap::new_bin(1, &f_price("0_1p")), + ratio_99p: BiMap::new_bin(3, &f_ratio("ratio_99p")), + ratio_99_5p: BiMap::new_bin(3, &f_ratio("ratio_99_5p")), + ratio_99_9p: BiMap::new_bin(3, &f_ratio("ratio_99_9p")), + ratio_1p: BiMap::new_bin(3, &f_ratio("ratio_1p")), + ratio_0_5p: BiMap::new_bin(3, &f_ratio("ratio_0_5p")), + ratio_0_1p: BiMap::new_bin(3, &f_ratio("ratio_0_1p")), + price_99p: BiMap::new_bin(3, &f_price("99p")), + price_99_5p: BiMap::new_bin(3, &f_price("99_5p")), + price_99_9p: BiMap::new_bin(3, &f_price("99_9p")), + price_1p: BiMap::new_bin(3, &f_price("1p")), + price_0_5p: BiMap::new_bin(3, &f_price("0_5p")), + price_0_1p: BiMap::new_bin(3, &f_price("0_1p")), }; s.min_initial_states diff --git a/parser/src/structs/date_map.rs b/parser/src/structs/date_map.rs index 62c3228f4..3e962f215 100644 --- a/parser/src/structs/date_map.rs +++ b/parser/src/structs/date_map.rs @@ -4,7 +4,7 @@ use std::{ fs, iter::Sum, mem, - ops::{Add, Div, Mul, Sub}, + ops::{Add, ControlFlow, Div, Mul, Sub}, path::{Path, PathBuf}, }; @@ -731,30 +731,37 @@ where let min_percentile_date = chrono::NaiveDate::from_ymd_opt(2012, 1, 1).unwrap(); let min_percentile_wdate = WNaiveDate::wrap(min_percentile_date); - dates - .iter() - .cloned() - .filter(|date| date < &min_percentile_wdate) - .for_each(|date| { - if let Some(start) = days.map_or(Some(min_percentile_date), |size| { - date.checked_sub_days(Days::new(size as u64)) - }) { - if sorted_vec.is_none() { - let mut vec = start - .iter_days() - .take_while(|d| *d <= *date) - .flat_map(|date| self.get_or_import(&WNaiveDate::wrap(date))) - .map(|f| OrderedFloat(f)) - .collect_vec(); + dates.iter().cloned().try_for_each(|date| { + if date < min_percentile_wdate { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(date, T::from(f32::NAN).unwrap()); + }); + return ControlFlow::Continue::<()>(()); + } - if days.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } + if let Some(start) = days.map_or(Some(min_percentile_date), |size| { + date.checked_sub_days(Days::new(size as u64)) + }) { + if sorted_vec.is_none() { + let mut vec = start + .iter_days() + .take_while(|d| *d <= *date) + .flat_map(|date| self.get_or_import(&WNaiveDate::wrap(date))) + .filter(|f| !f.is_nan()) + .map(|f| OrderedFloat(f)) + .collect_vec(); - vec.sort_unstable(); - sorted_vec.replace(vec); - } else { - let float_value = OrderedFloat(self.get_or_import(&date).unwrap()); + if days.is_some() { + ordered_vec.replace(VecDeque::from(vec.clone())); + } + + vec.sort_unstable(); + sorted_vec.replace(vec); + } else { + let float_value = self.get_or_import(&date).unwrap(); + + if !float_value.is_nan() { + let float_value = OrderedFloat(float_value); if let Some(days) = days { if let Some(ordered_vec) = ordered_vec.as_mut() { @@ -779,29 +786,38 @@ where sorted_vec.as_mut().unwrap().insert(pos, float_value); } + } - let vec = sorted_vec.as_ref().unwrap(); + let vec = sorted_vec.as_ref().unwrap(); - let len = vec.len(); + let len = vec.len(); - map_and_percentiles - .iter_mut() - .for_each(|(map, percentile)| { - if !(0.0..=1.0).contains(percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } + map_and_percentiles + .iter_mut() + .for_each(|(map, percentile)| { + if !(0.0..=1.0).contains(percentile) { + panic!("The percentile should be between 0.0 and 1.0"); + } - let value = { - if vec.is_empty() { - T::default() - } else { - let index = (len - 1) as f32 * *percentile; + let value = { + if vec.is_empty() { + T::default() + } else { + let index = (len - 1) as f32 * *percentile; - let fract = index.fract(); - let fract_t = T::from(fract).unwrap(); + let fract = index.fract(); + let fract_t = T::from(fract).unwrap(); - if fract != 0.0 { - (vec.get(index.ceil() as usize) + if fract != 0.0 { + (vec.get(index.ceil() as usize) + .unwrap_or_else(|| { + dbg!(vec, index, &self.path_all, &self.path_all, days); + panic!() + }) + .0 + * fract_t + + vec + .get(index.floor() as usize) .unwrap_or_else(|| { dbg!( vec, @@ -812,41 +828,29 @@ where ); panic!() }) - .0 - * fract_t - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!( - vec, - index, - &self.path_all, - &self.path_all, - days - ); - panic!() - }) - .0) - * T::from(1.0 - fract).unwrap() - } else { - vec.get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!(vec, index); - panic!(); - }) - .0 - } + .0) + * T::from(1.0 - fract).unwrap() + } else { + vec.get(index.floor() as usize) + .unwrap_or_else(|| { + dbg!(vec, index); + panic!(); + }) + .0 } - }; + } + }; - (*map).insert(date, value); - }); - } else { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(date, T::default()); + (*map).insert(date, value); }); - } - }); + } else { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(date, T::default()); + }); + } + + ControlFlow::Continue(()) + }); } // diff --git a/parser/src/structs/height_map.rs b/parser/src/structs/height_map.rs index ef25df39e..1b27e8f25 100644 --- a/parser/src/structs/height_map.rs +++ b/parser/src/structs/height_map.rs @@ -5,7 +5,7 @@ use std::{ fs, iter::Sum, mem, - ops::{Add, Div, Mul, RangeInclusive, Sub}, + ops::{Add, ControlFlow, Div, Mul, RangeInclusive, Sub}, path::{Path, PathBuf}, }; @@ -723,28 +723,36 @@ where panic!("Should be 0"); } - heights - .iter() - .cloned() - .filter(|height| height < &min_percentile_height) - .for_each(|height| { - if let Some(start) = - block_time.map_or(Some(min_percentile_height), |size| height.checked_sub(size)) - { - if sorted_vec.is_none() { - let mut vec = (start..=height) - .map(|height| OrderedFloat(self.get_or_import(&height))) - .collect_vec(); + heights.iter().cloned().try_for_each(|height| { + if height < min_percentile_height { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(height, T::from(f32::NAN).unwrap()); + }); + return ControlFlow::Continue::<()>(()); + } - if block_time.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } + if let Some(start) = + block_time.map_or(Some(min_percentile_height), |size| height.checked_sub(size)) + { + if sorted_vec.is_none() { + let mut vec = (start..=height) + .map(|height| self.get_or_import(&height)) + .filter(|f| !f.is_nan()) + .map(|f| OrderedFloat(f)) + .collect_vec(); - vec.sort_unstable(); + if block_time.is_some() { + ordered_vec.replace(VecDeque::from(vec.clone())); + } - sorted_vec.replace(vec); - } else { - let float_value = OrderedFloat(self.get_or_import(&height)); + vec.sort_unstable(); + + sorted_vec.replace(vec); + } else { + let float_value = self.get_or_import(&height); + + if !float_value.is_nan() { + let float_value = OrderedFloat(float_value); if block_time.is_some() { let first = ordered_vec.as_mut().unwrap().pop_front().unwrap(); @@ -762,69 +770,72 @@ where sorted_vec.as_mut().unwrap().insert(pos, float_value); } + } - let vec = sorted_vec.as_ref().unwrap(); + let vec = sorted_vec.as_ref().unwrap(); - let len = vec.len(); + let len = vec.len(); - map_and_percentiles - .iter_mut() - .for_each(|(map, percentile)| { - if !(0.0..=1.0).contains(percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } + map_and_percentiles + .iter_mut() + .for_each(|(map, percentile)| { + if !(0.0..=1.0).contains(percentile) { + panic!("The percentile should be between 0.0 and 1.0"); + } - let value = { - if vec.is_empty() { - T::default() - } else { - let index = (len - 1) as f32 * *percentile; + let value = { + if vec.is_empty() { + T::default() + } else { + let index = (len - 1) as f32 * *percentile; - let fract = index.fract(); - let fract_t = T::from(fract).unwrap(); + let fract = index.fract(); + let fract_t = T::from(fract).unwrap(); - if fract != 0.0 { - (vec.get(index.ceil() as usize) + if fract != 0.0 { + (vec.get(index.ceil() as usize) + .unwrap_or_else(|| { + dbg!( + index, + &self.path_all, + &self.path_all, + &self.to_insert, + block_time, + vec + ); + panic!() + }) + .0 + * fract_t + + vec + .get(index.floor() as usize) .unwrap_or_else(|| { dbg!( index, &self.path_all, &self.path_all, - &self.to_insert, - block_time, - vec + block_time ); panic!() }) - .0 - * fract_t - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!( - index, - &self.path_all, - &self.path_all, - block_time - ); - panic!() - }) - .0) - * T::from(1.0 - fract).unwrap() - } else { - vec.get(index as usize).unwrap().0 - } + .0) + * T::from(1.0 - fract).unwrap() + } else { + vec.get(index as usize).unwrap().0 } - }; + } + }; - (*map).insert(height, value); - }); - } else { - map_and_percentiles.iter_mut().for_each(|(map, _)| { - (*map).insert(height, T::default()); + (*map).insert(height, value); }); - } - }); + } else { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(height, T::default()); + }); + } + + ControlFlow::Continue(()) + }); } // pub fn insert_cumulative(&mut self, height: usize, source: &HeightMap) -> T