git: reset

parser/src/utils/arr.rs

@@ -0,0 +1,219 @@
+use std::{
+    iter::Sum,
+    ops::{Add, AddAssign, Div, Mul, Sub, SubAssign},
+};
+
+use itertools::Itertools;
+use ordered_float::{FloatCore, OrderedFloat};
+
+use super::ToF32;
+
+pub trait ArrayOperations<T> {
+    fn transform<F>(&self, transform: F) -> Vec<T>
+    where
+        T: Copy + Default,
+        F: Fn((usize, &T, &[T])) -> T;
+
+    fn add(&self, other: &[T]) -> Vec<T>
+    where
+        T: Add<Output = T> + Copy + Default;
+
+    fn subtract(&self, other: &[T]) -> Vec<T>
+    where
+        T: Sub<Output = T> + Copy + Default;
+
+    fn multiply(&self, other: &[T]) -> Vec<T>
+    where
+        T: Mul<Output = T> + Copy + Default;
+
+    fn divide(&self, other: &[T]) -> Vec<T>
+    where
+        T: Div<Output = T> + Copy + Default;
+
+    fn match_size<'a>(&'a self, other: &'a [T]) -> &'a [T];
+
+    fn cumulate(&self) -> Vec<T>
+    where
+        T: Sum + Copy + Default + AddAssign;
+
+    fn last_x_sum(&self, x: usize) -> Vec<T>
+    where
+        T: Sum + Copy + Default + AddAssign + SubAssign;
+
+    fn moving_average(&self, x: usize) -> Vec<f32>
+    where
+        T: Sum + Copy + Default + AddAssign + SubAssign + ToF32;
+
+    fn net_change(&self, offset: usize) -> Vec<T>
+    where
+        T: Copy + Default + Sub<Output = T>;
+
+    fn median(&self, size: usize) -> Vec<Option<T>>
+    where
+        T: FloatCore;
+}
+
+impl<T> ArrayOperations<T> for &[T] {
+    fn transform<F>(&self, transform: F) -> Vec<T>
+    where
+        T: Copy + Default,
+        F: Fn((usize, &T, &[T])) -> T,
+    {
+        self.iter()
+            .enumerate()
+            .map(|(index, value)| transform((index, value, self)))
+            .collect_vec()
+    }
+
+    fn add(&self, other: &[T]) -> Vec<T>
+    where
+        T: Add<Output = T> + Copy + Default,
+    {
+        self.match_size(other)
+            .transform(|(index, value, _)| *value + *other.get(index).unwrap())
+    }
+
+    fn subtract(&self, other: &[T]) -> Vec<T>
+    where
+        T: Sub<Output = T> + Copy + Default,
+    {
+        self.match_size(other)
+            .transform(|(index, value, _)| *value - *other.get(index).unwrap())
+    }
+
+    fn multiply(&self, other: &[T]) -> Vec<T>
+    where
+        T: Mul<Output = T> + Copy + Default,
+    {
+        self.match_size(other)
+            .transform(|(index, value, _)| *value * *other.get(index).unwrap())
+    }
+
+    fn divide(&self, other: &[T]) -> Vec<T>
+    where
+        T: Div<Output = T> + Copy + Default,
+    {
+        self.match_size(other)
+            .transform(|(index, value, _)| *value / *other.get(index).unwrap())
+    }
+
+    fn match_size(&self, other: &[T]) -> &[T] {
+        let len = other.len();
+        if self.len() > len {
+            &self[..len]
+        } else {
+            self
+        }
+    }
+
+    fn cumulate(&self) -> Vec<T>
+    where
+        T: Sum + Copy + Default + AddAssign,
+    {
+        let mut sum = T::default();
+
+        self.iter()
+            .map(|value| {
+                sum += *value;
+                sum
+            })
+            .collect_vec()
+    }
+
+    fn last_x_sum(&self, x: usize) -> Vec<T>
+    where
+        T: Sum + Copy + Default + AddAssign + SubAssign,
+    {
+        let mut sum = T::default();
+
+        self.iter()
+            .enumerate()
+            .map(|(index, value)| {
+                sum += *value;
+
+                if index >= x - 1 {
+                    let previous_index = index + 1 - x;
+
+                    sum -= *self.get(previous_index).unwrap()
+                }
+
+                sum
+            })
+            .collect_vec()
+    }
+
+    fn moving_average(&self, x: usize) -> Vec<f32>
+    where
+        T: Sum + Copy + Default + AddAssign + SubAssign + ToF32,
+    {
+        let mut sum = T::default();
+
+        self.iter()
+            .enumerate()
+            .map(|(index, value)| {
+                sum += *value;
+
+                if index >= x - 1 {
+                    sum -= *self.get(index + 1 - x).unwrap()
+                }
+
+                sum.to_f32() / x as f32
+            })
+            .collect_vec()
+    }
+
+    fn net_change(&self, offset: usize) -> Vec<T>
+    where
+        T: Copy + Default + Sub<Output = T>,
+    {
+        self.transform(|(index, value, arr)| {
+            let previous = {
+                if let Some(previous_index) = index.checked_sub(offset) {
+                    *arr.get(previous_index).unwrap()
+                } else {
+                    T::default()
+                }
+            };
+
+            *value - previous
+        })
+    }
+
+    fn median(&self, size: usize) -> Vec<Option<T>>
+    where
+        T: FloatCore,
+    {
+        let even = size % 2 == 0;
+        let median_index = size / 2;
+
+        if size < 3 {
+            panic!("Computing a median for a size lower than 3 is useless");
+        }
+
+        self.iter()
+            .enumerate()
+            .map(|(index, _)| {
+                if index >= size - 1 {
+                    let mut arr = self[index - (size - 1)..index + 1]
+                        .iter()
+                        .map(|value| OrderedFloat(*value))
+                        .collect_vec();
+
+                    arr.sort_unstable();
+
+                    if even {
+                        Some(
+                            (**arr.get(median_index).unwrap()
+                                + **arr.get(median_index - 1).unwrap())
+                                / T::from(2.0).unwrap(),
+                        )
+                    } else {
+                        Some(**arr.get(median_index).unwrap())
+                    }
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+}

parser/src/utils/bytes.rs

@@ -0,0 +1 @@
+pub const BYTES_IN_MB: usize = 1_000_000;

parser/src/utils/date.rs

@@ -0,0 +1,10 @@
+pub const ONE_DAY_IN_DAYS: usize = 1;
+pub const ONE_WEEK_IN_DAYS: usize = 7;
+pub const TWO_WEEK_IN_DAYS: usize = 2 * ONE_WEEK_IN_DAYS;
+pub const ONE_MONTH_IN_DAYS: usize = 30;
+pub const THREE_MONTHS_IN_DAYS: usize = 3 * ONE_MONTH_IN_DAYS;
+pub const ONE_YEAR_IN_DAYS: usize = 365;
+
+pub const ONE_MINUTE_IN_S: usize = 60;
+pub const ONE_HOUR_IN_S: usize = 60 * ONE_MINUTE_IN_S;
+pub const ONE_DAY_IN_S: usize = 24 * ONE_HOUR_IN_S;

parser/src/utils/flamegraph.rs

@@ -0,0 +1,42 @@
+use std::{fs, path::PathBuf};
+
+use chrono::Local;
+
+use crate::{databases::Databases, datasets::AllDatasets, states::States};
+
+pub fn generate_allocation_files(
+    datasets: &AllDatasets,
+    databases: &Databases,
+    states: &States,
+    last_height: usize,
+) -> color_eyre::Result<()> {
+    let mut flamegraph = allocative::FlameGraphBuilder::default();
+    flamegraph.visit_root(datasets);
+    flamegraph.visit_root(databases);
+    flamegraph.visit_root(states);
+    let output = flamegraph.finish();
+
+    let folder = format!(
+        "at-{}-result-of-{}",
+        Local::now().format("%Y-%m-%d_%Hh%Mm%Ss"),
+        last_height
+    );
+
+    let path = PathBuf::from(&format!("./target/flamegraph/{folder}"));
+    fs::create_dir_all(&path)?;
+
+    // fs::write(path.join("flamegraph.src"), &output.flamegraph())?;
+
+    let mut fg_svg = Vec::new();
+    inferno::flamegraph::from_reader(
+        &mut inferno::flamegraph::Options::default(),
+        output.flamegraph().write().as_bytes(),
+        &mut fg_svg,
+    )?;
+
+    fs::write(path.join("flamegraph.svg"), &fg_svg)?;
+
+    fs::write(path.join("warnings.txt"), output.warnings())?;
+
+    Ok(())
+}

parser/src/utils/log.rs

@@ -0,0 +1,25 @@
+use std::process::Output;
+
+use chrono::Local;
+use color_eyre::owo_colors::OwoColorize;
+
+#[inline(always)]
+pub fn log(str: &str) {
+    let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -"));
+
+    str.lines()
+        .filter(|line| !line.is_empty())
+        .for_each(|line| {
+            println!("{} {}", date_time.bright_black(), line);
+        });
+}
+
+pub fn log_output(output: &Output) {
+    if !output.stdout.is_empty() {
+        log(&String::from_utf8_lossy(&output.stdout));
+    }
+
+    if !output.stderr.is_empty() {
+        log(&String::from_utf8_lossy(&output.stderr));
+    }
+}

parser/src/utils/lossy.rs

@@ -0,0 +1,110 @@
+pub trait LossyFrom<T> {
+    fn lossy_from(x: T) -> Self;
+}
+
+// ---
+// u64
+// ---
+
+impl LossyFrom<u64> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<usize> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as u64
+    }
+}
+
+// ---
+// usize
+// ---
+
+impl LossyFrom<usize> for usize {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<f32> for usize {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x.round() as usize
+    }
+}
+
+// ---
+// f32
+// ---
+
+impl LossyFrom<u32> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: u32) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<u64> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<usize> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<f32> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<f64> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        x as f32
+    }
+}
+
+// ---
+// f64
+// ---
+
+impl LossyFrom<u64> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<usize> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<f32> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<f64> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        x
+    }
+}

parser/src/utils/mod.rs

@@ -0,0 +1,15 @@
+mod bytes;
+mod date;
+mod flamegraph;
+mod log;
+mod lossy;
+mod retry;
+mod time;
+
+pub use bytes::*;
+pub use date::*;
+pub use flamegraph::*;
+pub use log::*;
+pub use lossy::*;
+pub use retry::*;
+pub use time::*;

parser/src/utils/retry.rs

@@ -0,0 +1,25 @@
+use std::{thread::sleep, time::Duration};
+
+pub fn retry<T>(
+    function: impl Fn() -> color_eyre::Result<T>,
+    sleep_in_s: u64,
+    retries: u64,
+) -> color_eyre::Result<T> {
+    if retries < 1 {
+        unreachable!()
+    }
+
+    let mut i = 0;
+
+    loop {
+        let res = function();
+
+        if i == retries || res.is_ok() {
+            return res;
+        } else {
+            sleep(Duration::from_secs(sleep_in_s));
+        }
+
+        i += 1;
+    }
+}

parser/src/utils/time.rs

@@ -0,0 +1,26 @@
+use std::time::Instant;
+
+use crate::utils::log;
+
+use super::ONE_DAY_IN_S;
+
+pub fn time<F, T>(name: &str, function: F) -> T
+where
+    F: FnOnce() -> T,
+{
+    let time = Instant::now();
+
+    let returned = function();
+
+    log(&format!("{name}: {} seconds", time.elapsed().as_secs_f32()));
+
+    returned
+}
+
+pub fn difference_in_days_between_timestamps(older: u32, younger: u32) -> u32 {
+    if younger <= older {
+        0
+    } else {
+        (younger - older) / ONE_DAY_IN_S as u32
+    }
+}