brk: first commit

_src/utils/consts.rs

@@ -0,0 +1,14 @@
+pub const BYTES_IN_MB: usize = 1_000_000;
+
+pub const TARGET_BLOCKS_PER_DAY: usize = 144;
+
+pub const ONE_DAY_IN_DAYS: usize = 1;
+pub const ONE_WEEK_IN_DAYS: usize = 7;
+pub const TWO_WEEK_IN_DAYS: usize = 2 * ONE_WEEK_IN_DAYS;
+pub const ONE_MONTH_IN_DAYS: usize = 30;
+pub const THREE_MONTHS_IN_DAYS: usize = 3 * ONE_MONTH_IN_DAYS;
+pub const ONE_YEAR_IN_DAYS: usize = 365;
+
+pub const ONE_MINUTE_IN_S: usize = 60;
+pub const ONE_HOUR_IN_S: usize = 60 * ONE_MINUTE_IN_S;
+pub const ONE_DAY_IN_S: usize = 24 * ONE_HOUR_IN_S;

_src/utils/flamegraph.rs

@@ -0,0 +1,47 @@
+use std::{fs, path::PathBuf};
+
+use chrono::Local;
+
+use crate::{
+    parser::{Databases, Datasets, States},
+    structs::Height,
+};
+
+// use crate::{databases::Databases, datasets::AllDatasets, states::States, structs::Height};
+
+pub fn generate_allocation_files(
+    datasets: &Datasets,
+    databases: &Databases,
+    states: &States,
+    last_height: Height,
+) -> color_eyre::Result<()> {
+    let mut flamegraph = allocative::FlameGraphBuilder::default();
+    flamegraph.visit_root(datasets);
+    flamegraph.visit_root(databases);
+    flamegraph.visit_root(states);
+    let output = flamegraph.finish();
+
+    let folder = format!(
+        "at-{}-result-of-{}",
+        Local::now().format("%Y-%m-%d_%Hh%Mm%Ss"),
+        last_height
+    );
+
+    let path = PathBuf::from(&format!("./target/flamegraph/{folder}"));
+    fs::create_dir_all(&path)?;
+
+    // fs::write(path.join("flamegraph.src"), &output.flamegraph())?;
+
+    let mut fg_svg = Vec::new();
+    inferno::flamegraph::from_reader(
+        &mut inferno::flamegraph::Options::default(),
+        output.flamegraph().write().as_bytes(),
+        &mut fg_svg,
+    )?;
+
+    fs::write(path.join("flamegraph.svg"), &fg_svg)?;
+
+    fs::write(path.join("warnings.txt"), output.warnings())?;
+
+    Ok(())
+}

_src/utils/log.rs

@@ -0,0 +1,82 @@
+use std::{
+    fmt::Display,
+    fs::{self, OpenOptions},
+    io::Write,
+};
+
+use chrono::Local;
+use color_eyre::owo_colors::OwoColorize;
+use env_logger::Env;
+
+use crate::structs::Config;
+
+#[inline(always)]
+pub fn init_log() {
+    let _ = fs::remove_file(Config::path_log());
+
+    let file = Box::new(
+        OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(Config::path_log())
+            .unwrap(),
+    );
+
+    env_logger::Builder::from_env(
+        Env::default().default_filter_or(format!("{}=info", env!("CARGO_PKG_NAME"))),
+    )
+    .format(move |buf, record| {
+        let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S"));
+        let level = record.level().as_str().to_lowercase();
+        let level = format!("{:5}", level);
+        let target = record.target();
+        let dash = "-";
+        let args = record.args();
+
+        let _ = write(
+            file.try_clone().unwrap(),
+            &date_time,
+            target,
+            &level,
+            dash,
+            args,
+        );
+
+        let colored_date_time = date_time.bright_black();
+        let colored_level = match level.chars().next().unwrap() {
+            'e' => level.red().to_string(),
+            'w' => level.yellow().to_string(),
+            'i' => level.green().to_string(),
+            'd' => level.blue().to_string(),
+            't' => level.cyan().to_string(),
+            _ => panic!(),
+        };
+        let colored_dash = dash.bright_black();
+
+        write(
+            buf,
+            colored_date_time,
+            target,
+            colored_level,
+            colored_dash,
+            args,
+        )
+    })
+    .init();
+}
+
+fn write(
+    mut buf: impl Write,
+    date_time: impl Display,
+    _target: impl Display,
+    level: impl Display,
+    dash: impl Display,
+    args: impl Display,
+) -> Result<(), std::io::Error> {
+    writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
+    // writeln!(
+    //     buf,
+    //     "{} {} {} {}  {}",
+    //     date_time, _target, level, dash, args
+    // )
+}

_src/utils/lossy.rs

@@ -0,0 +1,299 @@
+use ordered_float::OrderedFloat;
+
+pub trait LossyFrom<T> {
+    fn lossy_from(x: T) -> Self;
+}
+
+// ---
+// u32
+// ---
+
+impl LossyFrom<u32> for u32 {
+    #[inline(always)]
+    fn lossy_from(x: u32) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<u64> for u32 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x as u32
+    }
+}
+
+impl LossyFrom<usize> for u32 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as u32
+    }
+}
+
+impl LossyFrom<f32> for u32 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x as u32
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for u32 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        x.0 as u32
+    }
+}
+
+// ---
+// u64
+// ---
+
+impl LossyFrom<u32> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: u32) -> Self {
+        x as u64
+    }
+}
+
+impl LossyFrom<u64> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<usize> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as u64
+    }
+}
+
+impl LossyFrom<f32> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x as u64
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for u64 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        x.0 as u64
+    }
+}
+
+// ---
+// usize
+// ---
+
+impl LossyFrom<usize> for usize {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<f32> for usize {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x.round() as usize
+    }
+}
+
+// ---
+// f32
+// ---
+
+impl LossyFrom<u32> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: u32) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<u64> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<usize> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<f32> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        x.0
+    }
+}
+
+impl LossyFrom<f64> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        x as f32
+    }
+}
+
+impl LossyFrom<OrderedFloat<f64>> for f32 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f64>) -> Self {
+        x.0 as f32
+    }
+}
+
+// ---
+// OrderedFloat<f32>
+// ---
+
+impl LossyFrom<u32> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: u32) -> Self {
+        OrderedFloat(x as f32)
+    }
+}
+
+impl LossyFrom<u64> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        OrderedFloat(x as f32)
+    }
+}
+
+impl LossyFrom<usize> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        OrderedFloat(x as f32)
+    }
+}
+
+impl LossyFrom<f32> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        OrderedFloat(x)
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<f64> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        OrderedFloat(x as f32)
+    }
+}
+
+impl LossyFrom<OrderedFloat<f64>> for OrderedFloat<f32> {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f64>) -> Self {
+        OrderedFloat(x.0 as f32)
+    }
+}
+
+// ---
+// f64
+// ---
+
+impl LossyFrom<u64> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<usize> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<f32> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        x as f64
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        x.0 as f64
+    }
+}
+
+impl LossyFrom<f64> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        x
+    }
+}
+
+impl LossyFrom<OrderedFloat<f64>> for f64 {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f64>) -> Self {
+        x.0
+    }
+}
+
+// ---
+// OrderedFloat<f64>
+// ---
+
+impl LossyFrom<u64> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: u64) -> Self {
+        OrderedFloat(x as f64)
+    }
+}
+
+impl LossyFrom<usize> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: usize) -> Self {
+        OrderedFloat(x as f64)
+    }
+}
+
+impl LossyFrom<f32> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: f32) -> Self {
+        OrderedFloat(x as f64)
+    }
+}
+
+impl LossyFrom<OrderedFloat<f32>> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f32>) -> Self {
+        OrderedFloat(x.0 as f64)
+    }
+}
+
+impl LossyFrom<f64> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: f64) -> Self {
+        OrderedFloat(x)
+    }
+}
+
+impl LossyFrom<OrderedFloat<f64>> for OrderedFloat<f64> {
+    #[inline(always)]
+    fn lossy_from(x: OrderedFloat<f64>) -> Self {
+        x
+    }
+}

_src/utils/mod.rs

@@ -0,0 +1,15 @@
+mod consts;
+mod flamegraph;
+mod log;
+mod lossy;
+mod percentile;
+mod retry;
+mod time;
+
+pub use consts::*;
+pub use flamegraph::*;
+pub use log::*;
+pub use lossy::*;
+pub use percentile::*;
+pub use retry::*;
+pub use time::*;

_src/utils/percentile.rs

@@ -0,0 +1,28 @@
+use std::ops::Add;
+
+use super::LossyFrom;
+
+pub fn get_percentile<T>(sorted: &[T], percentile: f32) -> T
+where
+    T: Clone + Copy + LossyFrom<f32> + Add<Output = T>,
+    f32: LossyFrom<T>,
+{
+    let len = sorted.len();
+
+    if len < 2 {
+        T::lossy_from(f32::NAN)
+    } else {
+        let index = (len - 1) as f32 * percentile;
+
+        let fract = index.fract();
+
+        if fract != 0.0 {
+            let left = *sorted.get(index as usize).unwrap();
+            let right = *sorted.get(index.ceil() as usize).unwrap();
+
+            T::lossy_from(f32::lossy_from(left + right) / 2.0)
+        } else {
+            *sorted.get(index as usize).unwrap()
+        }
+    }
+}

_src/utils/retry.rs

@@ -0,0 +1,21 @@
+use std::{thread::sleep, time::Duration};
+
+pub fn retry<T>(
+    function: impl Fn(usize) -> color_eyre::Result<T>,
+    sleep_in_s: u64,
+    retries: usize,
+) -> color_eyre::Result<T> {
+    let mut i = 0;
+
+    loop {
+        let res = function(i);
+
+        if i == retries || res.is_ok() {
+            return res;
+        } else {
+            sleep(Duration::from_secs(sleep_in_s));
+        }
+
+        i += 1;
+    }
+}

_src/utils/time.rs

@@ -0,0 +1,18 @@
+use std::time::Instant;
+
+use log::info;
+
+use crate::structs::DisplayInstant;
+
+pub fn time<F, T>(text: &str, function: F) -> T
+where
+    F: FnOnce() -> T,
+{
+    let time = Instant::now();
+
+    let returned = function();
+
+    info!("{text} {}", time.display());
+
+    returned
+}