global: datasets compression via zstd

server/src/handler.rs

@@ -109,16 +109,16 @@ fn _file_handler(
     let type_name = route.values_type.split("::").last().unwrap();
 
     let value = match type_name {
-        "u8" => typed_value_to_response::<u8>(kind, &route.file_path, chunk)?,
-        "u16" => typed_value_to_response::<u16>(kind, &route.file_path, chunk)?,
-        "u32" => typed_value_to_response::<u32>(kind, &route.file_path, chunk)?,
-        "u64" => typed_value_to_response::<u64>(kind, &route.file_path, chunk)?,
-        "usize" => typed_value_to_response::<usize>(kind, &route.file_path, chunk)?,
-        "f32" => typed_value_to_response::<f32>(kind, &route.file_path, chunk)?,
-        "f64" => typed_value_to_response::<f64>(kind, &route.file_path, chunk)?,
-        "OHLC" => typed_value_to_response::<OHLC>(kind, &route.file_path, chunk)?,
-        "Date" => typed_value_to_response::<Date>(kind, &route.file_path, chunk)?,
-        "Height" => typed_value_to_response::<Height>(kind, &route.file_path, chunk)?,
+        "u8" => typed_value_to_response::<u8>(kind, &route, chunk)?,
+        "u16" => typed_value_to_response::<u16>(kind, &route, chunk)?,
+        "u32" => typed_value_to_response::<u32>(kind, &route, chunk)?,
+        "u64" => typed_value_to_response::<u64>(kind, &route, chunk)?,
+        "usize" => typed_value_to_response::<usize>(kind, &route, chunk)?,
+        "f32" => typed_value_to_response::<f32>(kind, &route, chunk)?,
+        "f64" => typed_value_to_response::<f64>(kind, &route, chunk)?,
+        "OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk)?,
+        "Date" => typed_value_to_response::<Date>(kind, &route, chunk)?,
+        "Height" => typed_value_to_response::<Height>(kind, &route, chunk)?,
         _ => panic!("Incompatible type: {type_name}"),
     };
 

server/src/imports.rs

@@ -1,27 +1,25 @@
-use std::fmt::Debug;
+use std::{fmt::Debug, path::Path};
 
 use bincode::Decode;
-use parser::{Date, Serialization, SerializedBTreeMap, SerializedVec};
+use parser::{Date, SerializedBTreeMap, SerializedVec};
 use serde::{de::DeserializeOwned, Serialize};
 
-pub fn import_map<T>(relative_path: &str) -> color_eyre::Result<SerializedBTreeMap<Date, T>>
+use crate::routes::Route;
+
+pub fn import_map<T>(route: &Route) -> color_eyre::Result
 where
     T: Serialize + Debug + DeserializeOwned + Decode,
 {
-    Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
 }
 
-pub fn import_vec<T>(relative_path: &str) -> color_eyre::Result<SerializedVec<T>>
+pub fn import_vec<T>(route: &Route) -> color_eyre::Result
 where
     T: Serialize + Debug + DeserializeOwned + Decode,
 {
-    Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
 }
 
-pub fn import_value<T>(relative_path: &str) -> color_eyre::Result<T>
+pub fn import_value<T>(route: &Route) -> color_eyre::Result<T>
 where
     T: Serialize + Debug + DeserializeOwned + Decode,
 {
-    Serialization::from_extension(relative_path.split('.').last().unwrap())
-        .import::<T>(relative_path)
 }

server/src/main.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router};
-use parser::log;
+use parser::{log, reset_logs};
 use reqwest::header::HOST;
 use response::generic_to_reponse;
 use routes::Routes;
@@ -12,7 +12,6 @@ use tower_http::compression::CompressionLayer;
 mod chunk;
 mod handler;
 mod headers;
-mod imports;
 mod kind;
 mod paths;
 mod response;
@@ -36,6 +35,8 @@ pub struct AppState {
 async fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
+    reset_logs();
+
     let routes = Routes::build();
 
     routes.generate_dts_file();

server/src/response.rs

@@ -1,15 +1,16 @@
-use std::fmt::Debug;
+use std::{fmt::Debug, path::Path};
 
 use axum::response::{IntoResponse, Json, Response};
 use bincode::Decode;
+use parser::{Date, SerializedBTreeMap, SerializedVec};
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 
 use crate::{
     chunk::Chunk,
     headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers},
-    imports::{import_map, import_value, import_vec},
     kind::Kind,
+    routes::Route,
 };
 
 #[derive(Serialize)]
@@ -24,16 +25,30 @@ where
 
 pub fn typed_value_to_response<T>(
     kind: Kind,
-    relative_path: &str,
+    route: &Route,
     chunk: Option<Chunk>,
 ) -> color_eyre::Result<Response>
 where
     T: Serialize + Debug + DeserializeOwned + Decode,
 {
     Ok(match kind {
-        Kind::Date => dataset_to_response(import_map::<T>(relative_path)?, chunk.unwrap()),
-        Kind::Height => dataset_to_response(import_vec::<T>(relative_path)?, chunk.unwrap()),
-        Kind::Last => value_to_response(import_value::<T>(relative_path)?),
+        Kind::Date => dataset_to_response(
+            route
+                .serialization
+                .import::<SerializedBTreeMap<Date, T>>(Path::new(&route.file_path))?,
+            chunk.unwrap(),
+        ),
+        Kind::Height => dataset_to_response(
+            route
+                .serialization
+                .import::<SerializedVec<T>>(Path::new(&route.file_path))?,
+            chunk.unwrap(),
+        ),
+        Kind::Last => value_to_response(
+            route
+                .serialization
+                .import::<T>(Path::new(&route.file_path))?,
+        ),
     })
 }
 

server/src/routes.rs

@@ -1,6 +1,7 @@
 use std::{
     collections::{BTreeMap, HashMap},
     fs,
+    path::Path,
 };
 
 use derive_deref::{Deref, DerefMut};
@@ -26,31 +27,22 @@ const APP_TYPES_PATH: &str = "../app/src/types";
 impl Routes {
     pub fn build() -> Self {
         let path_to_type: BTreeMap<String, String> =
-            Json::import(&format!("{INPUTS_PATH}/disk_path_to_type.json")).unwrap();
+            Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap();
 
         let mut routes = Routes::default();
 
         path_to_type.into_iter().for_each(|(key, value)| {
             let mut split_key = key.split('/').collect_vec();
-
-            let mut split_last = split_key.pop().unwrap().split('.').rev().collect_vec();
-
-            let last = split_last.pop().unwrap().to_owned();
+            let last = split_key.pop().unwrap().to_owned();
 
             let mut skip = 2;
 
-            let serialization = split_last.pop().map_or_else(
-                || {
-                    if *split_key.get(1).unwrap() == "price" {
-                        skip = 1;
+            let mut serialization = Serialization::Binary;
 
-                        Serialization::Json
-                    } else {
-                        Serialization::Binary
-                    }
-                },
-                Serialization::from_extension,
-            );
+            if *split_key.get(1).unwrap() == "price" {
+                skip = 1;
+                serialization = Serialization::Json;
+            }
 
             let split_key = split_key.iter().skip(skip).collect_vec();