server: add support for .json .csv and ?all=true

server/src/api/handlers/dataset.rs

@@ -20,11 +20,15 @@ use crate::{
     AppState,
 };
 
-use super::response::{typed_value_to_response, value_to_response};
+use super::{
+    extension::Extension,
+    response::{typed_value_to_response, value_to_response},
+};
 
 #[derive(Deserialize)]
 pub struct Params {
     chunk: Option<usize>,
+    all: Option<bool>,
 }
 
 pub async fn dataset_handler(
@@ -46,43 +50,44 @@ pub async fn dataset_handler(
     }
 }
 
+const DATE_PREFIX: &str = "date-to-";
+const HEIGHT_PREFIX: &str = "height-to-";
+
 fn _dataset_handler(
     headers: HeaderMap,
     Path(path): Path<String>,
     query: Query<Params>,
     AppState { routes }: AppState,
 ) -> color_eyre::Result<Response> {
+    if query.chunk.is_some() && query.all.is_some() {
+        return Err(eyre!("chunk and all are exclusive"));
+    }
+
     log(&format!(
-        "{}{}",
+        "{}{}{}",
         path,
         query.chunk.map_or("".to_string(), |chunk| format!(
             "{}{chunk}",
             "?chunk=".bright_black()
+        )),
+        query.all.map_or("".to_string(), |all| format!(
+            "{}{all}",
+            "?all=".bright_black()
         ))
     ));
 
-    let date_prefix = "date-to-";
-    let height_prefix = "height-to-";
-
-    let (kind, route) = if path.starts_with(date_prefix) {
-        (
-            Kind::Date,
-            routes.date.get(&replace_dash_by_underscore(
-                path.strip_prefix(date_prefix).unwrap(),
-            )),
-        )
-    } else if path.starts_with(height_prefix) {
-        (
-            Kind::Height,
-            routes.height.get(&replace_dash_by_underscore(
-                path.strip_prefix(height_prefix).unwrap(),
-            )),
-        )
+    let (kind, id, route) = if path.starts_with(DATE_PREFIX) {
+        let id = convert_path_to_id(path.strip_prefix(DATE_PREFIX).unwrap());
+        let route = routes.date.get(&id);
+        (Kind::Date, id, route)
+    } else if path.starts_with(HEIGHT_PREFIX) {
+        let id = convert_path_to_id(path.strip_prefix(HEIGHT_PREFIX).unwrap());
+        let route = routes.height.get(&id);
+        (Kind::Height, id, route)
     } else {
-        (
-            Kind::Last,
-            routes.last.get(&replace_dash_by_underscore(&path)),
-        )
+        let id = convert_path_to_id(&path);
+        let route = routes.last.get(&id);
+        (Kind::Last, id, route)
     };
 
     if route.is_none() {
@@ -93,25 +98,28 @@ fn _dataset_handler(
 
     let mut chunk = None;
 
-    match kind {
-        Kind::Date => {
-            let datasets = DateMap::<usize>::_read_dir(&route.file_path, &route.serialization);
+    if query.all.map_or(true, |b| !b) {
+        match kind {
+            Kind::Date => {
+                let datasets = DateMap::<usize>::_read_dir(&route.file_path, &route.serialization);
 
-            process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
-        }
-        Kind::Height => {
-            let datasets = HeightMap::<usize>::_read_dir(&route.file_path, &route.serialization);
+                process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
+            }
+            Kind::Height => {
+                let datasets =
+                    HeightMap::<usize>::_read_dir(&route.file_path, &route.serialization);
 
-            process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
-        }
-        Kind::Last => {
-            if !route.values_type.ends_with("Value") {
-                route.file_path.set_extension(COMPRESSED_BIN_EXTENSION);
-            } else {
-                route.file_path.set_extension(JSON_EXTENSION);
+                process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
+            }
+            Kind::Last => {
+                if !route.values_type.ends_with("Value") {
+                    route.file_path.set_extension(COMPRESSED_BIN_EXTENSION);
+                } else {
+                    route.file_path.set_extension(JSON_EXTENSION);
+                }
             }
         }
-    };
+    }
 
     let (date, response) = headers.check_if_modified_since(&route.file_path).unwrap();
 
@@ -121,18 +129,22 @@ fn _dataset_handler(
 
     let type_name = route.values_type.split("::").last().unwrap();
 
+    let extension = Extension::from(&std::path::PathBuf::from(&path));
+
     let mut response = match type_name {
-        "u8" => typed_value_to_response::<u8>(kind, &route, chunk)?,
-        "u16" => typed_value_to_response::<u16>(kind, &route, chunk)?,
-        "u32" => typed_value_to_response::<u32>(kind, &route, chunk)?,
-        "u64" => typed_value_to_response::<u64>(kind, &route, chunk)?,
-        "usize" => typed_value_to_response::<usize>(kind, &route, chunk)?,
-        "f32" => typed_value_to_response::<f32>(kind, &route, chunk)?,
-        "f64" => typed_value_to_response::<f64>(kind, &route, chunk)?,
-        "OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk)?,
-        "Date" => typed_value_to_response::<Date>(kind, &route, chunk)?,
-        "Height" => typed_value_to_response::<Height>(kind, &route, chunk)?,
-        "Value" => value_to_response::<serde_json::Value>(Json::import(&route.file_path)?),
+        "u8" => typed_value_to_response::<u8>(kind, &route, chunk, id, extension)?,
+        "u16" => typed_value_to_response::<u16>(kind, &route, chunk, id, extension)?,
+        "u32" => typed_value_to_response::<u32>(kind, &route, chunk, id, extension)?,
+        "u64" => typed_value_to_response::<u64>(kind, &route, chunk, id, extension)?,
+        "usize" => typed_value_to_response::<usize>(kind, &route, chunk, id, extension)?,
+        "f32" => typed_value_to_response::<f32>(kind, &route, chunk, id, extension)?,
+        "f64" => typed_value_to_response::<f64>(kind, &route, chunk, id, extension)?,
+        "OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk, id, extension)?,
+        "Date" => typed_value_to_response::<Date>(kind, &route, chunk, id, extension)?,
+        "Height" => typed_value_to_response::<Height>(kind, &route, chunk, id, extension)?,
+        "Value" => {
+            value_to_response::<serde_json::Value>(Json::import(&route.file_path)?, extension)
+        }
         _ => panic!("Incompatible type: {type_name}"),
     };
 
@@ -142,8 +154,8 @@ fn _dataset_handler(
     Ok(response)
 }
 
-fn replace_dash_by_underscore(s: &str) -> String {
-    s.replace('-', "_")
+fn convert_path_to_id(s: &str) -> String {
+    Extension::remove_extension(s).replace('-', "_")
 }
 
 fn process_datasets<ChunkId>(
@@ -174,7 +186,6 @@ where
     }
 
     let path = path.unwrap();
-
     route.file_path = path.clone();
 
     let offset = match kind {

server/src/api/handlers/extension.rs

@@ -0,0 +1,41 @@
+use std::path::Path;
+
+#[derive(PartialEq, Eq)]
+pub enum Extension {
+    CSV,
+    JSON,
+}
+
+impl Extension {
+    pub fn from(path: &Path) -> Option<Self> {
+        if let Some(extension) = path.extension() {
+            let extension = extension.to_str().unwrap();
+
+            if extension == Self::CSV.to_str() {
+                Some(Self::CSV)
+            } else if extension == Self::JSON.to_str() {
+                Some(Self::JSON)
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    }
+
+    pub fn to_str(&self) -> &str {
+        match self {
+            Extension::CSV => "csv",
+            Extension::JSON => "json",
+        }
+    }
+
+    pub fn to_dot_str(&self) -> String {
+        format!(".{}", self.to_str())
+    }
+
+    pub fn remove_extension(s: &str) -> String {
+        s.replace(&Self::CSV.to_dot_str(), "")
+            .replace(&Self::JSON.to_dot_str(), "")
+    }
+}

server/src/api/handlers/fallback.rs

@@ -3,14 +3,17 @@ use reqwest::header::HOST;
 
 use crate::AppState;
 
-use super::response::generic_to_reponse;
+use super::response::{generic_to_reponse, update_reponse_headers};
 
 pub async fn fallback(headers: HeaderMap, State(app_state): State<AppState>) -> Response {
-    generic_to_reponse(
-        app_state
-            .routes
-            .to_full_paths(headers[HOST].to_str().unwrap().to_string()),
-        None,
+    update_reponse_headers(
+        generic_to_reponse(
+            app_state
+                .routes
+                .to_full_paths(headers[HOST].to_str().unwrap().to_string()),
+            None,
+        ),
         60,
+        None,
     )
 }

server/src/api/handlers/mod.rs

@@ -1,5 +1,7 @@
 mod dataset;
+mod extension;
 mod fallback;
+
 mod response;
 
 pub use dataset::*;

server/src/api/handlers/response.rs

@@ -1,8 +1,8 @@
-use std::{fmt::Debug, path::Path};
+use std::fmt::Debug;
 
 use axum::response::{IntoResponse, Json, Response};
 use bincode::Decode;
-use parser::{Date, SerializedBTreeMap, SerializedVec};
+use parser::{Date, MapValue, SerializedBTreeMap, SerializedVec};
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 
@@ -11,6 +11,8 @@ use crate::{
     header_map::HeaderMapUtils,
 };
 
+use super::extension::Extension;
+
 #[derive(Serialize)]
 struct WrappedDataset<'a, T>
 where
@@ -25,70 +27,119 @@ pub fn typed_value_to_response<T>(
     kind: Kind,
     route: &Route,
     chunk: Option<Chunk>,
+    id: String,
+    extension: Option<Extension>,
 ) -> color_eyre::Result<Response>
 where
-    T: Serialize + Debug + DeserializeOwned + Decode,
+    T: Serialize + Debug + DeserializeOwned + Decode + MapValue,
 {
     Ok(match kind {
-        Kind::Date => dataset_to_response(
-            route
-                .serialization
-                .import::<SerializedBTreeMap<Date, T>>(Path::new(&route.file_path))?,
-            chunk.unwrap(),
-        ),
-        Kind::Height => dataset_to_response(
-            route
-                .serialization
-                .import::<SerializedVec<T>>(Path::new(&route.file_path))?,
-            chunk.unwrap(),
-        ),
+        Kind::Date => {
+            let dataset = if chunk.is_some() {
+                route
+                    .serialization
+                    .import::<SerializedBTreeMap<Date, T>>(&route.file_path)?
+            } else {
+                SerializedBTreeMap::<Date, T>::import_all(&route.file_path, &route.serialization)
+            };
+
+            if extension == Some(Extension::CSV) {
+                let mut csv = format!("date,{}\n", id);
+
+                dataset.map.iter().for_each(|(k, v)| {
+                    csv += &format!("{},{:?}\n", k, v);
+                });
+
+                string_to_response(csv, extension)
+            } else {
+                dataset_to_response(dataset, chunk, extension)
+            }
+        }
+        Kind::Height => {
+            let dataset = if chunk.is_some() {
+                route
+                    .serialization
+                    .import::<SerializedVec<T>>(&route.file_path)?
+            } else {
+                SerializedVec::<T>::import_all(&route.file_path, &route.serialization)
+            };
+
+            if extension == Some(Extension::CSV) {
+                let mut csv = format!("height,{}\n", id);
+
+                let starting_height = chunk.map_or(0, |chunk| chunk.id);
+
+                dataset.map.iter().enumerate().for_each(|(k, v)| {
+                    csv += &format!("{},{:?}\n", starting_height + k, v);
+                });
+
+                string_to_response(csv, extension)
+            } else {
+                dataset_to_response(dataset, chunk, extension)
+            }
+        }
         Kind::Last => value_to_response(
-            route
-                .serialization
-                .import::<T>(Path::new(&route.file_path))?,
+            route.serialization.import::<T>(&route.file_path)?,
+            extension,
         ),
     })
 }
 
-pub fn value_to_response<T>(value: T) -> Response
-where
-    T: Serialize,
-{
-    generic_to_reponse(value, None, 1)
+pub fn string_to_response(s: String, extension: Option<Extension>) -> Response {
+    update_reponse_headers(s.into_response(), 5, extension)
 }
 
-fn dataset_to_response<T>(dataset: T, chunk: Chunk) -> Response
+pub fn value_to_response<T>(value: T, extension: Option<Extension>) -> Response
 where
     T: Serialize,
 {
-    generic_to_reponse(dataset, Some(chunk), 5)
+    update_reponse_headers(generic_to_reponse(value, None), 1, extension)
 }
 
-pub fn generic_to_reponse<T>(generic: T, chunk: Option<Chunk>, cache_time: u64) -> Response
+fn dataset_to_response<T>(
+    dataset: T,
+    chunk: Option<Chunk>,
+    extension: Option<Extension>,
+) -> Response
 where
     T: Serialize,
 {
-    let mut response = {
-        if let Some(chunk) = chunk {
-            Json(WrappedDataset {
-                source: "https://kibo.money",
-                chunk,
-                dataset: generic,
-            })
-            .into_response()
-        } else {
-            Json(generic).into_response()
-        }
-    };
+    update_reponse_headers(generic_to_reponse(dataset, chunk), 5, extension)
+}
 
+pub fn generic_to_reponse<T>(generic: T, chunk: Option<Chunk>) -> Response
+where
+    T: Serialize,
+{
+    if let Some(chunk) = chunk {
+        Json(WrappedDataset {
+            source: "https://kibo.money",
+            chunk,
+            dataset: generic,
+        })
+        .into_response()
+    } else {
+        Json(generic).into_response()
+    }
+}
+
+pub fn update_reponse_headers(
+    mut response: Response,
+    cache_time: u64,
+    extension: Option<Extension>,
+) -> Response {
     let headers = response.headers_mut();
 
     let max_age = cache_time;
     let stale_while_revalidate = 2 * max_age;
 
     headers.insert_cors();
-    headers.insert_content_type_application_json();
     headers.insert_cache_control_revalidate(max_age, stale_while_revalidate);
 
+    match extension {
+        Some(Extension::CSV) => headers.insert_content_type_text_csv(),
+        _ => headers.insert_content_type_application_json(),
+    }
+
     response
 }

server/src/api/structs/routes.rs

@@ -24,7 +24,7 @@ pub struct Route {
 pub struct Routes(pub Grouped<HashMap<String, Route>>);
 
 const INPUTS_PATH: &str = "./in";
-const WEBSITE_TYPES_PATH: &str = "../website/types";
+const WEBSITE_TYPES_PATH: &str = "../website/scripts/types";
 
 impl Routes {
     pub fn build() -> Self {

server/src/header_map.rs

@@ -34,6 +34,8 @@ pub trait HeaderMapUtils {
     fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
     fn insert_last_modified(&mut self, date: DateTime<Utc>);
 
+    fn insert_content_disposition_attachment(&mut self);
+
     fn insert_content_type(&mut self, path: &Path);
     fn insert_content_type_image_icon(&mut self);
     fn insert_content_type_image_jpeg(&mut self);
@@ -43,6 +45,7 @@ pub trait HeaderMapUtils {
     fn insert_content_type_application_manifest_json(&mut self);
     fn insert_content_type_application_pdf(&mut self);
     fn insert_content_type_text_css(&mut self);
+    fn insert_content_type_text_csv(&mut self);
     fn insert_content_type_text_html(&mut self);
     fn insert_content_type_text_plain(&mut self);
     fn insert_content_type_font_woff2(&mut self);
@@ -181,6 +184,10 @@ impl HeaderMapUtils for HeaderMap {
         );
     }
 
+    fn insert_content_disposition_attachment(&mut self) {
+        self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
+    }
+
     fn insert_content_type_application_json(&mut self) {
         self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
     }
@@ -200,6 +207,10 @@ impl HeaderMapUtils for HeaderMap {
         self.insert(header::CONTENT_TYPE, "text/css".parse().unwrap());
     }
 
+    fn insert_content_type_text_csv(&mut self) {
+        self.insert(header::CONTENT_TYPE, "text/csv".parse().unwrap());
+    }
+
     fn insert_content_type_text_html(&mut self) {
         self.insert(header::CONTENT_TYPE, "text/html".parse().unwrap());
     }