server: rework api side

src/structs/any_map.rs

@@ -1,35 +1,23 @@
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 use serde_json::Value;
 
-use super::{MapKind, MapPath};
+use crate::io::Serialization;
+
+use super::{Config, MapKind, MapPath};
 
 pub trait AnyMap {
     fn path(&self) -> &Path;
+    fn path_parent(&self) -> &Path;
     fn path_last(&self) -> &Option<MapPath>;
-
     fn last_value(&self) -> Option<Value>;
-
-    fn t_name(&self) -> &str;
-
-    fn get_paths_to_type(&self) -> Vec<(PathBuf, String)> {
-        let t_name = self.t_name().to_string();
-
-        if let Some(path_last) = self.path_last() {
-            vec![
-                (self.path().to_owned(), t_name.clone()),
-                (path_last.unwrap().to_owned(), t_name),
-            ]
-        } else {
-            vec![(self.path().to_owned(), t_name)]
-        }
-    }
-
+    fn serialization(&self) -> Serialization;
+    fn type_name(&self) -> &str;
+    fn key_name(&self) -> &str;
     fn pre_export(&mut self);
     fn export(&self) -> color_eyre::Result<()>;
     fn post_export(&mut self);
-
     fn delete_files(&self);
-
     fn kind(&self) -> MapKind;
+    fn id(&self, config: &Config) -> String;
 }

src/structs/config.rs

@@ -10,6 +10,8 @@ use color_eyre::eyre::eyre;
 use log::info;
 use serde::{Deserialize, Serialize};
 
+use crate::io::JSON_EXTENSION;
+
 use super::MapPath;
 
 #[derive(Parser, Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
@@ -74,6 +76,8 @@ pub struct Config {
 }
 
 impl Config {
+    pub const DATASET_DIR_NAME: &str = "datasets";
+
     pub fn import() -> color_eyre::Result<Self> {
         let path = Self::path_dot_kibo();
         let _ = fs::create_dir_all(&path);
@@ -285,7 +289,7 @@ impl Config {
     }
 
     pub fn path_datasets_last_values(&self) -> MapPath {
-        self.path_datasets().join("last.json")
+        self.path_datasets().join(&format!("last.{JSON_EXTENSION}"))
     }
 
     pub fn path_price(&self) -> MapPath {
@@ -293,7 +297,7 @@ impl Config {
     }
 
     pub fn path_databases(&self) -> PathBuf {
-        self.path_kibodir().join("databases")
+        self.path_kibodir().join(Self::DATASET_DIR_NAME)
     }
 
     pub fn path_states(&self) -> PathBuf {

src/structs/date_map.rs

@@ -1,10 +1,10 @@
 use std::iter::Sum;
 
 use super::{
-    AnyMap, Date, DateMapChunkId, GenericMap, Height, HeightMap, MapValue, SerializedBTreeMap,
+    AnyMap, Date, DateMapChunkId, GenericMap, Height, HeightMap, MapValue, SerializedDateMap,
 };
 
-pub type DateMap<Value> = GenericMap<Date, Value, DateMapChunkId, SerializedBTreeMap<Date, Value>>;
+pub type DateMap<Value> = GenericMap<Date, Value, DateMapChunkId, SerializedDateMap<Value>>;
 
 impl<Value> DateMap<Value>
 where

src/structs/date_map_chunk_id.rs

@@ -15,7 +15,7 @@ impl DateMapChunkId {
 }
 
 impl MapChunkId for DateMapChunkId {
-    fn to_name(&self) -> String {
+    fn to_string(&self) -> String {
         self.0.to_string()
     }
 
@@ -39,4 +39,12 @@ impl MapChunkId for DateMapChunkId {
     fn from_usize(id: usize) -> Self {
         Self(id as i32)
     }
+
+    fn next(&self) -> Option<Self> {
+        self.0.checked_add(1).map(Self)
+    }
+
+    fn previous(&self) -> Option<Self> {
+        self.0.checked_sub(1).map(Self)
+    }
 }

src/structs/generic_map.rs

@@ -1,6 +1,6 @@
 use std::{
     collections::{BTreeMap, VecDeque},
-    fmt::Debug,
+    fmt::{Debug, Display},
     fs,
     iter::Sum,
     mem,
@@ -19,7 +19,7 @@ use crate::{
     utils::{get_percentile, LossyFrom},
 };
 
-use super::{AnyMap, MapPath, MapValue};
+use super::{AnyMap, Config, MapPath, MapValue};
 
 #[derive(Debug, Clone, Copy, Allocative, PartialEq, Eq)]
 pub enum MapKind {
@@ -29,7 +29,7 @@ pub enum MapKind {
 
 pub trait MapKey<ChunkId>
 where
-    Self: Sized + PartialOrd + Ord + Clone + Copy + Debug,
+    Self: Sized + PartialOrd + Ord + Clone + Copy + Debug + Display,
     ChunkId: MapChunkId,
 {
     fn to_chunk_id(&self) -> ChunkId;
@@ -61,16 +61,21 @@ where
     fn get(&self, serialized_key: &Key) -> Option<&Value>;
     fn last(&self) -> Option<&Value>;
     fn extend(&mut self, map: BTreeMap<Key, Value>);
+    fn import_all(path: &Path, serialization: &Serialization) -> Self;
+    fn to_csv(self, id: &str) -> String;
+    fn map(&self) -> &impl Serialize;
 }
 
 pub trait MapChunkId
 where
     Self: Ord + Debug + Copy + Clone,
 {
-    fn to_name(&self) -> String;
+    fn to_string(&self) -> String;
     fn from_path(path: &Path) -> color_eyre::Result<Self>;
     fn to_usize(self) -> usize;
     fn from_usize(id: usize) -> Self;
+    fn previous(&self) -> Option<Self>;
+    fn next(&self) -> Option<Self>;
 }
 
 #[derive(Debug, Allocative)]
@@ -79,6 +84,7 @@ pub struct GenericMap<Key, Value, ChunkId, Serialized> {
     kind: MapKind,
 
     path_all: MapPath,
+    path_parent: MapPath,
     path_last: Option<MapPath>,
 
     chunks_in_memory: usize,
@@ -150,6 +156,7 @@ where
             kind,
 
             path_all,
+            path_parent: path.to_owned(),
             path_last,
 
             chunks_in_memory,
@@ -177,15 +184,7 @@ where
                 }
             });
 
-        s.initial_last_key = s
-            .imported
-            .iter()
-            .last()
-            .and_then(|(last_chunk_id, serialized)| serialized.get_last_key(last_chunk_id));
-
-        s.initial_first_unsafe_key = s
-            .initial_last_key
-            .and_then(|last_key| last_key.to_first_unsafe());
+        s.set_initial_keys();
 
         // if s.initial_first_unsafe_key.is_none() {
         //     log(&format!("Missing dataset: {path:?}/{}", Key::map_name()));
@@ -194,6 +193,18 @@ where
         s
     }
 
+    fn set_initial_keys(&mut self) {
+        self.initial_last_key = self
+            .imported
+            .iter()
+            .last()
+            .and_then(|(last_chunk_id, serialized)| serialized.get_last_key(last_chunk_id));
+
+        self.initial_first_unsafe_key = self
+            .initial_last_key
+            .and_then(|last_key| last_key.to_first_unsafe());
+    }
+
     fn read_dir(&self) -> BTreeMap<ChunkId, PathBuf> {
         Self::_read_dir(&self.path_all, &self.serialization)
     }
@@ -308,10 +319,26 @@ where
     Key: MapKey<ChunkId>,
     Serialized: MapSerialized<Key, Value, ChunkId>,
 {
+    fn id(&self, config: &Config) -> String {
+        let path_to_string = |p: &Path| p.to_str().unwrap().to_owned();
+        path_to_string(self.path_parent())
+            .replace(&path_to_string(&config.path_kibodir()), "")
+            .replace(&format!("/{}/", Config::DATASET_DIR_NAME), "")
+            .replace("/", "-")
+    }
+
+    fn serialization(&self) -> Serialization {
+        self.serialization
+    }
+
     fn path(&self) -> &Path {
         &self.path_all
     }
 
+    fn path_parent(&self) -> &Path {
+        &self.path_parent
+    }
+
     fn path_last(&self) -> &Option<MapPath> {
         &self.path_last
     }
@@ -323,10 +350,14 @@ where
             .and_then(|v| serde_json::to_value(v).ok())
     }
 
-    fn t_name(&self) -> &str {
+    fn type_name(&self) -> &str {
         std::any::type_name::<Value>()
     }
 
+    fn key_name(&self) -> &str {
+        Key::map_name()
+    }
+
     fn pre_export(&mut self) {
         self.to_insert.iter_mut().for_each(|(chunk_id, map)| {
             if let Some((key, _)) = map.first_key_value() {
@@ -351,6 +382,8 @@ where
                 .or_insert(Serialized::new(self.version))
                 .extend(mem::take(map));
         });
+
+        self.set_initial_keys();
     }
 
     fn export(&self) -> color_eyre::Result<()> {
@@ -367,7 +400,7 @@ where
                     panic!();
                 });
 
-                let path = self.path_all.join(&chunk_id.to_name());
+                let path = self.path_all.join(&chunk_id.to_string());
 
                 self.serialization.export(&path, serialized)?;
 

src/structs/height_map_chunk_id.rs

@@ -19,7 +19,7 @@ impl HeightMapChunkId {
 }
 
 impl MapChunkId for HeightMapChunkId {
-    fn to_name(&self) -> String {
+    fn to_string(&self) -> String {
         let start = ***self;
         let end = start + HEIGHT_MAP_CHUNK_SIZE;
 
@@ -46,4 +46,14 @@ impl MapChunkId for HeightMapChunkId {
     fn from_usize(id: usize) -> Self {
         Self(Height::new(id as u32))
     }
+
+    fn next(&self) -> Option<Self> {
+        self.checked_add(HEIGHT_MAP_CHUNK_SIZE)
+            .map(|h| Self(Height::new(h)))
+    }
+
+    fn previous(&self) -> Option<Self> {
+        self.checked_sub(HEIGHT_MAP_CHUNK_SIZE)
+            .map(|h| Self(Height::new(h)))
+    }
 }

src/structs/map_value.rs

@@ -1,4 +1,4 @@
-use std::fmt::Debug;
+use std::fmt::{Debug, Display};
 
 use allocative::Allocative;
 use bincode::{Decode, Encode};
@@ -18,6 +18,7 @@ pub trait MapValue:
     + Sync
     + Send
     + Allocative
+    + Display
 {
 }
 

src/structs/mod.rs

@@ -72,7 +72,6 @@ pub use map_value::*;
 pub use ohlc::*;
 pub use partial_txout_data::*;
 pub use price::*;
-pub use rpc::*;
 pub use sent_data::*;
 pub use serialized_btreemap::*;
 pub use serialized_vec::*;

src/structs/ohlc.rs

@@ -1,3 +1,5 @@
+use std::fmt::{self};
+
 use allocative::Allocative;
 use bincode::{Decode, Encode};
 use serde::{Deserialize, Serialize};
@@ -10,3 +12,13 @@ pub struct OHLC {
     pub low: f32,
     pub close: f32,
 }
+
+impl fmt::Display for OHLC {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{{ open: {}, high: {}, low: {}, close: {} }}",
+            self.open, self.high, self.low, self.close
+        )
+    }
+}

src/structs/serialized_btreemap.rs

@@ -6,7 +6,9 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
 use crate::io::Serialization;
 
-use super::{DateMap, MapChunkId, MapKey, MapSerialized, MapValue};
+use super::{Date, DateMap, MapChunkId, MapKey, MapSerialized, MapValue};
+
+pub type SerializedDateMap<T> = SerializedBTreeMap<Date, T>;
 
 #[derive(Debug, Default, Serialize, Deserialize, Encode, Decode, Allocative)]
 pub struct SerializedBTreeMap<Key, Value>
@@ -17,41 +19,11 @@ where
     pub map: BTreeMap<Key, Value>,
 }
 
-impl<Key, Value> SerializedBTreeMap<Key, Value>
-where
-    Key: Ord,
-{
-    pub fn import_all<ChunkId>(path: &Path, serialization: &Serialization) -> Self
-    where
-        Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
-        ChunkId: MapChunkId,
-        Key: MapKey<ChunkId>,
-        Value: MapValue,
-    {
-        let mut s = None;
-
-        DateMap::<usize>::_read_dir(path, serialization)
-            .iter()
-            .for_each(|(_, path)| {
-                let map = serialization.import::<Self>(path).unwrap();
-
-                if s.is_none() {
-                    s.replace(map);
-                } else {
-                    #[allow(clippy::unnecessary_unwrap)]
-                    s.as_mut().unwrap().map.extend(map.map);
-                }
-            });
-
-        s.unwrap()
-    }
-}
-
 impl<Key, Value, ChunkId> MapSerialized<Key, Value, ChunkId> for SerializedBTreeMap<Key, Value>
 where
     Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
     ChunkId: MapChunkId,
-    Key: MapKey<ChunkId>,
+    Key: MapKey<ChunkId> + Serialize,
     Value: MapValue,
 {
     fn new(version: u32) -> Self {
@@ -80,4 +52,35 @@ where
     fn extend(&mut self, map: BTreeMap<Key, Value>) {
         self.map.extend(map)
     }
+
+    fn import_all(path: &Path, serialization: &Serialization) -> Self {
+        let mut s = None;
+
+        DateMap::<usize>::_read_dir(path, serialization)
+            .iter()
+            .for_each(|(_, path)| {
+                let map = serialization.import::<Self>(path).unwrap();
+
+                if s.is_none() {
+                    s.replace(map);
+                } else {
+                    #[allow(clippy::unnecessary_unwrap)]
+                    s.as_mut().unwrap().map.extend(map.map);
+                }
+            });
+
+        s.unwrap()
+    }
+
+    fn to_csv(self, id: &str) -> String {
+        let mut csv = format!("{},{}\n", Key::map_name(), id);
+        self.map.iter().for_each(|(k, v)| {
+            csv += &format!("{},{}\n", k, v);
+        });
+        csv
+    }
+
+    fn map(&self) -> &impl Serialize {
+        &self.map
+    }
 }

src/structs/serialized_vec.rs

@@ -14,31 +14,6 @@ pub struct SerializedVec<Value> {
     pub map: Vec<Value>,
 }
 
-impl<Value> SerializedVec<Value> {
-    pub fn import_all(path: &Path, serialization: &Serialization) -> Self
-    where
-        Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
-        Value: MapValue,
-    {
-        let mut s = None;
-
-        HeightMap::<usize>::_read_dir(path, serialization)
-            .iter()
-            .for_each(|(_, path)| {
-                let mut map = serialization.import::<Self>(path).unwrap();
-
-                if s.is_none() {
-                    s.replace(map);
-                } else {
-                    #[allow(clippy::unnecessary_unwrap)]
-                    s.as_mut().unwrap().map.append(&mut map.map);
-                }
-            });
-
-        s.unwrap()
-    }
-}
-
 impl<Key, Value, ChunkId> MapSerialized<Key, Value, ChunkId> for SerializedVec<Value>
 where
     Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
@@ -87,4 +62,39 @@ where
             }
         });
     }
+
+    fn import_all(path: &Path, serialization: &Serialization) -> Self
+    where
+        Self: Debug + Serialize + DeserializeOwned + Encode + Decode,
+        Value: MapValue,
+    {
+        let mut s = None;
+
+        HeightMap::<usize>::_read_dir(path, serialization)
+            .iter()
+            .for_each(|(_, path)| {
+                let mut map = serialization.import::<Self>(path).unwrap();
+
+                if s.is_none() {
+                    s.replace(map);
+                } else {
+                    #[allow(clippy::unnecessary_unwrap)]
+                    s.as_mut().unwrap().map.append(&mut map.map);
+                }
+            });
+
+        s.unwrap()
+    }
+
+    fn to_csv(self, id: &str) -> String {
+        let mut csv = format!("{},{}\n", Key::map_name(), id);
+        self.map.iter().enumerate().for_each(|(k, v)| {
+            csv += &format!("{:?},{:?}\n", k, v);
+        });
+        csv
+    }
+
+    fn map(&self) -> &impl Serialize {
+        &self.map
+    }
 }

src/structs/timestamp.rs

@@ -1,4 +1,4 @@
-use std::ops::Sub;
+use std::{fmt, ops::Sub};
 
 use allocative::Allocative;
 use bincode::{Decode, Encode};
@@ -81,3 +81,9 @@ impl Sub for Timestamp {
         Self::wrap(self.0 - rhs.0)
     }
 }
+
+impl fmt::Display for Timestamp {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", **self)
+    }
+}