global: datasets compression via zstd

parser/src/structs/block_data.rs

@@ -1,9 +1,10 @@
 use allocative::Allocative;
 use bincode::{Decode, Encode};
+use serde::{Deserialize, Serialize};
 
 use super::{Amount, Height, Price};
 
-#[derive(Debug, Encode, Decode, Allocative)]
+#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
 pub struct BlockData {
     pub height: Height,
     pub price: Price,

parser/src/structs/counter.rs

@@ -1,8 +1,11 @@
 use allocative::Allocative;
 use bincode::{Decode, Encode};
 use derive_deref::{Deref, DerefMut};
+use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Allocative)]
+#[derive(
+    Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Serialize, Deserialize, Allocative,
+)]
 pub struct Counter(u32);
 
 impl Counter {

parser/src/structs/date_data.rs

@@ -1,9 +1,10 @@
 use allocative::Allocative;
 use bincode::{Decode, Encode};
+use serde::{Deserialize, Serialize};
 
 use super::{BlockData, BlockPath, Date};
 
-#[derive(Debug, Encode, Decode, Allocative)]
+#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
 pub struct DateData {
     pub date: Date,
     pub blocks: Vec<BlockData>,

parser/src/structs/date_map_chunk_id.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use allocative::Allocative;
 use chrono::Datelike;
 
@@ -19,8 +21,18 @@ impl MapChunkId for DateMapChunkId {
         self.0.to_string()
     }
 
-    fn from_name(name: &str) -> Self {
-        Self(name.parse::<i32>().unwrap())
+    fn from_path(path: &Path) -> Self {
+        Self(
+            path.file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .split(".")
+                .next()
+                .unwrap()
+                .parse::<i32>()
+                .unwrap(),
+        )
     }
 
     fn to_usize(self) -> usize {

parser/src/structs/generic_map.rs

@@ -63,7 +63,7 @@ where
     Self: Ord + Debug + Copy + Clone,
 {
     fn to_name(&self) -> String;
-    fn from_name(name: &str) -> Self;
+    fn from_path(path: &Path) -> Self;
     fn to_usize(self) -> usize;
     fn from_usize(id: usize) -> Self;
 }
@@ -124,7 +124,7 @@ where
 
         let path_last = {
             if export_last {
-                Some(serialization.append_extension(&format!("{path}/last")))
+                Some(format!("{path}/last"))
             } else {
                 None
             }
@@ -188,22 +188,16 @@ where
         fs::read_dir(path)
             .unwrap()
             .map(|entry| entry.unwrap().path())
-            .filter(|path| {
-                let extension = path.extension().unwrap().to_str().unwrap();
-
-                path.is_file() && extension == serialization.to_extension()
-            })
+            .filter(|path| serialization.is_serializable(path))
             .map(|path| {
-                let chunk_id = ChunkId::from_name(path.file_stem().unwrap().to_str().unwrap());
-
+                let chunk_id = ChunkId::from_path(&path);
                 (chunk_id, path)
             })
             .collect()
     }
 
     fn import(&self, path: &Path) -> color_eyre::Result<Serialized> {
-        self.serialization
-            .import::<Serialized>(path.to_str().unwrap())
+        self.serialization.import::<Serialized>(path)
     }
 
     pub fn insert(&mut self, key: Key, value: Value) -> Value {
@@ -309,10 +303,7 @@ where
                         panic!();
                     });
 
-                    let serialized = self
-                        .serialization
-                        .import::<Serialized>(path.to_str().unwrap())
-                        .unwrap();
+                    let serialized = self.serialization.import::<Serialized>(path).unwrap();
 
                     self.imported.insert(*chunk_id, serialized);
                 }
@@ -334,23 +325,18 @@ where
                     unreachable!()
                 }
 
-                let path = self.serialization.append_extension(&format!(
-                    "{}/{}",
-                    self.path_all,
-                    chunk_id.to_name()
-                ));
-
                 let serialized = self.imported.get(chunk_id).unwrap_or_else(|| {
                     dbg!(&self.path_all, chunk_id, &self.imported);
                     panic!();
                 });
 
-                self.serialization.export(&path, serialized)?;
+                let path = format!("{}/{}", self.path_all, chunk_id.to_name());
+                self.serialization.export(Path::new(&path), serialized)?;
 
                 if index == len - 1 {
                     if let Some(path_last) = self.path_last.as_ref() {
                         self.serialization
-                            .export(path_last, serialized.last().unwrap())?;
+                            .export(Path::new(path_last), serialized.last().unwrap())?;
                     }
                 }
 

parser/src/structs/height_map_chunk_id.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use allocative::Allocative;
 use derive_deref::{Deref, DerefMut};
 
@@ -26,9 +28,20 @@ impl MapChunkId for HeightMapChunkId {
         format!("{start}..{end}")
     }
 
-    fn from_name(name: &str) -> Self {
+    fn from_path(path: &Path) -> Self {
         Self(Height::new(
-            name.split("..").next().unwrap().parse::<u32>().unwrap(),
+            path.file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .split("..")
+                .next()
+                .unwrap()
+                .parse::<u32>()
+                .unwrap_or_else(|_| {
+                    dbg!(path);
+                    panic!()
+                }),
         ))
     }
 

parser/src/structs/price.rs

@@ -2,11 +2,24 @@ use std::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign};
 
 use allocative::Allocative;
 use bincode::{Decode, Encode};
+use serde::{Deserialize, Serialize};
 
 use super::Amount;
 
 #[derive(
-    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Allocative,
+    Debug,
+    Default,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    Allocative,
 )]
 pub struct Price(u64);