general: add /api/last route

CHANGELOG.md

@@ -16,9 +16,15 @@
 - Fixed prices on charts not having a wide enough background due to the font not being fully loaded during the creation of the chart
 - Fixed window being moveable on iOS when in standalone mode when it shouldn't be
 
+## Parser
+
+- Added a `/datasets/last` json file with all the latest va
+
 ## Server
 
 - Fixed links in several places missing the `/api` part and thus not working
+- Fixed broken last values routes
+- Add support for the `/datasets/last` file via the `/api/last` route
 
 ## v. 0.4.0 | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
 

parser/Cargo.toml

@@ -7,7 +7,9 @@ edition = "2021"
 
 [dependencies]
 allocative = "0.3.3"
-bincode = { git = "https://github.com/bincode-org/bincode.git" }
+bincode = { git = "https://github.com/bincode-org/bincode.git", features = [
+    "serde",
+] }
 biter = "0.1.0"
 bitcoin_hashes = { version = "0.14.0" }
 chrono = { version = "0.4.38", features = ["serde"] }

parser/src/datasets/mod.rs

@@ -28,13 +28,14 @@ pub use constant::*;
 pub use date_metadata::*;
 pub use mining::*;
 pub use price::*;
+use serde_json::Value;
 pub use subs::*;
 pub use transaction::*;
 pub use utxo::*;
 
 use crate::{
     databases::Databases,
-    io::Json,
+    io::{Json, JSON_EXTENSION},
     states::{
         AddressCohortsInputStates,
         AddressCohortsOneShotStates,
@@ -98,9 +99,11 @@ pub struct AllDatasets {
     pub utxo: UTXODatasets,
 }
 
+const DATASETS_PATH: &str = "../datasets";
+
 impl AllDatasets {
     pub fn import() -> color_eyre::Result<Self> {
-        let path = "../datasets";
+        let path = DATASETS_PATH;
 
         let price = PriceDatasets::import(path)?;
 
@@ -264,17 +267,15 @@ impl AllDatasets {
     }
 
     pub fn export_meta_files(&self) -> color_eyre::Result<()> {
-        let path_to_type: BTreeMap<&Path, &str> = self
+        let mut path_to_type: BTreeMap<&Path, &str> = self
             .to_any_dataset_vec()
             .into_iter()
-            .flat_map(|dataset| {
+            .flat_map(|dataset| dataset.to_all_map_vec())
-                dataset
+            .flat_map(|map| map.exported_path_with_t_name())
-                    .to_all_map_vec()
-                    .into_iter()
-                    .flat_map(|map| map.exported_path_with_t_name())
-            })
             .collect();
 
+        path_to_type.insert(Path::new("../datasets/last"), "Value");
+
         let datasets_len = path_to_type.len();
 
         let server_inputs_path = "../server/in";
@@ -310,9 +311,40 @@ impl AllDatasets {
             .into_par_iter()
             .try_for_each(|dataset| -> color_eyre::Result<()> { dataset.export() })?;
 
+        let mut path_to_last: BTreeMap<String, Value> = BTreeMap::default();
+
         self.to_mut_any_dataset_vec()
             .into_iter()
-            .for_each(|dataset| dataset.post_export());
+            .for_each(|dataset| {
+                dataset.post_export();
+
+                dataset.to_all_map_vec().iter().for_each(|map| {
+                    if let Some(last_path) = map.path_last() {
+                        if let Some(last_value) = map.last_value() {
+                            let mut last_path = last_path.clone();
+                            last_path.pop();
+
+                            let last_path = last_path.to_str().unwrap();
+
+                            let skip = if last_path.starts_with(DATASETS_PATH) {
+                                2
+                            } else {
+                                1
+                            };
+
+                            path_to_last.insert(
+                                last_path.split('/').skip(skip).join("-").to_string(),
+                                last_value,
+                            );
+                        }
+                    }
+                });
+            });
+
+        Json::export(
+            Path::new(&format!("{DATASETS_PATH}/last.{JSON_EXTENSION}")),
+            &path_to_last,
+        )?;
 
         Ok(())
     }

parser/src/lib.rs

@@ -10,7 +10,7 @@ mod utils;
 pub use crate::{
     actions::iter_blocks,
     datasets::OHLC,
-    io::{Binary, Json, Serialization},
+    io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION},
     structs::{
         Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
         HEIGHT_MAP_CHUNK_SIZE,

parser/src/structs/any_map.rs

@@ -1,9 +1,13 @@
 use std::path::{Path, PathBuf};
 
+use serde_json::Value;
+
 pub trait AnyMap {
     fn path(&self) -> &Path;
     fn path_last(&self) -> &Option<PathBuf>;
 
+    fn last_value(&self) -> Option<Value>;
+
     fn t_name(&self) -> &str;
 
     fn exported_path_with_t_name(&self) -> Vec<(&Path, &str)> {
@@ -16,8 +20,6 @@ pub trait AnyMap {
         }
     }
 
-    // fn reset(&mut self) -> color_eyre::Result<()>;
-
     fn pre_export(&mut self);
     fn export(&self) -> color_eyre::Result<()>;
     fn post_export(&mut self);

parser/src/structs/generic_map.rs

@@ -286,6 +286,13 @@ where
         &self.path_last
     }
 
+    fn last_value(&self) -> Option<serde_json::Value> {
+        self.imported
+            .last_key_value()
+            .and_then(|(_, serialized)| serialized.last())
+            .and_then(|v| serde_json::to_value(v).ok())
+    }
+
     fn t_name(&self) -> &str {
         std::any::type_name::<Value>()
     }

server/src/api/handlers/dataset.rs

@@ -9,7 +9,10 @@ use color_eyre::{eyre::eyre, owo_colors::OwoColorize};
 use reqwest::StatusCode;
 use serde::Deserialize;
 
-use parser::{log, Date, DateMap, Height, HeightMap, MapChunkId, HEIGHT_MAP_CHUNK_SIZE, OHLC};
+use parser::{
+    log, Date, DateMap, Height, HeightMap, Json, MapChunkId, COMPRESSED_BIN_EXTENSION,
+    HEIGHT_MAP_CHUNK_SIZE, JSON_EXTENSION, OHLC,
+};
 
 use crate::{
     api::structs::{Chunk, Kind, Route},
@@ -17,7 +20,7 @@ use crate::{
     AppState,
 };
 
-use super::response::typed_value_to_response;
+use super::response::{typed_value_to_response, value_to_response};
 
 #[derive(Deserialize)]
 pub struct Params {
@@ -90,22 +93,25 @@ fn _dataset_handler(
 
     let mut chunk = None;
 
-    if kind != Kind::Last {
+    match kind {
-        match kind {
+        Kind::Date => {
-            Kind::Date => {
+            let datasets = DateMap::<usize>::_read_dir(&route.file_path, &route.serialization);
-                let datasets = DateMap::<usize>::_read_dir(&route.file_path, &route.serialization);
 
-                process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
+            process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
-            }
+        }
-            Kind::Height => {
+        Kind::Height => {
-                let datasets =
+            let datasets = HeightMap::<usize>::_read_dir(&route.file_path, &route.serialization);
-                    HeightMap::<usize>::_read_dir(&route.file_path, &route.serialization);
 
-                process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
+            process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
+        }
+        Kind::Last => {
+            if !route.values_type.ends_with("Value") {
+                route.file_path.set_extension(COMPRESSED_BIN_EXTENSION);
+            } else {
+                route.file_path.set_extension(JSON_EXTENSION);
             }
-            _ => panic!(),
+        }
-        };
+    };
-    }
 
     let (date, response) = headers.check_if_modified_since(&route.file_path).unwrap();
 
@@ -126,6 +132,7 @@ fn _dataset_handler(
         "OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk)?,
         "Date" => typed_value_to_response::<Date>(kind, &route, chunk)?,
         "Height" => typed_value_to_response::<Height>(kind, &route, chunk)?,
+        "Value" => value_to_response::<serde_json::Value>(Json::import(&route.file_path)?),
         _ => panic!("Incompatible type: {type_name}"),
     };
 

server/src/api/handlers/response.rs

@@ -50,7 +50,7 @@ where
     })
 }
 
-fn value_to_response<T>(value: T) -> Response
+pub fn value_to_response<T>(value: T) -> Response
 where
     T: Serialize,
 {

server/src/api/structs/routes.rs

@@ -46,7 +46,12 @@ impl Routes {
                 serialization = Serialization::Json;
             }
 
-            let split_key = split_key.iter().skip(skip).collect_vec();
+            let mut split_key = split_key.iter().skip(skip).collect_vec();
+
+            // Use case for: "../datasets/last": "Value",
+            if split_key.is_empty() {
+                split_key.push(&"last");
+            }
 
             let map_key = split_key.iter().join("_");