global: snapshot

.gitignore

@@ -1,16 +1,46 @@
+# Mac OS
 .DS_Store
 
-/app-next
-/datasets
-/datasets2
-/datasets_*
-
+# To do
+/charts
 TODO.md
 
-.stfolder
-/charts
-/price
-*\ copy*
-/dist
+# Builds
+dist
+target
 
+# I/O
+in
+out
+.log
+/datasets
+/price
+
+# Sync
+.stfolder
+
+# Copies
+*\ copy*
+
+# Ignored
+ignore
+
+# Scripts
 /start-node.sh
+
+# Editors
+.vscode
+.zed
+
+# Configs
+config.toml
+
+# Flamegraph
+flamegraph/
+flamegraph.svg
+
+# Benchmarks
+benches
+
+# Snapshots
+snapshots*/

GUIDELINES.md

@@ -0,0 +1,8 @@
+# Guidelines
+
+## Parser
+
+- Avoid floats as much as possible
+  - Use structs like `WAmount` and `Price` for calculations
+  - **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
+- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower

README.md

@@ -4,7 +4,7 @@
 
 ## Description
 
-Satonomics is a better, FOSS, Bitcoin-only, self-hostable Glassnode.
+kibō (hope) is a better, FOSS, Bitcoin-only, self-hostable Glassnode.
 
 While [mempool.space](https://mempool.space) gives a very micro view of the network where you can follow the journey of any address, this tool is the exact opposite and very complimentary by giving you a much more global/macro view of the flow and various dynamics of the network via thousands of charts.
 
@@ -39,6 +39,34 @@ API:
 - `server`: A small server which automatically creates routes to access through an API all created datasets.
 - `app`: A web app which displays the generated datasets in various charts and dashboards.
 
+## How to run
+
+### Requirements
+
+- `rustup`
+
+### Parser
+
+```bash
+./run.sh --datadir=$HOME/Developer/bitcoin
+```
+
+### Server
+
+```bash
+# Install rustup
+# Update ./run.sh if needed
+./run.sh
+```
+
+Then the easiest to let others access your server is with `cloudflared` which will also cache requests.
+
+
+## Limitations
+
+- Needs to stop the node to parse the files (at least for now)
+- Needs a **LOT** a disk space for databases (~700 GB for data from 2009 to mid 2024)
+
 ## Goals / Philosophy
 
 Adjectives that describe what this project is or strives to be, in no particular order:

app-html/.gitignore

@@ -1 +0,0 @@
-/ignore

parser/.gitignore

@@ -1,22 +0,0 @@
-.DS_Store
-/target
-/.vscode
-/.zed
-
-flamegraph/
-flamegraph.svg
-/profile.json
-
-/inputs*/
-/in
-/outputs*/
-/out
-/snapshots*/
-/exports*/
-/imports*/
-benches
-
-parser.log
-config.toml
-*\ copy*
-/.log

parser/README.md

@@ -1,27 +0,0 @@
-# Satonomics - Parser
-
-## Description
-
-The backbone of the project, it does most of the work by parsing and then computing datasets from the timechain
-
-## Requirements
-
-- `rustup`
-
-## Run
-
-```bash
-./run.sh --datadir=$HOME/Developer/bitcoin
-```
-
-## Limitations
-
-- Needs to stop the node to parse the files (at least for now)
-- Needs a **LOT** a disk space for databases (~700 GB for data from 2009 to mid 2024)
-
-## Guidelines
-
-- Avoid floats as much as possible
-  - Use structs like `WAmount` and `Price` for calculations
-  - **Only** use `WAmount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
-- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower

parser/src/databases/_database.rs

@@ -26,7 +26,7 @@ use crate::io::OUTPUTS_FOLDER_PATH;
 /// There is no `cached_gets` since it's much cheaper and faster to do a parallel search first using `unsafe_get` than caching gets along the way.
 pub struct Database<Key, Value>
 where
-    Key: Ord + Clone + Debug + ?Sized + Storable,
+    Key: Ord + Clone + Debug + Storable,
     Value: Storable + PartialEq,
 {
     pub cached_puts: BTreeMap<Key, Value>,
@@ -42,7 +42,7 @@ const PAGE_SIZE: u64 = 4096;
 
 impl<Key, Value> Database<Key, Value>
 where
-    Key: Ord + Clone + Debug + ?Sized + Storable,
+    Key: Ord + Clone + Debug + Storable,
     Value: Storable + PartialEq,
 {
     pub fn open(folder: &str, file: &str) -> color_eyre::Result<Self> {

server/.gitignore

@@ -1,4 +0,0 @@
-/target
-.DS_Store
-/.log
-/in

server/Cargo.toml

@@ -1,18 +1,21 @@
 [package]
 name = "server"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 
 [dependencies]
 axum = "0.7.5"
 color-eyre = "0.6.3"
 itertools = "0.13.0"
-regex = "1.10.5"
+regex = "1.10.6"
 bincode = { git = "https://github.com/bincode-org/bincode.git" }
-reqwest = { version = "0.12.5", features = ["json"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120" }
-tokio = { version = "1.38.0", features = ["full"] }
+reqwest = { version = "0.12.7", features = ["json"] }
+serde = { version = "1.0.210", features = ["derive"] }
+serde_json = { version = "1.0.128" }
+tokio = { version = "1.40.0", features = ["full"] }
 tower-http = { version = "0.5.2", features = ["compression-full"] }
 parser = { path = "../parser" }
 derive_deref = "1.1.1"
+swc = "0.286.0"
+swc_common = "0.37.5"
+swc_ecma_minifier = "0.205.2"

server/README.md

@@ -1,19 +0,0 @@
-# Satonomics - Server
-
-## Description
-
-A small server which automatically creates routes for all the created datasets
-
-## Requirements
-
-- `rustup`
-
-## Run
-
-```bash
-# Install rustup
-# Update ./run.sh if needed
-./run.sh
-```
-
-Then the easiest to let others access your server is with `cloudflared` which will also cache requests.

server/src/api/handlers/dataset.rs

@@ -12,8 +12,10 @@ use serde::Deserialize;
 use parser::{log, Date, DateMap, Height, HeightMap, MapChunkId, HEIGHT_MAP_CHUNK_SIZE, OHLC};
 
 use crate::{
-    chunk::Chunk, headers::add_cors_to_headers, kind::Kind, response::typed_value_to_response,
-    routes::Route, AppState,
+    api::structs::{Chunk, Kind, Route},
+    header_map::HeaderMapUtils,
+    response::typed_value_to_response,
+    AppState,
 };
 
 #[derive(Deserialize)]
@@ -21,26 +23,26 @@ pub struct Params {
     chunk: Option<usize>,
 }
 
-pub async fn file_handler(
+pub async fn dataset_handler(
     headers: HeaderMap,
     path: Path<String>,
     query: Query<Params>,
     State(app_state): State<AppState>,
 ) -> Response {
-    match _file_handler(headers, path, query, app_state) {
+    match _dataset_handler(headers, path, query, app_state) {
         Ok(response) => response,
         Err(error) => {
             let mut response =
                 (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
 
-            add_cors_to_headers(response.headers_mut());
+            response.headers_mut().insert_cors();
 
             response
         }
     }
 }
 
-fn _file_handler(
+fn _dataset_handler(
     headers: HeaderMap,
     Path(path): Path<String>,
     query: Query<Params>,

server/src/api/handlers/fallback.rs

@@ -0,0 +1,14 @@
+use axum::{extract::State, http::HeaderMap, response::Response};
+use reqwest::header::HOST;
+
+use crate::{response::generic_to_reponse, AppState};
+
+pub async fn fallback(headers: HeaderMap, State(app_state): State<AppState>) -> Response {
+    generic_to_reponse(
+        app_state
+            .routes
+            .to_full_paths(headers[HOST].to_str().unwrap().to_string()),
+        None,
+        60,
+    )
+}

server/src/api/handlers/mod.rs

@@ -0,0 +1,5 @@
+mod dataset;
+mod fallback;
+
+pub use dataset::*;
+pub use fallback::*;

server/src/api/mod.rs

@@ -0,0 +1,19 @@
+use axum::{routing::get, Router};
+use handlers::{dataset_handler, fallback};
+
+use crate::AppState;
+
+mod handlers;
+pub mod structs;
+
+pub trait ApiRoutes {
+    fn add_api_routes(self) -> Self;
+}
+
+impl ApiRoutes for Router<AppState> {
+    fn add_api_routes(self) -> Self {
+        self.route("/api/*path", get(dataset_handler))
+            .route("/api/", get(fallback))
+            .route("/api", get(fallback))
+    }
+}

server/src/api/structs/mod.rs

@@ -0,0 +1,9 @@
+mod chunk;
+mod kind;
+mod paths;
+mod routes;
+
+pub use chunk::*;
+pub use kind::*;
+pub use paths::*;
+pub use routes::*;

server/src/api/structs/routes.rs

@@ -8,7 +8,9 @@ use derive_deref::{Deref, DerefMut};
 use itertools::Itertools;
 use parser::{Json, Serialization};
 
-use crate::{paths::Paths, Grouped};
+use crate::Grouped;
+
+use super::Paths;
 
 #[derive(Clone, Debug)]
 pub struct Route {

server/src/header_map.rs

@@ -0,0 +1,74 @@
+use std::path::Path;
+
+use axum::http::{header, HeaderMap};
+use parser::log;
+
+const STALE_IF_ERROR: u64 = 31_536_000; // 1 Year
+
+pub trait HeaderMapUtils {
+    fn insert_cors(&mut self);
+
+    fn insert_cache_control(&mut self, max_age: u64, stale_while_revalidate: u64);
+
+    fn insert_content_type(&mut self, path: &Path);
+    fn insert_content_type_application_javascript(&mut self);
+    fn insert_content_type_application_json(&mut self);
+    fn insert_content_type_text_html(&mut self);
+    fn insert_content_type_text_plain(&mut self);
+    fn insert_content_type_font_woff2(&mut self);
+}
+
+impl HeaderMapUtils for HeaderMap {
+    fn insert_cors(&mut self) {
+        self.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
+        self.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
+    }
+
+    fn insert_cache_control(&mut self, max_age: u64, stale_while_revalidate: u64) {
+        self.insert(
+        header::CACHE_CONTROL,
+        format!(
+            "public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
+        .parse()
+        .unwrap(),
+    );
+    }
+
+    // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+    fn insert_content_type(&mut self, path: &Path) {
+        match path.extension().unwrap().to_str().unwrap() {
+            "js" => self.insert_content_type_application_javascript(),
+            "json" => self.insert_content_type_application_json(),
+            "html" => self.insert_content_type_text_html(),
+            "txt" => self.insert_content_type_text_plain(),
+            "woff2" => self.insert_content_type_font_woff2(),
+            extension => {
+                log(&format!("Extension unsupported: {extension}"));
+                panic!()
+            }
+        }
+    }
+
+    fn insert_content_type_application_javascript(&mut self) {
+        self.insert(
+            header::CONTENT_TYPE,
+            "application/javascript".parse().unwrap(),
+        );
+    }
+
+    fn insert_content_type_application_json(&mut self) {
+        self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
+    }
+
+    fn insert_content_type_text_html(&mut self) {
+        self.insert(header::CONTENT_TYPE, "text/html".parse().unwrap());
+    }
+
+    fn insert_content_type_text_plain(&mut self) {
+        self.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap());
+    }
+
+    fn insert_content_type_font_woff2(&mut self) {
+        self.insert(header::CONTENT_TYPE, "font/woff2".parse().unwrap());
+    }
+}

server/src/headers.rs

@@ -1,26 +0,0 @@
-use axum::http::{header, HeaderMap};
-
-const STALE_IF_ERROR: u64 = 604800; // 1 Week
-
-pub fn add_cors_to_headers(headers: &mut HeaderMap) {
-    headers.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
-    headers.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
-}
-
-pub fn add_json_type_to_headers(headers: &mut HeaderMap) {
-    headers.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
-}
-
-pub fn add_cache_control_to_headers(
-    headers: &mut HeaderMap,
-    max_age: u64,
-    stale_while_revalidate: u64,
-) {
-    headers.insert(
-        header::CACHE_CONTROL,
-        format!(
-            "public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
-        .parse()
-        .unwrap(),
-    );
-}

server/src/main.rs

@@ -1,23 +1,17 @@
 use std::sync::Arc;
 
-use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router};
+use api::{structs::Routes, ApiRoutes};
+use axum::{serve, Router};
 use parser::{log, reset_logs};
-use reqwest::header::HOST;
-use response::generic_to_reponse;
-use routes::Routes;
 use serde::Serialize;
 use tokio::net::TcpListener;
 use tower_http::compression::CompressionLayer;
+use website::WebsiteRoutes;
 
-mod chunk;
-mod handler;
-mod headers;
-mod kind;
-mod paths;
+mod api;
+mod header_map;
 mod response;
-mod routes;
-
-use handler::file_handler;
+mod website;
 
 #[derive(Clone, Debug, Default, Serialize)]
 pub struct Grouped<T> {
@@ -52,28 +46,26 @@ async fn main() -> color_eyre::Result<()> {
         .zstd(true);
 
     let router = Router::new()
-        .route("/*path", get(file_handler))
-        .route("/", get(fallback))
+        .add_api_routes()
+        .add_website_routes()
         .with_state(state)
         .layer(compression_layer);
 
-    let port = 3110;
+    let mut port = 3110;
+
+    let mut listener;
+    loop {
+        listener = TcpListener::bind(format!("0.0.0.0:{port}")).await;
+        if listener.is_ok() {
+            break;
+        }
+        port += 1;
+    }
 
     log(&format!("Starting server on port {port}..."));
-
-    let listener = TcpListener::bind(format!("0.0.0.0:{port}")).await?;
+    let listener = listener.unwrap();
 
     serve(listener, router).await?;
 
     Ok(())
 }
-
-pub async fn fallback(headers: HeaderMap, State(app_state): State<AppState>) -> Response {
-    generic_to_reponse(
-        app_state
-            .routes
-            .to_full_paths(headers[HOST].to_str().unwrap().to_string()),
-        None,
-        60,
-    )
-}

server/src/response.rs

@@ -7,10 +7,8 @@ use serde::de::DeserializeOwned;
 use serde::Serialize;
 
 use crate::{
-    chunk::Chunk,
-    headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers},
-    kind::Kind,
-    routes::Route,
+    api::structs::{Chunk, Kind, Route},
+    header_map::HeaderMapUtils,
 };
 
 #[derive(Serialize)]
@@ -88,9 +86,9 @@ where
     let max_age = cache_time;
     let stale_while_revalidate = 2 * max_age;
 
-    add_cors_to_headers(headers);
-    add_json_type_to_headers(headers);
-    add_cache_control_to_headers(headers, max_age, stale_while_revalidate);
+    headers.insert_cors();
+    headers.insert_content_type_application_json();
+    headers.insert_cache_control(max_age, stale_while_revalidate);
 
     response
 }

server/src/website/handlers/file.rs

@@ -0,0 +1,56 @@
+use std::{
+    fs::{self},
+    path::{Path, PathBuf},
+};
+
+use axum::{extract, http::HeaderMap, response::Response};
+use parser::log;
+
+use crate::header_map::HeaderMapUtils;
+
+use super::minify_js;
+
+const WEBSITE_PATH: &str = "../website";
+
+pub async fn file_handler(headers: HeaderMap, path: extract::Path<String>) -> Response {
+    let path = path.0.replace("..", "").replace("\\", "");
+    let mut path = str_to_path(&path);
+    if path.extension().is_none() && !path.exists() {
+        path = str_to_path("index.html");
+    }
+
+    path_to_response(&path)
+}
+
+pub async fn index_handler(headers: HeaderMap) -> Response {
+    path_to_response(&str_to_path("index.html"))
+}
+
+fn path_to_response(path: &Path) -> Response {
+    let mut response;
+
+    if path.extension().unwrap() == "js" {
+        let content = minify_js(path);
+
+        response = Response::new(content.into());
+    } else {
+        let content = fs::read(path).unwrap_or_else(|error| {
+            log(&error.to_string());
+            let path = path.to_str().unwrap();
+            log(&format!("Can't read file {path}"));
+            panic!("")
+        });
+
+        response = Response::new(content.into());
+    }
+
+    let headers = response.headers_mut();
+    headers.insert_cors();
+    headers.insert_content_type(path);
+
+    response
+}
+
+fn str_to_path(path: &str) -> PathBuf {
+    PathBuf::from(&format!("{WEBSITE_PATH}/{path}"))
+}

server/src/website/handlers/minify/js.rs

@@ -0,0 +1,29 @@
+// Simplified version of: https://github.com/swc-project/swc/blob/main/crates/swc/examples/minify.rs
+
+use std::{path::Path, sync::Arc};
+
+use swc::{config::JsMinifyOptions, try_with_handler, JsMinifyExtras};
+use swc_common::{SourceMap, GLOBALS};
+
+pub fn minify_js(path: &Path) -> String {
+    let cm = Arc::<SourceMap>::default();
+
+    let c = swc::Compiler::new(cm.clone());
+
+    let output = GLOBALS
+        .set(&Default::default(), || {
+            try_with_handler(cm.clone(), Default::default(), |handler| {
+                let fm = cm.load_file(path).expect("failed to load file");
+
+                c.minify(
+                    fm,
+                    handler,
+                    &JsMinifyOptions::default(),
+                    JsMinifyExtras::default(),
+                )
+            })
+        })
+        .unwrap();
+
+    output.code
+}

server/src/website/handlers/minify/mod.rs

@@ -0,0 +1,3 @@
+mod js;
+
+pub use js::*;

server/src/website/handlers/mod.rs

@@ -0,0 +1,5 @@
+mod file;
+mod minify;
+
+pub use file::*;
+use minify::*;

server/src/website/mod.rs

@@ -0,0 +1,18 @@
+use axum::{routing::get, Router};
+
+mod handlers;
+
+use handlers::{file_handler, index_handler};
+
+use crate::AppState;
+
+pub trait WebsiteRoutes {
+    fn add_website_routes(self) -> Self;
+}
+
+impl WebsiteRoutes for Router<AppState> {
+    fn add_website_routes(self) -> Self {
+        self.route("/*path", get(file_handler))
+            .route("/", get(index_handler))
+    }
+}

website/index.html

@@ -269,7 +269,7 @@
      */
     @font-face {
       font-family: "Satoshi";
-      src: url("./fonts/Satoshi.var.woff2") format("woff2");
+      src: url("./fonts/satoshi.var.woff2") format("woff2");
       font-weight: 100 900;
       font-display: block;
       font-style: normal;
@@ -277,7 +277,7 @@
 
     @font-face {
       font-family: "Satoshi Chart";
-      src: url("./fonts/Satoshi.var.woff2") format("woff2");
+      src: url("./fonts/satoshi.var.woff2") format("woff2");
       font-weight: 500 900;
       font-display: block;
       font-style: normal;