diff --git a/server/Cargo.toml b/server/Cargo.toml index d249fd57d..4d209cafc 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,37 +1,34 @@ [package] -name = "kibo_server" +name = "berver" version = "0.6.0" edition = "2021" [dependencies] -# allocative = "0.3.4" -axum = "0.8.1" -# bincode = { git = "https://github.com/bincode-org/bincode.git", features = [ -# "serde", -# ] } # bitcoin_hashes = { version = "0.15.0" } # biter = { path = "./src/crates/biter" } # chrono = { version = "0.4.39", features = ["serde"] } # clap = { version = "4.5.26", features = ["derive"] } -color-eyre = { workspace = true } # ctrlc = { version = "3.4.5", features = ["termination"] } -# derive_deref = "1.1.1" -# env_logger = "0.11.6" # inferno = "0.12.1" # itertools = "0.13.0" -# log = { version = "0.4.25", features = ["std", "serde"] } # ordered-float = "4.6.0" # rayon = "1.10.0" +# rlimit = "0.10.2" +# serde_json = "1.0.135" +axum = "0.8.1" +color-eyre = { workspace = true } +computer = { workspace = true } +derive_deref = { workspace = true } +indexer = { workspace = true } +jiff = { workspace = true } +logger = { workspace = true } +oxc = { version = "0.49.0", features = ["codegen", "minifier"] } regex = "1.11.1" reqwest = { version = "0.12.12", features = ["blocking", "json"] } -# rlimit = "0.10.2" -# snkrj = { path = "./src/crates/snkrj" } -# serde = { version = "1.0.217", features = ["derive"] } -# serde_json = "1.0.135" -# struct_iterable = { path = "./src/crates/iterable" } -swc = "13.0.1" -swc_common = "6.0.0" +serde = { workspace = true } +serde_json = "1.0.138" +storable_vec = { workspace = true } +struct_iterable = { workspace = true } tokio = { version = "1.43.0", features = ["full"] } -# toml = "0.8.19" tower-http = { version = "0.6.2", features = ["compression-full"] } zstd = "0.13.2" diff --git a/server/src/api/handlers/dataset.rs b/server/src/api/handlers/dataset.rs index b7cc4154e..2a6e4d04f 100644 --- a/server/src/api/handlers/dataset.rs +++ b/server/src/api/handlers/dataset.rs @@ -15,9 +15,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{ server::{ api::{ - structs::{ - ChunkMetadata, DatasetRange, DatasetRangeChunk, Extension, Kind, Route, Routes, - }, + structs::{ChunkMetadata, DatasetRange, DatasetRangeChunk, Extension, Kind, Route, Routes}, API_URL_PREFIX, }, header_map::{HeaderMapExtended, Modified}, @@ -26,8 +24,8 @@ use crate::{ AppState, }, structs::{ - Date, GenericMap, Height, HeightMapChunkId, MapChunkId, MapKey, MapSerialized, MapValue, - SerializedBTreeMap, SerializedDateMap, SerializedTimeMap, SerializedVec, Timestamp, OHLC, + Date, GenericMap, Height, HeightMapChunkId, MapChunkId, MapKey, MapSerialized, MapValue, SerializedBTreeMap, + SerializedDateMap, SerializedTimeMap, SerializedVec, Timestamp, OHLC, }, }; @@ -50,12 +48,8 @@ pub async fn dataset_handler( "{API_URL_PREFIX}/{}?kind={}{}{}", path.0, query.kind, - query - .chunk - .map_or("".to_string(), |chunk| format!("&chunk={chunk}")), - query - .all - .map_or("".to_string(), |all| format!("&all={all}")) + query.chunk.map_or("".to_string(), |chunk| format!("&chunk={chunk}")), + query.all.map_or("".to_string(), |all| format!("&all={all}")) ); match result_handler(headers, &path, &query, app_state) { @@ -64,8 +58,7 @@ pub async fn dataset_handler( response } Err(error) => { - let mut response = - (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); log_result(response.status(), &ser_path, instant); response.headers_mut().insert_cors(); response @@ -129,64 +122,49 @@ where let last_value: T = route.serialization.import(&route.path.join("last"))?; return Ok(axum::response::Json(last_value).into_response()); } - Kind::Date => match read_serialized::>( - id, &headers, route, &range, query, - )? { + Kind::Date => match read_serialized::>(id, &headers, route, &range, query)? { ReadSerialized::DatasetAndDate((dataset, date, chunk_meta)) => { (serialized_to_response(dataset, id, chunk_meta, ext), date) } ReadSerialized::NotModified => return Ok(Response::new_not_modified()), ReadSerialized::_Phantom(_) => unreachable!(), }, - Kind::Height => match read_serialized::>( - id, &headers, route, &range, query, - )? { + Kind::Height => match read_serialized::>(id, &headers, route, &range, query)? { ReadSerialized::DatasetAndDate((dataset, date, chunk_meta)) => ( - serialized_to_response::>( - dataset, id, chunk_meta, ext, - ), + serialized_to_response::>(dataset, id, chunk_meta, ext), date, ), ReadSerialized::NotModified => return Ok(Response::new_not_modified()), ReadSerialized::_Phantom(_) => unreachable!(), }, Kind::Timestamp => { - let (dataset, date, chunk_meta) = match read_serialized::>( - id, &headers, route, &range, query, + let (dataset, date, chunk_meta) = + match read_serialized::>(id, &headers, route, &range, query)? { + ReadSerialized::DatasetAndDate(tuple) => tuple, + ReadSerialized::NotModified => return Ok(Response::new_not_modified()), + ReadSerialized::_Phantom(_) => unreachable!(), + }; + + let (timestamp_dataset, _, _) = match read_serialized::>( + "timestamp", + &headers, + routes.get("timestamp").unwrap(), + &range, + query, )? { ReadSerialized::DatasetAndDate(tuple) => tuple, ReadSerialized::NotModified => return Ok(Response::new_not_modified()), ReadSerialized::_Phantom(_) => unreachable!(), }; - let (timestamp_dataset, _, _) = - match read_serialized::>( - "timestamp", - &headers, - routes.get("timestamp").unwrap(), - &range, - query, - )? { - ReadSerialized::DatasetAndDate(tuple) => tuple, - ReadSerialized::NotModified => return Ok(Response::new_not_modified()), - ReadSerialized::_Phantom(_) => unreachable!(), - }; - let mut serialized_timemap: SerializedTimeMap = SerializedBTreeMap::default(); - dataset - .map - .into_iter() - .enumerate() - .for_each(|(index, value)| { - serialized_timemap.map.insert( - timestamp_dataset - .get_index(index) - .cloned() - .unwrap_or(Timestamp::now()), - value, - ); - }); + dataset.map.into_iter().enumerate().for_each(|(index, value)| { + serialized_timemap.map.insert( + timestamp_dataset.get_index(index).cloned().unwrap_or(Timestamp::now()), + value, + ); + }); ( serialized_to_response::>( @@ -287,19 +265,13 @@ where let date_modified; - let datasets = - GenericMap::::_read_dir(&folder_path, serialization); + let datasets = GenericMap::::_read_dir(&folder_path, serialization); let mut chunk_meta = None; let dataset = if let DatasetRange::Chunk(range_chunk) = range { let chunk_id = match range_chunk { - DatasetRangeChunk::Last => { - *datasets - .last_key_value() - .context("Last tuple of dataset directory")? - .0 - } + DatasetRangeChunk::Last => *datasets.last_key_value().context("Last tuple of dataset directory")?.0, DatasetRangeChunk::Chunk(chunk) => ChunkId::from_usize(*chunk), }; @@ -358,11 +330,7 @@ where Serialized::import_all(&folder_path, serialization) }; - Ok(ReadSerialized::DatasetAndDate(( - dataset, - date_modified, - chunk_meta, - ))) + Ok(ReadSerialized::DatasetAndDate((dataset, date_modified, chunk_meta))) } #[derive(Serialize)] diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 9ea585089..8cbd71a6f 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,12 +1,25 @@ -use axum::{routing::get, Router}; -use handlers::{dataset_handler, last_values_handler}; +use std::time::Instant; + +use axum::{ + extract::{Query, State}, + http::HeaderMap, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use color_eyre::eyre::eyre; +use reqwest::StatusCode; +use serde::Deserialize; +use structs::{Format, Index}; + +use crate::{log_result, traits::HeaderMapExtended}; use super::AppState; -mod handlers; +// mod handlers; pub mod structs; -pub const API_URL_PREFIX: &str = "/api"; +pub const VECS_URL_PREFIX: &str = "/api/vecs"; pub trait ApiRoutes { fn add_api_routes(self) -> Self; @@ -14,7 +27,81 @@ pub trait ApiRoutes { impl ApiRoutes for Router { fn add_api_routes(self) -> Self { - self.route(&format!("{API_URL_PREFIX}/last"), get(last_values_handler)) - .route(&format!("{API_URL_PREFIX}/*path"), get(dataset_handler)) + self.route(VECS_URL_PREFIX, get(handler)) + } +} + +#[derive(Debug, Deserialize)] +pub struct DatasetParams { + pub i: String, + pub v: String, + pub from: Option, + pub to: Option, + pub format: Option, +} + +pub async fn handler(headers: HeaderMap, query: Query, State(app_state): State) -> Response { + let instant = Instant::now(); + + let path = format!( + "{VECS_URL_PREFIX}?i={}&v={}{}{}", + query.i, + query.v, + query.from.map_or("".to_string(), |from| format!("&from={from}")), + query.to.map_or("".to_string(), |to| format!("&to={to}")), + ); + + match req_to_response_res(headers, query, app_state) { + Ok(response) => { + log_result(response.status(), &path, instant); + response + } + Err(error) => { + let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + log_result(response.status(), &path, instant); + response.headers_mut().insert_cors(); + response + } + } +} + +fn req_to_response_res( + headers: HeaderMap, + Query(DatasetParams { format, from, i, to, v }): Query, + AppState { vecs, .. }: AppState, +) -> color_eyre::Result { + let format = Format::try_from(format).ok(); + let indexes = i + .to_lowercase() + .split(",") + .flat_map(|s| Index::try_from(s).ok()) + .collect::>(); + + if indexes.len() > 1 { + return Err(eyre!("Multiple indexes aren't supported yet !")); + } else if indexes.is_empty() { + return Err(eyre!("Bad index(es)")); + } + + dbg!(format, &indexes, &v, from, to); + + let values = v + .to_lowercase() + .split(",") + .flat_map(|s| vecs.get(&s.replace("_", "-"))) + .flat_map(|i_to_v| i_to_v.get(indexes.first().unwrap())) + .map(|vec| vec.collect_range(from, to).unwrap()) + .collect::>(); + + if values.len() == 1 { + let values = values.first().unwrap(); + if values.len() == 1 { + let value = values.first().unwrap(); + Ok(Json(value).into_response()) + } else { + Ok(Json(values).into_response()) + } + } else { + Ok(Json(values).into_response()) } } diff --git a/server/src/api/structs/chunk_metadata.rs b/server/src/api/structs/chunk_metadata.rs deleted file mode 100644 index 9dc2535e0..000000000 --- a/server/src/api/structs/chunk_metadata.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -pub struct ChunkMetadata { - pub id: usize, - pub previous: Option, - pub next: Option, -} diff --git a/server/src/api/structs/extension.rs b/server/src/api/structs/extension.rs deleted file mode 100644 index 1a4d5a33c..000000000 --- a/server/src/api/structs/extension.rs +++ /dev/null @@ -1,34 +0,0 @@ -use std::path::Path; - -#[derive(PartialEq, Eq, Clone, Copy)] -pub enum Extension { - #[allow(clippy::upper_case_acronyms)] - CSV, - #[allow(clippy::upper_case_acronyms)] - JSON, -} - -impl Extension { - pub fn from(path: &Path) -> Option { - if let Some(extension) = path.extension() { - let extension = extension.to_str().unwrap(); - - if extension == Self::CSV.to_str() { - Some(Self::CSV) - } else if extension == Self::JSON.to_str() { - Some(Self::JSON) - } else { - None - } - } else { - None - } - } - - pub fn to_str(&self) -> &str { - match self { - Extension::CSV => "csv", - Extension::JSON => "json", - } - } -} diff --git a/server/src/api/structs/format.rs b/server/src/api/structs/format.rs new file mode 100644 index 000000000..5e0da56a8 --- /dev/null +++ b/server/src/api/structs/format.rs @@ -0,0 +1,27 @@ +use color_eyre::eyre::eyre; + +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + CSV, + JSON, +} + +impl TryFrom> for Format { + type Error = color_eyre::Report; + fn try_from(value: Option) -> Result { + if let Some(value) = value { + let value = value.to_lowercase(); + let value = value.as_str(); + if value == "csv" { + Ok(Self::CSV) + } else if value == "json" { + Ok(Self::JSON) + } else { + Err(eyre!("Fail")) + } + } else { + Err(eyre!("Fail")) + } + } +} diff --git a/server/src/api/structs/index.rs b/server/src/api/structs/index.rs new file mode 100644 index 000000000..5de782cdf --- /dev/null +++ b/server/src/api/structs/index.rs @@ -0,0 +1,122 @@ +use std::fmt::{self, Debug}; + +use computer::Date; +use indexer::{ + Addressindex, Height, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Txindex, + Txinindex, Txoutindex, +}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum Index { + Addressindex, + Date, + Height, + P2PK33index, + P2PK65index, + P2PKHindex, + P2SHindex, + P2TRindex, + P2WPKHindex, + P2WSHindex, + Txindex, + Txinindex, + Txoutindex, +} + +impl TryFrom<&str> for Index { + type Error = (); + fn try_from(value: &str) -> Result { + Ok(match value { + "addri" | "addressindex" => Self::Addressindex, + "d" | "date" => Self::Date, + "h" | "height" => Self::Height, + "p2pk33index" => Self::P2PK33index, + "p2pk65index" => Self::P2PK65index, + "p2pkhindex" => Self::P2PKHindex, + "p2shindex" => Self::P2SHindex, + "p2trindex" => Self::P2TRindex, + "p2wpkhindex" => Self::P2WPKHindex, + "p2wshindex" => Self::P2WSHindex, + "txi" | "txindex" => Self::Txindex, + "txini" | "txinindex" => Self::Txinindex, + "txouti" | "txoutindex" => Self::Txoutindex, + _ => return Err(()), + }) + } +} + +impl fmt::Display for Index { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(self, f) + } +} + +pub trait IndexTypeToIndexEnum { + fn to_enum() -> Index; +} + +impl IndexTypeToIndexEnum for Addressindex { + fn to_enum() -> Index { + Index::Addressindex + } +} +impl IndexTypeToIndexEnum for Date { + fn to_enum() -> Index { + Index::Date + } +} +impl IndexTypeToIndexEnum for Height { + fn to_enum() -> Index { + Index::Height + } +} +impl IndexTypeToIndexEnum for Txindex { + fn to_enum() -> Index { + Index::Txindex + } +} +impl IndexTypeToIndexEnum for Txinindex { + fn to_enum() -> Index { + Index::Txinindex + } +} +impl IndexTypeToIndexEnum for Txoutindex { + fn to_enum() -> Index { + Index::Txoutindex + } +} +impl IndexTypeToIndexEnum for P2PK33index { + fn to_enum() -> Index { + Index::P2PK33index + } +} +impl IndexTypeToIndexEnum for P2PK65index { + fn to_enum() -> Index { + Index::P2PK65index + } +} +impl IndexTypeToIndexEnum for P2PKHindex { + fn to_enum() -> Index { + Index::P2PKHindex + } +} +impl IndexTypeToIndexEnum for P2SHindex { + fn to_enum() -> Index { + Index::P2SHindex + } +} +impl IndexTypeToIndexEnum for P2TRindex { + fn to_enum() -> Index { + Index::P2TRindex + } +} +impl IndexTypeToIndexEnum for P2WPKHindex { + fn to_enum() -> Index { + Index::P2WPKHindex + } +} +impl IndexTypeToIndexEnum for P2WSHindex { + fn to_enum() -> Index { + Index::P2WSHindex + } +} diff --git a/server/src/api/structs/kind.rs b/server/src/api/structs/kind.rs index 67456ceb5..31e6a852a 100644 --- a/server/src/api/structs/kind.rs +++ b/server/src/api/structs/kind.rs @@ -9,8 +9,8 @@ use crate::structs::{AnyMap, Date, Height, MapKey}; pub enum Kind { Date, Height, - Timestamp, - Last, + // Timestamp, + // Last, } impl TryFrom<&String> for Kind { @@ -26,8 +26,8 @@ impl TryFrom<&String> for Kind { { 'd' => Self::Date, 'h' => Self::Height, - 't' => Self::Timestamp, - 'l' => Self::Last, + // 't' => Self::Timestamp, + // 'l' => Self::Last, _ => return Err(eyre!("Bad kind")), }, ) diff --git a/server/src/api/structs/mod.rs b/server/src/api/structs/mod.rs index 76df682d8..33cc08128 100644 --- a/server/src/api/structs/mod.rs +++ b/server/src/api/structs/mod.rs @@ -1,13 +1,15 @@ -mod chunk_metadata; -mod extension; -mod kind; -mod range; -mod route; -mod routes; +// mod chunk_metadata; +mod format; +mod index; +// mod kind; +// mod range; +// mod route; +// mod routes; -pub use chunk_metadata::*; -pub use extension::*; -pub use kind::*; -pub use range::*; -pub use route::*; -pub use routes::*; +// pub use chunk_metadata::*; +pub use format::*; +pub use index::*; +// pub use kind::*; +// pub use range::*; +// pub use route::*; +// pub use routes::*; diff --git a/server/src/lib.rs b/server/src/lib.rs index 6cdd0422e..bf1271520 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,40 +1,155 @@ -use std::{sync::Arc, time::Instant}; +use std::{collections::BTreeMap, time::Instant}; -use api::{structs::Routes, ApiRoutes}; -use axum::{routing::get, serve, Router}; +use api::{ + structs::{Index, IndexTypeToIndexEnum}, + ApiRoutes, +}; +use axum::{body::Body, response::IntoResponse, routing::get, serve, Router}; use color_eyre::owo_colors::OwoColorize; -use log::{error, info}; +use computer::Computer; +use derive_deref::{Deref, DerefMut}; +use indexer::Indexer; +use logger::{error, info}; use reqwest::StatusCode; +use serde::Serialize; +use serde_json::Value; +use storable_vec::{StorableVecIndex, StorableVecType, STATELESS}; use tokio::net::TcpListener; use tower_http::compression::CompressionLayer; use website::WebsiteRoutes; -use crate::structs::Config; - -pub mod api; -mod header_map; -mod response; +mod api; +mod traits; mod website; #[derive(Clone)] pub struct AppState { - routes: Arc, - config: Config, + vecs: &'static VecIdToIndexToVec, + indexer: &'static Indexer, + computer: &'static Computer, } -pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> { - routes.generate_dts_file(); +#[derive(Default, Deref, DerefMut)] +pub struct VecIdToIndexToVec(BTreeMap); + +impl VecIdToIndexToVec { + pub fn insert(&mut self, vec: &'static storable_vec::StorableVec) + where + I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync, + T: StorableVecType + Send + Sync + Serialize, + { + let file_name = vec.file_name(); + let split = file_name.split("_to_").collect::>(); + if split.len() != 2 { + panic!(); + } + let index = vec.key_to_enum(); + if split[0] != index.to_string().to_lowercase() { + dbg!(split[0], index.to_string()); + panic!(); + } + let key = split[1].to_string().replace("_", "-"); + let prev = self.entry(key).or_default().insert(index, vec); + if prev.is_some() { + panic!() + } + } +} + +#[derive(Default, Deref, DerefMut)] +pub struct IndexToVec { + pub index_to_vec: BTreeMap, +} + +pub trait AnyStatelessStorableVec { + fn key_to_enum(&self) -> Index; + fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result>; + // fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result>; +} + +impl AnyStatelessStorableVec for storable_vec::StorableVec +where + I: StorableVecIndex + IndexTypeToIndexEnum + Send + Sync, + T: StorableVecType + Send + Sync + Serialize, +{ + fn key_to_enum(&self) -> Index { + I::to_enum() + } + + fn collect_range(&self, from: Option, to: Option) -> storable_vec::Result> { + Ok(self + .collect_range(from, to)? + .into_iter() + .map(|v| serde_json::to_value(v).unwrap()) + .collect::>()) + } +} + +trait StatelessVecs { + fn parse(&'static self, vecs: &mut VecIdToIndexToVec); +} + +impl StatelessVecs for Indexer { + fn parse(&'static self, vecs: &mut VecIdToIndexToVec) { + vecs.insert(&self.vecs.addressindex_to_addresstype); + vecs.insert(&self.vecs.addressindex_to_addresstypeindex); + vecs.insert(&self.vecs.addressindex_to_height); + vecs.insert(&self.vecs.height_to_blockhash); + vecs.insert(&self.vecs.height_to_difficulty); + vecs.insert(&self.vecs.height_to_first_addressindex); + vecs.insert(&self.vecs.height_to_first_emptyindex); + vecs.insert(&self.vecs.height_to_first_multisigindex); + vecs.insert(&self.vecs.height_to_first_opreturnindex); + vecs.insert(&self.vecs.height_to_first_pushonlyindex); + vecs.insert(&self.vecs.height_to_first_txindex); + vecs.insert(&self.vecs.height_to_first_txinindex); + vecs.insert(&self.vecs.height_to_first_txoutindex); + vecs.insert(&self.vecs.height_to_first_unknownindex); + vecs.insert(&self.vecs.height_to_first_p2pk33index); + vecs.insert(&self.vecs.height_to_first_p2pk65index); + vecs.insert(&self.vecs.height_to_first_p2pkhindex); + vecs.insert(&self.vecs.height_to_first_p2shindex); + vecs.insert(&self.vecs.height_to_first_p2trindex); + vecs.insert(&self.vecs.height_to_first_p2wpkhindex); + vecs.insert(&self.vecs.height_to_first_p2wshindex); + vecs.insert(&self.vecs.height_to_size); + vecs.insert(&self.vecs.height_to_timestamp); + vecs.insert(&self.vecs.height_to_weight); + vecs.insert(&self.vecs.p2pk33index_to_p2pk33addressbytes); + vecs.insert(&self.vecs.p2pk65index_to_p2pk65addressbytes); + vecs.insert(&self.vecs.p2pkhindex_to_p2pkhaddressbytes); + vecs.insert(&self.vecs.p2shindex_to_p2shaddressbytes); + vecs.insert(&self.vecs.p2trindex_to_p2traddressbytes); + vecs.insert(&self.vecs.p2wpkhindex_to_p2wpkhaddressbytes); + vecs.insert(&self.vecs.p2wshindex_to_p2wshaddressbytes); + vecs.insert(&self.vecs.txindex_to_first_txinindex); + vecs.insert(&self.vecs.txindex_to_first_txoutindex); + vecs.insert(&self.vecs.txindex_to_height); + vecs.insert(&self.vecs.txindex_to_locktime); + vecs.insert(&self.vecs.txindex_to_txid); + vecs.insert(&self.vecs.txindex_to_txversion); + vecs.insert(&self.vecs.txinindex_to_txoutindex); + vecs.insert(&self.vecs.txoutindex_to_addressindex); + vecs.insert(&self.vecs.txoutindex_to_amount); + } +} + +pub async fn main(indexer: Indexer, computer: Computer) -> color_eyre::Result<()> { + // pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> { + // routes.generate_dts_file(); + + let indexer = Box::leak(Box::new(indexer)); + let computer = Box::leak(Box::new(computer)); + let vecs = Box::leak(Box::new(VecIdToIndexToVec::default())); + indexer.parse(vecs); let state = AppState { - routes: Arc::new(routes), - config, + vecs, + indexer, + computer, }; - let compression_layer = CompressionLayer::new() - .br(true) - .deflate(true) - .gzip(true) - .zstd(true); + let compression_layer = CompressionLayer::new().br(true).deflate(true).gzip(true).zstd(true); let router = Router::new() .add_api_routes() @@ -55,6 +170,7 @@ pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> { } info!("Starting server on port {port}..."); + let listener = listener.unwrap(); serve(listener, router).await?; diff --git a/server/src/main.rs b/server/src/main.rs new file mode 100644 index 000000000..b01a47c11 --- /dev/null +++ b/server/src/main.rs @@ -0,0 +1,20 @@ +use std::path::Path; + +use computer::Computer; +use indexer::Indexer; +use storable_vec::STATELESS; + +#[tokio::main] +pub async fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + + logger::init_log(None); + + let path = Path::new("../_outputs"); + let indexer: Indexer = Indexer::import(&path.join("indexes"))?; + let computer: Computer = Computer::import(&path.join("computed"))?; + + berver::main(indexer, computer).await.unwrap(); + + Ok(()) +} diff --git a/server/src/header_map.rs b/server/src/traits/header_map.rs similarity index 80% rename from server/src/header_map.rs rename to server/src/traits/header_map.rs index c9b63c42c..68aaf902d 100644 --- a/server/src/header_map.rs +++ b/server/src/traits/header_map.rs @@ -1,15 +1,15 @@ -use std::path::Path; +use std::{path::Path, time}; use axum::http::{header, HeaderMap}; -use chrono::{DateTime, Timelike, Utc}; -use log::info; +use jiff::{civil::DateTime, fmt::strtime, tz::TimeZone, Timestamp}; +use logger::info; use reqwest::header::{HOST, IF_MODIFIED_SINCE}; const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; #[derive(PartialEq, Eq)] -pub enum Modified { +pub enum ModifiedState { ModifiedSince, NotModifiedSince, } @@ -23,14 +23,13 @@ pub trait HeaderMapExtended { fn insert_cors(&mut self); - fn get_if_modified_since(&self) -> Option>; - fn check_if_modified_since(&self, path: &Path) - -> color_eyre::Result<(Modified, DateTime)>; + fn get_if_modified_since(&self) -> Option; + fn check_if_modified_since(&self, path: &Path) -> color_eyre::Result<(ModifiedState, DateTime)>; fn insert_cache_control_immutable(&mut self); #[allow(unused)] fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64); - fn insert_last_modified(&mut self, date: DateTime); + fn insert_last_modified(&mut self, date: DateTime); fn insert_content_disposition_attachment(&mut self); @@ -102,40 +101,39 @@ impl HeaderMapExtended for HeaderMap { ); } - fn insert_last_modified(&mut self, date: DateTime) { - let formatted = date.format(MODIFIED_SINCE_FORMAT).to_string(); + fn insert_last_modified(&mut self, date: DateTime) { + let formatted = date + .to_zoned(TimeZone::system()) + .unwrap() + .strftime(MODIFIED_SINCE_FORMAT) + .to_string(); self.insert(header::LAST_MODIFIED, formatted.parse().unwrap()); } - fn check_if_modified_since( - &self, - path: &Path, - ) -> color_eyre::Result<(Modified, DateTime)> { - let time = path.metadata()?.modified()?; - let date: DateTime = time.into(); - let date = date.with_nanosecond(0).unwrap(); + fn check_if_modified_since(&self, path: &Path) -> color_eyre::Result<(ModifiedState, DateTime)> { + let duration = path.metadata()?.modified()?.duration_since(time::UNIX_EPOCH).unwrap(); + let date = Timestamp::new(duration.as_secs() as i64, 0) + .unwrap() + .to_zoned(TimeZone::UTC) + .datetime(); if let Some(if_modified_since) = self.get_if_modified_since() { if if_modified_since == date { - return Ok((Modified::NotModifiedSince, date)); + return Ok((ModifiedState::NotModifiedSince, date)); } } - Ok((Modified::ModifiedSince, date)) + Ok((ModifiedState::ModifiedSince, date)) } - fn get_if_modified_since(&self) -> Option> { + fn get_if_modified_since(&self) -> Option { if let Some(modified_since) = self.get(IF_MODIFIED_SINCE) { if let Ok(modified_since) = modified_since.to_str() { - let date = DateTime::parse_from_str( - &format!("{modified_since} +00:00"), - &format!("{MODIFIED_SINCE_FORMAT} %z"), - ); - - if let Ok(x) = date { - return Some(x.to_utc()); - } + return strtime::parse(MODIFIED_SINCE_FORMAT, modified_since) + .unwrap() + .to_datetime() + .ok(); } } @@ -176,10 +174,7 @@ impl HeaderMapExtended for HeaderMap { } fn insert_content_type_application_javascript(&mut self) { - self.insert( - header::CONTENT_TYPE, - "application/javascript".parse().unwrap(), - ); + self.insert(header::CONTENT_TYPE, "application/javascript".parse().unwrap()); } fn insert_content_type_application_json(&mut self) { @@ -187,10 +182,7 @@ impl HeaderMapExtended for HeaderMap { } fn insert_content_type_application_manifest_json(&mut self) { - self.insert( - header::CONTENT_TYPE, - "application/manifest+json".parse().unwrap(), - ); + self.insert(header::CONTENT_TYPE, "application/manifest+json".parse().unwrap()); } fn insert_content_type_application_pdf(&mut self) { diff --git a/server/src/traits/mod.rs b/server/src/traits/mod.rs new file mode 100644 index 000000000..7055e7209 --- /dev/null +++ b/server/src/traits/mod.rs @@ -0,0 +1,5 @@ +mod header_map; +mod response; + +pub use header_map::*; +pub use response::*; diff --git a/server/src/response.rs b/server/src/traits/response.rs similarity index 100% rename from server/src/response.rs rename to server/src/traits/response.rs diff --git a/server/src/website/handlers/_minify.rs b/server/src/website/handlers/_minify.rs deleted file mode 100644 index aa5497b55..000000000 --- a/server/src/website/handlers/_minify.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Files are bigger than with SWC, to retest later - -// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs - -use std::{fs, path::Path}; - -use oxc::{ - allocator::Allocator, - codegen::{CodeGenerator, CodegenOptions}, - minifier::{MinifierOptions, MinifierReturn}, - parser::{Parser, ParserReturn}, - span::SourceType, -}; - -// -pub fn minify_js(path: &Path) -> String { - let allocator = Allocator::default(); - - let source_type = SourceType::from_path(path).unwrap(); - - let source_text = fs::read_to_string(path).unwrap(); - - let ParserReturn { mut program, .. } = - Parser::new(&allocator, &source_text, source_type).parse(); - - let minifier = oxc::minifier::Minifier::new(MinifierOptions::default()); - - let MinifierReturn { mangler } = minifier.build(&allocator, &mut program); - - CodeGenerator::new() - .with_options(CodegenOptions { - single_quote: false, - minify: true, - comments: false, - annotation_comments: false, - source_map_path: None, - }) - .with_mangler(mangler) - .build(&program) - .code -} diff --git a/server/src/website/handlers/file.rs b/server/src/website/handlers/file.rs index c2788c951..e94022da8 100644 --- a/server/src/website/handlers/file.rs +++ b/server/src/website/handlers/file.rs @@ -10,18 +10,17 @@ use axum::{ http::HeaderMap, response::{IntoResponse, Response}, }; -use log::{error, info}; +use logger::{error, info}; use reqwest::StatusCode; -use crate::server::{ - header_map::{HeaderMapExtended, Modified}, +use crate::{ log_result, - response::ResponseExtended, + traits::{HeaderMapExtended, ModifiedState, ResponseExtended}, }; use super::minify_js; -const WEBSITE_PATH: &str = "./src/website/"; +const WEBSITE_DEV_PATH: &str = "../website/"; pub async fn file_handler(headers: HeaderMap, path: extract::Path) -> Response { any_handler(headers, Some(path)) @@ -41,11 +40,8 @@ fn any_handler(headers: HeaderMap, path: Option>) -> Respo if !path.exists() { if path.extension().is_some() { - let mut response: Response = ( - StatusCode::INTERNAL_SERVER_ERROR, - "File doesn't exist".to_string(), - ) - .into_response(); + let mut response: Response = + (StatusCode::INTERNAL_SERVER_ERROR, "File doesn't exist".to_string()).into_response(); response.headers_mut().insert_cors(); @@ -70,11 +66,10 @@ fn any_handler(headers: HeaderMap, path: Option>) -> Respo } fn path_to_response(headers: &HeaderMap, path: &Path) -> Response { - match _path_to_response(headers, path) { + match path_to_response_(headers, path) { Ok(response) => response, Err(error) => { - let mut response = - (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); response.headers_mut().insert_cors(); @@ -83,9 +78,9 @@ fn path_to_response(headers: &HeaderMap, path: &Path) -> Response { } } -fn _path_to_response(headers: &HeaderMap, path: &Path) -> color_eyre::Result { +fn path_to_response_(headers: &HeaderMap, path: &Path) -> color_eyre::Result { let (modified, date) = headers.check_if_modified_since(path)?; - if modified == Modified::NotModifiedSince { + if modified == ModifiedState::NotModifiedSince { return Ok(Response::new_not_modified()); } @@ -124,10 +119,7 @@ fn _path_to_response(headers: &HeaderMap, path: &Path) -> color_eyre::Result color_eyre::Result PathBuf { - PathBuf::from(&format!("{WEBSITE_PATH}{path}")) + PathBuf::from(&format!("{WEBSITE_DEV_PATH}{path}")) } diff --git a/server/src/website/handlers/minify.rs b/server/src/website/handlers/minify.rs index 5391b718a..15705778b 100644 --- a/server/src/website/handlers/minify.rs +++ b/server/src/website/handlers/minify.rs @@ -1,22 +1,43 @@ -// Simplified version of: https://github.com/swc-project/swc/blob/main/crates/swc/examples/minify.rs +// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs -use std::{path::Path, sync::Arc}; +use std::{fs, path::Path}; -use swc::{config::JsMinifyOptions, try_with_handler, JsMinifyExtras}; -use swc_common::{SourceMap, GLOBALS}; +use oxc::{ + allocator::Allocator, + codegen::{CodeGenerator, CodegenOptions, LegalComment}, + minifier::{CompressOptions, MangleOptions, Minifier, MinifierOptions}, + parser::Parser, + span::SourceType, +}; +// pub fn minify_js(path: &Path) -> String { - let source_map = Arc::::default(); - let compiler = swc::Compiler::new(source_map.clone()); + let allocator = Allocator::default(); - GLOBALS - .set(&Default::default(), || { - try_with_handler(source_map.clone(), Default::default(), |handler| { - let fm = source_map.load_file(path).expect("failed to load file"); + let source_type = SourceType::from_path(path).unwrap(); - compiler.minify(fm, handler, &JsMinifyOptions::default(), JsMinifyExtras::default()) - }) + let source_text = fs::read_to_string(path).unwrap(); + + let parser_return = Parser::new(&allocator, &source_text, source_type).parse(); + + let mut program = parser_return.program; + + let minifier_return = Minifier::new(MinifierOptions { + mangle: Some(MangleOptions::default()), + compress: Some(CompressOptions::default()), + }) + .build(&allocator, &mut program); + + CodeGenerator::new() + .with_options(CodegenOptions { + minify: true, + single_quote: false, + comments: false, + annotation_comments: false, + source_map_path: None, + legal_comments: LegalComment::None, }) - .unwrap() + .with_symbol_table(minifier_return.symbol_table) + .build(&program) .code } diff --git a/server/src/website/mod.rs b/server/src/website/mod.rs index fcb31516e..5aa3a0d09 100644 --- a/server/src/website/mod.rs +++ b/server/src/website/mod.rs @@ -12,7 +12,6 @@ pub trait WebsiteRoutes { impl WebsiteRoutes for Router { fn add_website_routes(self) -> Self { - self.route("/*path", get(file_handler)) - .route("/", get(index_handler)) + self.route("/{*path}", get(file_handler)).route("/", get(index_handler)) } }