mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-05-27 01:54:47 -07:00
global: snapshot
This commit is contained in:
nym21
@@ -1,214 +0,0 @@
|
||||
use std::{collections::BTreeMap, path::PathBuf};
|
||||
|
||||
use axum::{
|
||||
extract::{Path, Query, State},
|
||||
http::HeaderMap,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use color_eyre::{eyre::eyre, owo_colors::OwoColorize};
|
||||
use reqwest::StatusCode;
|
||||
use serde::Deserialize;
|
||||
|
||||
use parser::{
|
||||
log, Date, DateMap, Height, HeightMap, Json, MapChunkId, COMPRESSED_BIN_EXTENSION,
|
||||
HEIGHT_MAP_CHUNK_SIZE, JSON_EXTENSION, OHLC,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
api::structs::{Chunk, Kind, Route},
|
||||
header_map::HeaderMapUtils,
|
||||
AppState,
|
||||
};
|
||||
|
||||
use super::{
|
||||
extension::Extension,
|
||||
response::{typed_value_to_response, value_to_response},
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Params {
|
||||
chunk: Option<usize>,
|
||||
all: Option<bool>,
|
||||
}
|
||||
|
||||
pub async fn dataset_handler(
|
||||
headers: HeaderMap,
|
||||
path: Path<String>,
|
||||
query: Query<Params>,
|
||||
State(app_state): State<AppState>,
|
||||
) -> Response {
|
||||
match _dataset_handler(headers, path, query, app_state) {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
let mut response =
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
|
||||
|
||||
response.headers_mut().insert_cors();
|
||||
|
||||
response
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DATE_PREFIX: &str = "date-to-";
|
||||
const HEIGHT_PREFIX: &str = "height-to-";
|
||||
|
||||
fn _dataset_handler(
|
||||
headers: HeaderMap,
|
||||
Path(path): Path<String>,
|
||||
query: Query<Params>,
|
||||
AppState { routes }: AppState,
|
||||
) -> color_eyre::Result<Response> {
|
||||
if query.chunk.is_some() && query.all.is_some() {
|
||||
return Err(eyre!("chunk and all are exclusive"));
|
||||
}
|
||||
|
||||
log(&format!(
|
||||
"{}{}{}",
|
||||
path,
|
||||
query.chunk.map_or("".to_string(), |chunk| format!(
|
||||
"{}{chunk}",
|
||||
"?chunk=".bright_black()
|
||||
)),
|
||||
query.all.map_or("".to_string(), |all| format!(
|
||||
"{}{all}",
|
||||
"?all=".bright_black()
|
||||
))
|
||||
));
|
||||
|
||||
let (kind, id, route) = if path.starts_with(DATE_PREFIX) {
|
||||
let id = convert_path_to_id(path.strip_prefix(DATE_PREFIX).unwrap());
|
||||
let route = routes.date.get(&id);
|
||||
(Kind::Date, id, route)
|
||||
} else if path.starts_with(HEIGHT_PREFIX) {
|
||||
let id = convert_path_to_id(path.strip_prefix(HEIGHT_PREFIX).unwrap());
|
||||
let route = routes.height.get(&id);
|
||||
(Kind::Height, id, route)
|
||||
} else {
|
||||
let id = convert_path_to_id(&path);
|
||||
let route = routes.last.get(&id);
|
||||
(Kind::Last, id, route)
|
||||
};
|
||||
|
||||
if route.is_none() {
|
||||
return Err(eyre!("Path error"));
|
||||
}
|
||||
|
||||
let mut route = route.unwrap().to_owned();
|
||||
|
||||
let mut chunk = None;
|
||||
|
||||
if query.all.map_or(true, |b| !b) {
|
||||
match kind {
|
||||
Kind::Date => {
|
||||
let datasets = DateMap::<usize>::_read_dir(&route.file_path, &route.serialization);
|
||||
|
||||
process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
|
||||
}
|
||||
Kind::Height => {
|
||||
let datasets =
|
||||
HeightMap::<usize>::_read_dir(&route.file_path, &route.serialization);
|
||||
|
||||
process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?;
|
||||
}
|
||||
Kind::Last => {
|
||||
if !route.values_type.ends_with("Value") {
|
||||
route.file_path.set_extension(COMPRESSED_BIN_EXTENSION);
|
||||
} else {
|
||||
route.file_path.set_extension(JSON_EXTENSION);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (date, response) = headers.check_if_modified_since(&route.file_path).unwrap();
|
||||
|
||||
if let Some(response) = response {
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let type_name = route.values_type.split("::").last().unwrap();
|
||||
|
||||
let extension = Extension::from(&std::path::PathBuf::from(&path));
|
||||
|
||||
let mut response = match type_name {
|
||||
"u8" => typed_value_to_response::<u8>(kind, &route, chunk, id, extension)?,
|
||||
"u16" => typed_value_to_response::<u16>(kind, &route, chunk, id, extension)?,
|
||||
"u32" => typed_value_to_response::<u32>(kind, &route, chunk, id, extension)?,
|
||||
"u64" => typed_value_to_response::<u64>(kind, &route, chunk, id, extension)?,
|
||||
"usize" => typed_value_to_response::<usize>(kind, &route, chunk, id, extension)?,
|
||||
"f32" => typed_value_to_response::<f32>(kind, &route, chunk, id, extension)?,
|
||||
"f64" => typed_value_to_response::<f64>(kind, &route, chunk, id, extension)?,
|
||||
"OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk, id, extension)?,
|
||||
"Date" => typed_value_to_response::<Date>(kind, &route, chunk, id, extension)?,
|
||||
"Height" => typed_value_to_response::<Height>(kind, &route, chunk, id, extension)?,
|
||||
"Value" => {
|
||||
value_to_response::<serde_json::Value>(Json::import(&route.file_path)?, extension)
|
||||
}
|
||||
_ => panic!("Incompatible type: {type_name}"),
|
||||
};
|
||||
|
||||
let headers = response.headers_mut();
|
||||
headers.insert_last_modified(date);
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn convert_path_to_id(s: &str) -> String {
|
||||
Extension::remove_extension(s).replace('-', "_")
|
||||
}
|
||||
|
||||
fn process_datasets<ChunkId>(
|
||||
headers: &HeaderMap,
|
||||
kind: Kind,
|
||||
chunk: &mut Option<Chunk>,
|
||||
route: &mut Route,
|
||||
query: Query<Params>,
|
||||
datasets: BTreeMap<ChunkId, PathBuf>,
|
||||
) -> color_eyre::Result<()>
|
||||
where
|
||||
ChunkId: MapChunkId,
|
||||
{
|
||||
let (last_chunk_id, _) = datasets.last_key_value().unwrap_or_else(|| {
|
||||
dbg!(&datasets, &route);
|
||||
panic!()
|
||||
});
|
||||
|
||||
let chunk_id = query
|
||||
.chunk
|
||||
.map(|id| ChunkId::from_usize(id))
|
||||
.unwrap_or(*last_chunk_id);
|
||||
|
||||
let path = datasets.get(&chunk_id);
|
||||
|
||||
if path.is_none() {
|
||||
return Err(eyre!("Couldn't find chunk"));
|
||||
}
|
||||
|
||||
let path = path.unwrap();
|
||||
route.file_path = path.clone();
|
||||
|
||||
let offset = match kind {
|
||||
Kind::Date => 1,
|
||||
Kind::Height => HEIGHT_MAP_CHUNK_SIZE as usize,
|
||||
_ => panic!(),
|
||||
};
|
||||
|
||||
let offsetted_to_url = |offseted| {
|
||||
datasets.get(&ChunkId::from_usize(offseted)).map(|_| {
|
||||
let scheme = headers.get_scheme();
|
||||
let host = headers.get_host();
|
||||
format!("{scheme}://{host}/api/{}?chunk={offseted}", route.url_path)
|
||||
})
|
||||
};
|
||||
|
||||
let chunk_id = chunk_id.to_usize();
|
||||
|
||||
chunk.replace(Chunk {
|
||||
id: chunk_id,
|
||||
next: chunk_id.checked_add(offset).and_then(offsetted_to_url),
|
||||
previous: chunk_id.checked_sub(offset).and_then(offsetted_to_url),
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub enum Extension {
|
||||
#[allow(clippy::upper_case_acronyms)]
|
||||
CSV,
|
||||
#[allow(clippy::upper_case_acronyms)]
|
||||
JSON,
|
||||
}
|
||||
|
||||
impl Extension {
|
||||
pub fn from(path: &Path) -> Option<Self> {
|
||||
if let Some(extension) = path.extension() {
|
||||
let extension = extension.to_str().unwrap();
|
||||
|
||||
if extension == Self::CSV.to_str() {
|
||||
Some(Self::CSV)
|
||||
} else if extension == Self::JSON.to_str() {
|
||||
Some(Self::JSON)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_str(&self) -> &str {
|
||||
match self {
|
||||
Extension::CSV => "csv",
|
||||
Extension::JSON => "json",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_dot_str(&self) -> String {
|
||||
format!(".{}", self.to_str())
|
||||
}
|
||||
|
||||
pub fn remove_extension(s: &str) -> String {
|
||||
s.replace(&Self::CSV.to_dot_str(), "")
|
||||
.replace(&Self::JSON.to_dot_str(), "")
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
use axum::{extract::State, http::HeaderMap, response::Response};
|
||||
use reqwest::header::HOST;
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
use super::response::{generic_to_reponse, update_reponse_headers};
|
||||
|
||||
pub async fn fallback(headers: HeaderMap, State(app_state): State<AppState>) -> Response {
|
||||
update_reponse_headers(
|
||||
generic_to_reponse(
|
||||
app_state
|
||||
.routes
|
||||
.to_full_paths(headers[HOST].to_str().unwrap().to_string()),
|
||||
None,
|
||||
),
|
||||
60,
|
||||
None,
|
||||
)
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
mod dataset;
|
||||
mod extension;
|
||||
mod fallback;
|
||||
|
||||
mod response;
|
||||
|
||||
pub use dataset::*;
|
||||
pub use fallback::*;
|
||||
@@ -1,165 +0,0 @@
|
||||
use std::fmt::Debug;
|
||||
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use bincode::Decode;
|
||||
use parser::{Date, MapValue, SerializedBTreeMap, SerializedVec};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{
|
||||
api::structs::{Chunk, Kind, Route},
|
||||
header_map::HeaderMapUtils,
|
||||
};
|
||||
|
||||
use super::extension::Extension;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct WrappedDataset<'a, T>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
source: &'a str,
|
||||
chunk: Chunk,
|
||||
dataset: T,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct WrappedValue<T>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
value: T,
|
||||
}
|
||||
|
||||
pub fn typed_value_to_response<T>(
|
||||
kind: Kind,
|
||||
route: &Route,
|
||||
chunk: Option<Chunk>,
|
||||
id: String,
|
||||
extension: Option<Extension>,
|
||||
) -> color_eyre::Result<Response>
|
||||
where
|
||||
T: Serialize + Debug + DeserializeOwned + Decode + MapValue,
|
||||
{
|
||||
Ok(match kind {
|
||||
Kind::Date => {
|
||||
let dataset = if chunk.is_some() {
|
||||
route
|
||||
.serialization
|
||||
.import::<SerializedBTreeMap<Date, T>>(&route.file_path)?
|
||||
} else {
|
||||
SerializedBTreeMap::<Date, T>::import_all(&route.file_path, &route.serialization)
|
||||
};
|
||||
|
||||
if extension == Some(Extension::CSV) {
|
||||
let mut csv = format!("date,{}\n", id);
|
||||
|
||||
dataset.map.iter().for_each(|(k, v)| {
|
||||
csv += &format!("{},{:?}\n", k, v);
|
||||
});
|
||||
|
||||
string_to_response(csv, extension)
|
||||
} else {
|
||||
dataset_to_response(dataset, chunk, extension)
|
||||
}
|
||||
}
|
||||
Kind::Height => {
|
||||
let dataset = if chunk.is_some() {
|
||||
route
|
||||
.serialization
|
||||
.import::<SerializedVec<T>>(&route.file_path)?
|
||||
} else {
|
||||
SerializedVec::<T>::import_all(&route.file_path, &route.serialization)
|
||||
};
|
||||
|
||||
if extension == Some(Extension::CSV) {
|
||||
let mut csv = format!("height,{}\n", id);
|
||||
|
||||
let starting_height = chunk.map_or(0, |chunk| chunk.id);
|
||||
|
||||
dataset.map.iter().enumerate().for_each(|(k, v)| {
|
||||
csv += &format!("{},{:?}\n", starting_height + k, v);
|
||||
});
|
||||
|
||||
string_to_response(csv, extension)
|
||||
} else {
|
||||
dataset_to_response(dataset, chunk, extension)
|
||||
}
|
||||
}
|
||||
Kind::Last => {
|
||||
let value = route.serialization.import::<T>(&route.file_path)?;
|
||||
|
||||
if extension == Some(Extension::JSON) {
|
||||
value_to_response(WrappedValue { value }, extension)
|
||||
} else {
|
||||
value_to_response(value, extension)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn string_to_response(s: String, extension: Option<Extension>) -> Response {
|
||||
update_reponse_headers(s.into_response(), 5, extension)
|
||||
}
|
||||
|
||||
pub fn value_to_response<T>(value: T, extension: Option<Extension>) -> Response
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
update_reponse_headers(generic_to_reponse(value, None), 1, extension)
|
||||
}
|
||||
|
||||
fn dataset_to_response<T>(
|
||||
dataset: T,
|
||||
chunk: Option<Chunk>,
|
||||
extension: Option<Extension>,
|
||||
) -> Response
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
update_reponse_headers(generic_to_reponse(dataset, chunk), 5, extension)
|
||||
}
|
||||
|
||||
pub fn generic_to_reponse<T>(generic: T, chunk: Option<Chunk>) -> Response
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
if let Some(chunk) = chunk {
|
||||
Json(WrappedDataset {
|
||||
source: "https://kibo.money",
|
||||
chunk,
|
||||
dataset: generic,
|
||||
})
|
||||
.into_response()
|
||||
} else {
|
||||
Json(generic).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_reponse_headers(
|
||||
mut response: Response,
|
||||
cache_time: u64,
|
||||
extension: Option<Extension>,
|
||||
) -> Response {
|
||||
let headers = response.headers_mut();
|
||||
|
||||
let max_age = cache_time;
|
||||
let stale_while_revalidate = max_age;
|
||||
|
||||
headers.insert_cors();
|
||||
headers.insert_cache_control_revalidate(max_age, stale_while_revalidate);
|
||||
|
||||
match extension {
|
||||
Some(extension) => {
|
||||
headers.insert_content_disposition_attachment();
|
||||
|
||||
match extension {
|
||||
Extension::CSV => headers.insert_content_type_text_csv(),
|
||||
Extension::JSON => headers.insert_content_type_application_json(),
|
||||
}
|
||||
}
|
||||
_ => headers.insert_content_type_application_json(),
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
use axum::{routing::get, Router};
|
||||
use handlers::{dataset_handler, fallback};
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
mod handlers;
|
||||
pub mod structs;
|
||||
|
||||
pub trait ApiRoutes {
|
||||
fn add_api_routes(self) -> Self;
|
||||
}
|
||||
|
||||
impl ApiRoutes for Router<AppState> {
|
||||
fn add_api_routes(self) -> Self {
|
||||
self.route("/api/*path", get(dataset_handler))
|
||||
.route("/api/", get(fallback))
|
||||
.route("/api", get(fallback))
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct Chunk {
|
||||
pub id: usize,
|
||||
pub previous: Option<String>,
|
||||
pub next: Option<String>,
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
#[derive(PartialEq, Eq, Clone, Copy)]
|
||||
pub enum Kind {
|
||||
Date,
|
||||
Height,
|
||||
Last,
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
mod chunk;
|
||||
mod kind;
|
||||
mod paths;
|
||||
mod routes;
|
||||
|
||||
pub use chunk::*;
|
||||
pub use kind::*;
|
||||
pub use paths::*;
|
||||
pub use routes::*;
|
||||
@@ -1,9 +0,0 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::Grouped;
|
||||
|
||||
#[derive(Clone, Default, Deref, DerefMut, Debug, Serialize)]
|
||||
pub struct Paths(pub Grouped<BTreeMap<String, String>>);
|
||||
@@ -1,157 +0,0 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use itertools::Itertools;
|
||||
use parser::{Json, Serialization};
|
||||
|
||||
use crate::Grouped;
|
||||
|
||||
use super::Paths;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Route {
|
||||
pub url_path: String,
|
||||
pub file_path: PathBuf,
|
||||
pub values_type: String,
|
||||
pub serialization: Serialization,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deref, DerefMut)]
|
||||
pub struct Routes(pub Grouped<HashMap<String, Route>>);
|
||||
|
||||
const INPUTS_PATH: &str = "./in";
|
||||
const WEBSITE_TYPES_PATH: &str = "../website/scripts/types";
|
||||
|
||||
impl Routes {
|
||||
pub fn build() -> Self {
|
||||
let path_to_type: BTreeMap<String, String> =
|
||||
Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap();
|
||||
|
||||
let mut routes = Routes::default();
|
||||
|
||||
path_to_type.into_iter().for_each(|(key, value)| {
|
||||
let mut split_key = key.split('/').collect_vec();
|
||||
let last = split_key.pop().unwrap().to_owned();
|
||||
|
||||
let mut skip = 2;
|
||||
|
||||
let mut serialization = Serialization::Binary;
|
||||
|
||||
if *split_key.get(1).unwrap() == "price" {
|
||||
skip = 1;
|
||||
serialization = Serialization::Json;
|
||||
}
|
||||
|
||||
let mut split_key = split_key.iter().skip(skip).collect_vec();
|
||||
|
||||
// Use case for: "../datasets/last": "Value",
|
||||
if split_key.is_empty() {
|
||||
split_key.push(&"last");
|
||||
}
|
||||
|
||||
let map_key = split_key.iter().join("_");
|
||||
|
||||
let url_path = split_key.iter().join("-");
|
||||
|
||||
let file_path = PathBuf::from(key.to_owned());
|
||||
let values_type = value.to_owned();
|
||||
|
||||
if last == "date" {
|
||||
routes.date.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path: format!("date-to-{url_path}"),
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else if last == "height" {
|
||||
routes.height.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path: format!("height-to-{url_path}"),
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else if last == "last" {
|
||||
routes.last.insert(
|
||||
map_key,
|
||||
Route {
|
||||
url_path,
|
||||
file_path,
|
||||
values_type,
|
||||
serialization,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
dbg!(&key, value, &last);
|
||||
panic!("")
|
||||
}
|
||||
});
|
||||
|
||||
routes
|
||||
}
|
||||
|
||||
pub fn generate_dts_file(&self) {
|
||||
let map_to_type = |name: &str, map: &HashMap<String, Route>| -> String {
|
||||
let paths = map
|
||||
.values()
|
||||
.map(|route| format!("\"{}\"", route.url_path))
|
||||
.join(" | ");
|
||||
|
||||
format!("export type {}Path = {};\n", name, paths)
|
||||
};
|
||||
|
||||
let date_type = map_to_type("Date", &self.date);
|
||||
|
||||
let height_type = map_to_type("Height", &self.height);
|
||||
|
||||
let last_type = map_to_type("Last", &self.last);
|
||||
|
||||
fs::write(
|
||||
format!("{WEBSITE_TYPES_PATH}/paths.d.ts"),
|
||||
format!("// This file is auto generated by the server\n// Manual changes are forbidden\n\n{date_type}\n{height_type}\n{last_type}"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub fn to_full_paths(&self, host: String) -> Paths {
|
||||
let url = {
|
||||
let scheme = if host.contains("0.0.0.0") || host.contains("localhost") {
|
||||
"http"
|
||||
} else {
|
||||
"https"
|
||||
};
|
||||
|
||||
format!("{scheme}://{host}")
|
||||
};
|
||||
|
||||
let transform = |map: &HashMap<String, Route>| -> BTreeMap<String, String> {
|
||||
map.iter()
|
||||
.map(|(key, route)| {
|
||||
(
|
||||
key.to_owned(),
|
||||
format!("{url}/api/{}", route.url_path.to_owned()),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let date_paths = transform(&self.date);
|
||||
let height_paths = transform(&self.height);
|
||||
let last_paths = transform(&self.last);
|
||||
|
||||
Paths(Grouped {
|
||||
date: date_paths,
|
||||
height: height_paths,
|
||||
last: last_paths,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{header, HeaderMap, Response},
|
||||
response::IntoResponse,
|
||||
};
|
||||
use chrono::{DateTime, Timelike, Utc};
|
||||
use parser::log;
|
||||
use reqwest::{
|
||||
header::{HOST, IF_MODIFIED_SINCE},
|
||||
StatusCode,
|
||||
};
|
||||
|
||||
const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish
|
||||
const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";
|
||||
|
||||
pub trait HeaderMapUtils {
|
||||
fn get_scheme(&self) -> &str;
|
||||
fn get_host(&self) -> &str;
|
||||
fn check_if_host_is_any_local(&self) -> bool;
|
||||
fn check_if_host_is_0000(&self) -> bool;
|
||||
fn check_if_host_is_localhost(&self) -> bool;
|
||||
|
||||
fn insert_cors(&mut self);
|
||||
|
||||
fn get_if_modified_since(&self) -> Option<DateTime<Utc>>;
|
||||
fn check_if_modified_since(
|
||||
&self,
|
||||
path: &Path,
|
||||
) -> color_eyre::Result<(DateTime<Utc>, Option<Response<Body>>)>;
|
||||
|
||||
fn insert_cache_control_immutable(&mut self);
|
||||
fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
|
||||
fn insert_last_modified(&mut self, date: DateTime<Utc>);
|
||||
|
||||
fn insert_content_disposition_attachment(&mut self);
|
||||
|
||||
fn insert_content_type(&mut self, path: &Path);
|
||||
fn insert_content_type_image_icon(&mut self);
|
||||
fn insert_content_type_image_jpeg(&mut self);
|
||||
fn insert_content_type_image_png(&mut self);
|
||||
fn insert_content_type_application_javascript(&mut self);
|
||||
fn insert_content_type_application_json(&mut self);
|
||||
fn insert_content_type_application_manifest_json(&mut self);
|
||||
fn insert_content_type_application_pdf(&mut self);
|
||||
fn insert_content_type_text_css(&mut self);
|
||||
fn insert_content_type_text_csv(&mut self);
|
||||
fn insert_content_type_text_html(&mut self);
|
||||
fn insert_content_type_text_plain(&mut self);
|
||||
fn insert_content_type_font_woff2(&mut self);
|
||||
}
|
||||
|
||||
impl HeaderMapUtils for HeaderMap {
|
||||
fn get_scheme(&self) -> &str {
|
||||
if self.check_if_host_is_any_local() {
|
||||
"http"
|
||||
} else {
|
||||
"https"
|
||||
}
|
||||
}
|
||||
|
||||
fn get_host(&self) -> &str {
|
||||
self[HOST].to_str().unwrap()
|
||||
}
|
||||
|
||||
fn check_if_host_is_any_local(&self) -> bool {
|
||||
self.check_if_host_is_localhost() || self.check_if_host_is_0000()
|
||||
}
|
||||
|
||||
fn check_if_host_is_0000(&self) -> bool {
|
||||
self.get_host().contains("0.0.0.0")
|
||||
}
|
||||
|
||||
fn check_if_host_is_localhost(&self) -> bool {
|
||||
self.get_host().contains("localhost")
|
||||
}
|
||||
|
||||
fn insert_cors(&mut self) {
|
||||
self.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
|
||||
self.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_cache_control_immutable(&mut self) {
|
||||
self.insert(
|
||||
header::CACHE_CONTROL,
|
||||
format!("public, max-age=604800, immutable, stale-if-error={STALE_IF_ERROR}")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_content_disposition_attachment(&mut self) {
|
||||
self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
|
||||
self.insert(
|
||||
header::CACHE_CONTROL,
|
||||
format!(
|
||||
"public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_last_modified(&mut self, date: DateTime<Utc>) {
|
||||
let formatted = date.format(MODIFIED_SINCE_FORMAT).to_string();
|
||||
|
||||
self.insert(header::LAST_MODIFIED, formatted.parse().unwrap());
|
||||
}
|
||||
|
||||
fn check_if_modified_since(
|
||||
&self,
|
||||
path: &Path,
|
||||
) -> color_eyre::Result<(DateTime<Utc>, Option<Response<Body>>)> {
|
||||
let time = path.metadata()?.modified()?;
|
||||
let date: DateTime<Utc> = time.into();
|
||||
let date = date.with_nanosecond(0).unwrap();
|
||||
let mut response_opt = None;
|
||||
|
||||
if let Some(if_modified_since) = self.get_if_modified_since() {
|
||||
if if_modified_since == date {
|
||||
let mut response = (StatusCode::NOT_MODIFIED, "").into_response();
|
||||
let headers = response.headers_mut();
|
||||
headers.insert_cors();
|
||||
response_opt.replace(response);
|
||||
}
|
||||
}
|
||||
|
||||
Ok((date, response_opt))
|
||||
}
|
||||
|
||||
fn get_if_modified_since(&self) -> Option<DateTime<Utc>> {
|
||||
if let Some(modified_since) = self.get(IF_MODIFIED_SINCE) {
|
||||
if let Ok(modified_since) = modified_since.to_str() {
|
||||
let date = DateTime::parse_from_str(
|
||||
&format!("{modified_since} +00:00"),
|
||||
&format!("{MODIFIED_SINCE_FORMAT} %z"),
|
||||
);
|
||||
|
||||
if let Ok(x) = date {
|
||||
return Some(x.to_utc());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
|
||||
fn insert_content_type(&mut self, path: &Path) {
|
||||
match path.extension().unwrap().to_str().unwrap() {
|
||||
"js" => self.insert_content_type_application_javascript(),
|
||||
"json" => self.insert_content_type_application_json(),
|
||||
"html" => self.insert_content_type_text_html(),
|
||||
"css" => self.insert_content_type_text_css(),
|
||||
"toml" | "txt" => self.insert_content_type_text_plain(),
|
||||
"pdf" => self.insert_content_type_application_pdf(),
|
||||
"woff2" => self.insert_content_type_font_woff2(),
|
||||
"ico" => self.insert_content_type_image_icon(),
|
||||
"jpg" | "jpeg" => self.insert_content_type_image_jpeg(),
|
||||
"png" => self.insert_content_type_image_png(),
|
||||
"webmanifest" => self.insert_content_type_application_manifest_json(),
|
||||
extension => {
|
||||
log(&format!("Extension unsupported: {extension}"));
|
||||
panic!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_content_type_image_icon(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "image/x-icon".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_image_jpeg(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_image_png(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "image/png".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_application_javascript(&mut self) {
|
||||
self.insert(
|
||||
header::CONTENT_TYPE,
|
||||
"application/javascript".parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_content_type_application_json(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_application_manifest_json(&mut self) {
|
||||
self.insert(
|
||||
header::CONTENT_TYPE,
|
||||
"application/manifest+json".parse().unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_content_type_application_pdf(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "application/pdf".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_css(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/css".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_csv(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/csv".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_html(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/html".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_plain(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_font_woff2(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "font/woff2".parse().unwrap());
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::{structs::Routes, ApiRoutes};
|
||||
use axum::{serve, Router};
|
||||
use parser::{log, reset_logs};
|
||||
use serde::Serialize;
|
||||
use tokio::net::TcpListener;
|
||||
use tower_http::compression::CompressionLayer;
|
||||
use website::WebsiteRoutes;
|
||||
|
||||
mod api;
|
||||
mod header_map;
|
||||
mod website;
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct Grouped<T> {
|
||||
pub date: T,
|
||||
pub height: T,
|
||||
pub last: T,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
routes: Arc<Routes>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
reset_logs();
|
||||
|
||||
let routes = Routes::build();
|
||||
|
||||
routes.generate_dts_file();
|
||||
|
||||
let state = AppState {
|
||||
routes: Arc::new(routes),
|
||||
};
|
||||
|
||||
let compression_layer = CompressionLayer::new()
|
||||
.br(true)
|
||||
.deflate(true)
|
||||
.gzip(true)
|
||||
.zstd(true);
|
||||
|
||||
let router = Router::new()
|
||||
.add_api_routes()
|
||||
.add_website_routes()
|
||||
.with_state(state)
|
||||
.layer(compression_layer);
|
||||
|
||||
let mut port = 3110;
|
||||
|
||||
let mut listener;
|
||||
loop {
|
||||
listener = TcpListener::bind(format!("0.0.0.0:{port}")).await;
|
||||
if listener.is_ok() {
|
||||
break;
|
||||
}
|
||||
port += 1;
|
||||
}
|
||||
|
||||
log(&format!("Starting server on port {port}..."));
|
||||
let listener = listener.unwrap();
|
||||
|
||||
serve(listener, router).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Files are bigger than with SWC, to retest later
|
||||
|
||||
// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use oxc::{
|
||||
allocator::Allocator,
|
||||
codegen::{CodeGenerator, CodegenOptions},
|
||||
minifier::{MinifierOptions, MinifierReturn},
|
||||
parser::{Parser, ParserReturn},
|
||||
span::SourceType,
|
||||
};
|
||||
|
||||
//
|
||||
pub fn minify_js(path: &Path) -> String {
|
||||
let allocator = Allocator::default();
|
||||
|
||||
let source_type = SourceType::from_path(path).unwrap();
|
||||
|
||||
let source_text = fs::read_to_string(path).unwrap();
|
||||
|
||||
let ParserReturn { mut program, .. } =
|
||||
Parser::new(&allocator, &source_text, source_type).parse();
|
||||
|
||||
let minifier = oxc::minifier::Minifier::new(MinifierOptions::default());
|
||||
|
||||
let MinifierReturn { mangler } = minifier.build(&allocator, &mut program);
|
||||
|
||||
CodeGenerator::new()
|
||||
.with_options(CodegenOptions {
|
||||
single_quote: false,
|
||||
minify: true,
|
||||
comments: false,
|
||||
annotation_comments: false,
|
||||
source_map_path: None,
|
||||
})
|
||||
.with_mangler(mangler)
|
||||
.build(&program)
|
||||
.code
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
use std::{
|
||||
fs::{self},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract,
|
||||
http::HeaderMap,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use parser::log;
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use crate::header_map::HeaderMapUtils;
|
||||
|
||||
use super::minify_js;
|
||||
|
||||
const WEBSITE_PATH: &str = "../website/";
|
||||
|
||||
pub async fn file_handler(headers: HeaderMap, path: extract::Path<String>) -> Response {
|
||||
let mut path = path.0.replace("..", "").replace("\\", "");
|
||||
|
||||
if path.ends_with("Cargo.toml") {
|
||||
path = "../server/Cargo.toml".to_owned();
|
||||
}
|
||||
|
||||
let mut path = str_to_path(&path);
|
||||
|
||||
if !path.exists() {
|
||||
if path.extension().is_some() {
|
||||
let mut response: Response<Body> = (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"File doesn't exist".to_string(),
|
||||
)
|
||||
.into_response();
|
||||
|
||||
response.headers_mut().insert_cors();
|
||||
|
||||
return response;
|
||||
} else {
|
||||
path = str_to_path("index.html");
|
||||
}
|
||||
}
|
||||
|
||||
path_to_response(headers, &path)
|
||||
}
|
||||
|
||||
pub async fn index_handler(headers: HeaderMap) -> Response {
|
||||
path_to_response(headers, &str_to_path("index.html"))
|
||||
}
|
||||
|
||||
fn path_to_response(headers: HeaderMap, path: &Path) -> Response {
|
||||
log(&path.to_str().unwrap().replace(WEBSITE_PATH, ""));
|
||||
|
||||
let (date, response) = headers.check_if_modified_since(path).unwrap();
|
||||
|
||||
if let Some(response) = response {
|
||||
return response;
|
||||
}
|
||||
|
||||
let mut response;
|
||||
|
||||
let is_localhost = headers.check_if_host_is_localhost();
|
||||
|
||||
if !is_localhost
|
||||
&& path.extension().unwrap_or_else(|| {
|
||||
dbg!(path);
|
||||
panic!();
|
||||
}) == "js"
|
||||
{
|
||||
let content = minify_js(path);
|
||||
|
||||
response = Response::new(content.into());
|
||||
} else {
|
||||
let content = fs::read(path).unwrap_or_else(|error| {
|
||||
log(&error.to_string());
|
||||
let path = path.to_str().unwrap();
|
||||
log(&format!("Can't read file {path}"));
|
||||
panic!("")
|
||||
});
|
||||
|
||||
response = Response::new(content.into());
|
||||
}
|
||||
|
||||
let headers = response.headers_mut();
|
||||
headers.insert_cors();
|
||||
headers.insert_content_type(path);
|
||||
|
||||
if !is_localhost {
|
||||
let serialized_path = path.to_str().unwrap();
|
||||
|
||||
if serialized_path.contains("fonts/")
|
||||
|| serialized_path.contains("assets/")
|
||||
|| serialized_path.contains("packages/")
|
||||
|| path.extension().is_some_and(|extension| {
|
||||
extension == "pdf"
|
||||
|| extension == "jpg"
|
||||
|| extension == "png"
|
||||
|| extension == "woff2"
|
||||
})
|
||||
{
|
||||
headers.insert_cache_control_immutable();
|
||||
} else {
|
||||
headers.insert_cache_control_revalidate(1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
headers.insert_last_modified(date);
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
fn str_to_path(path: &str) -> PathBuf {
|
||||
PathBuf::from(&format!("{WEBSITE_PATH}{path}"))
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
// Simplified version of: https://github.com/swc-project/swc/blob/main/crates/swc/examples/minify.rs
|
||||
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use swc::{config::JsMinifyOptions, try_with_handler, JsMinifyExtras};
|
||||
use swc_common::{SourceMap, GLOBALS};
|
||||
|
||||
pub fn minify_js(path: &Path) -> String {
|
||||
let cm = Arc::<SourceMap>::default();
|
||||
|
||||
let c = swc::Compiler::new(cm.clone());
|
||||
|
||||
let output = GLOBALS
|
||||
.set(&Default::default(), || {
|
||||
try_with_handler(cm.clone(), Default::default(), |handler| {
|
||||
let fm = cm.load_file(path).expect("failed to load file");
|
||||
|
||||
c.minify(
|
||||
fm,
|
||||
handler,
|
||||
&JsMinifyOptions::default(),
|
||||
JsMinifyExtras::default(),
|
||||
)
|
||||
})
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
output.code
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
mod file;
|
||||
mod minify;
|
||||
|
||||
pub use file::*;
|
||||
use minify::*;
|
||||
@@ -1,18 +0,0 @@
|
||||
use axum::{routing::get, Router};
|
||||
|
||||
mod handlers;
|
||||
|
||||
use handlers::{file_handler, index_handler};
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
pub trait WebsiteRoutes {
|
||||
fn add_website_routes(self) -> Self;
|
||||
}
|
||||
|
||||
impl WebsiteRoutes for Router<AppState> {
|
||||
fn add_website_routes(self) -> Self {
|
||||
self.route("/*path", get(file_handler))
|
||||
.route("/", get(index_handler))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user