Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-30 09:30:00 -07:00
Code Issues Packages Projects Releases Wiki Activity

server: cleanup

Browse Source
This commit is contained in:
nym21
2025-02-13 11:10:34 +01:00
parent b034b4fe2f
commit 443a32dc81
19 changed files with 244 additions and 559 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
server/src/api/explorer/mod.rs Normal file
View File

166
server/src/api/mod.rs
View File

@@ -1,26 +1,11 @@
use std::time::Instant;
use axum::{
extract::{Query, State},
http::HeaderMap,
response::{IntoResponse, Response},
routing::get,
Json, Router,
};
use color_eyre::eyre::eyre;
use reqwest::StatusCode;
use serde::Deserialize;
use serde_json::Value;
use structs::{Format, Index};
use crate::{log_result, traits::HeaderMapExtended};
use axum::{routing::get, Router};
use super::AppState;
// mod handlers;
pub mod structs;
mod explorer;
mod vecs;
pub const VECS_URL_PREFIX: &str = "/api/vecs";
pub use vecs::VecIdToIndexToVec;
pub trait ApiRoutes {
fn add_api_routes(self) -> Self;
@@ -28,147 +13,6 @@ pub trait ApiRoutes {
impl ApiRoutes for Router<AppState> {
fn add_api_routes(self) -> Self {
self.route(VECS_URL_PREFIX, get(handler))
self.route("/api/vecs", get(vecs::handler))
}
}
#[derive(Debug, Deserialize)]
pub struct DatasetParams {
pub i: String,
pub v: String,
pub from: Option<i64>,
pub to: Option<i64>,
pub format: Option<String>,
}
pub async fn handler(headers: HeaderMap, query: Query<DatasetParams>, State(app_state): State<AppState>) -> Response {
let instant = Instant::now();
let path = format!(
"{VECS_URL_PREFIX}?i={}&v={}{}{}",
query.i,
query.v,
query.from.map_or("".to_string(), |from| format!("&from={from}")),
query.to.map_or("".to_string(), |to| format!("&to={to}")),
);
match req_to_response_res(headers, query, app_state) {
Ok(response) => {
log_result(response.status(), &path, instant);
response
}
Err(error) => {
let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
log_result(response.status(), &path, instant);
response.headers_mut().insert_cors();
response
}
}
}
fn req_to_response_res(
headers: HeaderMap,
Query(DatasetParams { format, from, i, to, v }): Query<DatasetParams>,
AppState { vecs, .. }: AppState,
) -> color_eyre::Result<Response> {
let format = Format::try_from(format).ok();
let indexes = i
.to_lowercase()
.split(",")
.flat_map(|s| Index::try_from(s).ok())
.collect::<Vec<_>>();
if indexes.len() > 1 {
return Err(eyre!("Multiple indexes aren't supported"));
} else if indexes.is_empty() {
return Err(eyre!("Unknown index"));
}
let ids = v
.to_lowercase()
.split(",")
.map(|s| (s.to_owned(), vecs.get(&s.replace("_", "-"))))
.filter(|(_, opt)| opt.is_some())
.map(|(id, vec)| (id, vec.unwrap()))
.collect::<Vec<_>>();
if ids.is_empty() {
return Ok(Json(()).into_response());
}
let values = ids
.iter()
.flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap()))
.map(|vec| -> storable_vec::Result<Vec<Value>> { vec.collect_range_values(from, to) })
.collect::<storable_vec::Result<Vec<_>>>()?;
if ids.is_empty() {
return Ok(Json(()).into_response());
}
let ids_last_i = ids.len() - 1;
let mut response = match format {
Some(Format::CSV) | Some(Format::TSV) => {
let delimiter = if format == Some(Format::CSV) { ',' } else { '\t' };
let mut csv = ids
.into_iter()
.map(|(id, _)| id)
.collect::<Vec<_>>()
.join(&delimiter.to_string());
csv.push('\n');
let values_len = values.first().unwrap().len();
(0..values_len).for_each(|i| {
let mut line = "".to_string();
values.iter().enumerate().for_each(|(id_i, v)| {
line += &v.get(i).unwrap().to_string();
if id_i == ids_last_i {
line.push('\n');
} else {
line.push(delimiter);
}
});
csv += &line;
});
csv.into_response()
}
Some(Format::JSON) | None => {
if values.len() == 1 {
let values = values.first().unwrap();
if values.len() == 1 {
let value = values.first().unwrap();
Json(value).into_response()
} else {
Json(values).into_response()
}
} else {
Json(values).into_response()
}
}
};
let headers = response.headers_mut();
headers.insert_cors();
// headers.insert_last_modified(date_modified);
match format {
Some(format) => {
headers.insert_content_disposition_attachment();
match format {
Format::CSV => headers.insert_content_type_text_csv(),
Format::TSV => headers.insert_content_type_text_tsv(),
Format::JSON => headers.insert_content_type_application_json(),
}
}
_ => headers.insert_content_type_application_json(),
};
Ok(response)
}

15
server/src/api/structs/mod.rs
View File

@@ -1,15 +0,0 @@
// mod chunk_metadata;
mod format;
mod index;
// mod kind;
// mod range;
// mod route;
// mod routes;
// pub use chunk_metadata::*;
pub use format::*;
pub use index::*;
// pub use kind::*;
// pub use range::*;
// pub use route::*;
// pub use routes::*;

0
server/src/api/structs/format.rs → server/src/api/vecs/format.rs
View File

38
server/src/api/vecs/id_to_i_to_vec.rs Normal file
View File

@@ -0,0 +1,38 @@
use std::collections::BTreeMap;
use derive_deref::{Deref, DerefMut};
use storable_vec::AnyJsonStorableVec;
use super::index::Index;
#[derive(Default, Deref, DerefMut)]
pub struct VecIdToIndexToVec(BTreeMap<String, IndexToVec>);
impl VecIdToIndexToVec {
// Not the most performant or type safe but only built once so that's okay
pub fn insert(&mut self, vec: &'static dyn AnyJsonStorableVec) {
let file_name = vec.file_name();
let split = file_name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
panic!();
}
let str = vec.index_type_to_string().split("::").last().unwrap().to_lowercase();
let index = Index::try_from(str.as_str())
.inspect_err(|_| {
dbg!(str);
})
.unwrap();
if split[0] != index.to_string().to_lowercase() {
dbg!(split[0], index.to_string());
panic!();
}
let key = split[1].to_string().replace("_", "-");
let prev = self.entry(key).or_default().insert(index, vec);
if prev.is_some() {
panic!()
}
}
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec(BTreeMap<Index, &'static dyn AnyJsonStorableVec>);

0
server/src/api/structs/index.rs → server/src/api/vecs/index.rs
View File

156
server/src/api/vecs/mod.rs Normal file
View File

@@ -0,0 +1,156 @@
use std::time::Instant;
use axum::{
extract::{Query, State},
http::{HeaderMap, Uri},
response::{IntoResponse, Response},
Json,
};
use color_eyre::eyre::eyre;
use reqwest::StatusCode;
use serde_json::Value;
use crate::{log_result, traits::HeaderMapExtended};
use super::AppState;
mod format;
mod id_to_i_to_vec;
mod index;
mod query;
use format::Format;
pub use id_to_i_to_vec::*;
use index::Index;
use query::QueryS;
pub async fn handler(
headers: HeaderMap,
uri: Uri,
query: Query<QueryS>,
State(app_state): State<AppState>,
) -> Response {
let instant = Instant::now();
let path = uri.path();
match req_to_response_res(headers, query, app_state) {
Ok(response) => {
log_result(response.status(), path, instant);
response
}
Err(error) => {
let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
log_result(response.status(), path, instant);
response.headers_mut().insert_cors();
response
}
}
}
fn req_to_response_res(
headers: HeaderMap,
Query(QueryS { format, from, i, to, v }): Query<QueryS>,
AppState { vecs, .. }: AppState,
) -> color_eyre::Result<Response> {
let format = Format::try_from(format).ok();
let indexes = i
.to_lowercase()
.split(",")
.flat_map(|s| Index::try_from(s).ok())
.collect::<Vec<_>>();
if indexes.len() > 1 {
return Err(eyre!("Multiple indexes aren't supported"));
} else if indexes.is_empty() {
return Err(eyre!("Unknown index"));
}
let ids = v
.to_lowercase()
.split(",")
.map(|s| (s.to_owned(), vecs.get(&s.replace("_", "-"))))
.filter(|(_, opt)| opt.is_some())
.map(|(id, vec)| (id, vec.unwrap()))
.collect::<Vec<_>>();
if ids.is_empty() {
return Ok(Json(()).into_response());
}
let values = ids
.iter()
.flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap()))
.map(|vec| -> storable_vec::Result<Vec<Value>> { vec.collect_range_values(from, to) })
.collect::<storable_vec::Result<Vec<_>>>()?;
if ids.is_empty() {
return Ok(Json(()).into_response());
}
let ids_last_i = ids.len() - 1;
let mut response = match format {
Some(Format::CSV) | Some(Format::TSV) => {
let delimiter = if format == Some(Format::CSV) { ',' } else { '\t' };
let mut csv = ids
.into_iter()
.map(|(id, _)| id)
.collect::<Vec<_>>()
.join(&delimiter.to_string());
csv.push('\n');
let values_len = values.first().unwrap().len();
(0..values_len).for_each(|i| {
let mut line = "".to_string();
values.iter().enumerate().for_each(|(id_i, v)| {
line += &v.get(i).unwrap().to_string();
if id_i == ids_last_i {
line.push('\n');
} else {
line.push(delimiter);
}
});
csv += &line;
});
csv.into_response()
}
Some(Format::JSON) | None => {
if values.len() == 1 {
let values = values.first().unwrap();
if values.len() == 1 {
let value = values.first().unwrap();
Json(value).into_response()
} else {
Json(values).into_response()
}
} else {
Json(values).into_response()
}
}
};
let headers = response.headers_mut();
headers.insert_cors();
// headers.insert_last_modified(date_modified);
match format {
Some(format) => {
headers.insert_content_disposition_attachment();
match format {
Format::CSV => headers.insert_content_type_text_csv(),
Format::TSV => headers.insert_content_type_text_tsv(),
Format::JSON => headers.insert_content_type_application_json(),
}
}
_ => headers.insert_content_type_application_json(),
};
Ok(response)
}

10
server/src/api/vecs/query.rs Normal file
View File

@@ -0,0 +1,10 @@
use serde::Deserialize;
#[derive(Debug, Deserialize)]
pub struct QueryS {
pub i: String,
pub v: String,
pub from: Option<i64>,
pub to: Option<i64>,
pub format: Option<String>,
}

2
server/src/website/handlers/file.rs → server/src/files/file.rs
View File

@@ -18,7 +18,7 @@ use crate::{
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
};
use super::minify_js;
use super::minify::minify_js;
const WEBSITE_DEV_PATH: &str = "../website/";

0
server/src/website/handlers/minify.rs → server/src/files/minify.rs
View File

13
server/src/website/mod.rs → server/src/files/mod.rs
View File

@@ -1,16 +1,17 @@
use axum::{routing::get, Router};
mod handlers;
use handlers::{file_handler, index_handler};
use super::AppState;
pub trait WebsiteRoutes {
mod file;
mod minify;
use file::{file_handler, index_handler};
pub trait FilesRoutes {
fn add_website_routes(self) -> Self;
}
impl WebsiteRoutes for Router<AppState> {
impl FilesRoutes for Router<AppState> {
fn add_website_routes(self) -> Self {
self.route("/{*path}", get(file_handler)).route("/", get(index_handler))
}

45
server/src/lib.rs
View File

@@ -1,21 +1,20 @@
use std::{collections::BTreeMap, time::Instant};
use std::time::Instant;
use api::{structs::Index, ApiRoutes};
use api::{ApiRoutes, VecIdToIndexToVec};
use axum::{routing::get, serve, Json, Router};
use color_eyre::owo_colors::OwoColorize;
use computer::Computer;
use derive_deref::{Deref, DerefMut};
use files::FilesRoutes;
use indexer::Indexer;
use logger::{error, info};
use reqwest::StatusCode;
use storable_vec::{AnyJsonStorableVec, STATELESS};
use storable_vec::STATELESS;
use tokio::net::TcpListener;
use tower_http::compression::CompressionLayer;
use website::WebsiteRoutes;
mod api;
mod files;
mod traits;
mod website;
#[derive(Clone)]
pub struct AppState {
@@ -24,40 +23,6 @@ pub struct AppState {
computer: &'static Computer<STATELESS>,
}
#[derive(Default, Deref, DerefMut)]
pub struct VecIdToIndexToVec(BTreeMap<String, IndexToVec>);
impl VecIdToIndexToVec {
// Not the most performant or type safe but only built once so that's okay
pub fn insert(&mut self, vec: &'static dyn AnyJsonStorableVec) {
let file_name = vec.file_name();
let split = file_name.split("_to_").collect::<Vec<_>>();
if split.len() != 2 {
panic!();
}
let str = vec.index_type_to_string().split("::").last().unwrap().to_lowercase();
let index = Index::try_from(str.as_str())
.inspect_err(|_| {
dbg!(str);
})
.unwrap();
if split[0] != index.to_string().to_lowercase() {
dbg!(split[0], index.to_string());
panic!();
}
let key = split[1].to_string().replace("_", "-");
let prev = self.entry(key).or_default().insert(index, vec);
if prev.is_some() {
panic!()
}
}
}
#[derive(Default, Deref, DerefMut)]
pub struct IndexToVec {
pub index_to_vec: BTreeMap<Index, &'static dyn AnyJsonStorableVec>,
}
pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) -> color_eyre::Result<()> {
// pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> {
// routes.generate_dts_file();

5
server/src/website/handlers/mod.rs
View File

@@ -1,5 +0,0 @@
mod file;
mod minify;
pub use file::*;
use minify::*;