mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-11 15:33:33 -07:00
server: add tsv support
This commit is contained in:
nym21
Generated
+36
-38
@@ -1639,9 +1639,9 @@ checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56"
|
||||
|
||||
[[package]]
|
||||
name = "oxc"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f346cc73c834df63403e8baa7b4e7efc9c6c4dbf45eb8fc6014bdb7b2d83b12"
|
||||
checksum = "1d064edaa54040423cb94b283026dfcd679b29c26c2f070021dc268cc96fb155"
|
||||
dependencies = [
|
||||
"oxc_allocator",
|
||||
"oxc_ast",
|
||||
@@ -1682,9 +1682,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_allocator"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba82e27c94fd90c94cba7f3ec9079c6eea4d2000b57c29c7d34d100bf8c80de7"
|
||||
checksum = "417522e84d8fdc87cab7bfc95b6419757c3d2f2013b78491c0295135d3516945"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"bumpalo",
|
||||
@@ -1695,15 +1695,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ast"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96cc90207e2212fa7b3511014cfa193ef2807bb504424e16ae8aa3e5a6715a3f"
|
||||
checksum = "e2e0b6014198425ee79e1e0c71c79fc874f3a7c7007043d9ce1b353f740de488"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cow-utils",
|
||||
"nonmax",
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"oxc_allocator",
|
||||
"oxc_ast_macros",
|
||||
"oxc_estree",
|
||||
@@ -1714,9 +1712,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ast_macros"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "315ff1ec55898f4d132d7f5ac3a8777d6c05e02d25bae7195e1eb0326146b299"
|
||||
checksum = "a58764050b69c30bdf45a6f803bff212306ba9ac50277f21a231ff6e110da682"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1725,9 +1723,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_cfg"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68480b093736d6ac483afead3b5d9942bd02ec6b62e414d4265482eebffe9788"
|
||||
checksum = "621ea878d0e487215822fe61c98afaa337e79f8290352a95bd8d37c45eab0484"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"itertools",
|
||||
@@ -1740,9 +1738,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_codegen"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef6d9fd6b5953dcb9f814bf9ecd93979193269b17927a38c916afb702629f754"
|
||||
checksum = "f68faeabfca1a9ad49b26939766185e87f4d03d08d917b465b1956db7d2ac466"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
@@ -1762,9 +1760,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_data_structures"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8342a5e7cb5ea7833a16b3075abb962094443b00caa22e2b2f93d47c44caf871"
|
||||
checksum = "7f3c3ebb1b248fa1a5ea965b7398eb0533f45545c6cbe28cd581f57211413096"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"ropey",
|
||||
@@ -1772,9 +1770,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_diagnostics"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e96ced1ce8a7a2c5972a9ac69a1df801966157cdb40c09e9c39b13efecdf63c9"
|
||||
checksum = "ae69c956abfbadf5c31f08dc2b901bcefadb87a472106e13ae2b6a01135c66c2"
|
||||
dependencies = [
|
||||
"cow-utils",
|
||||
"oxc-miette",
|
||||
@@ -1782,9 +1780,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_ecmascript"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05f3d220214466200c5a5c6475b681cad6aa9e076aa0e9a290c920375993cc75"
|
||||
checksum = "b2f773d18674b21240b42a7c8bcb3522de09b8841993620286e87812d7e7866a"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
@@ -1795,9 +1793,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_estree"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd0ba1ad13334dd7d203273c4d99b0f508c0d069888b5d2c36261c70630a0317"
|
||||
checksum = "244fd8ba7157c8f12910024373d4e3665b6a35fb5947edae1d4a6c084ac8e0c3"
|
||||
|
||||
[[package]]
|
||||
name = "oxc_index"
|
||||
@@ -1807,9 +1805,9 @@ checksum = "5eca5d9726cd0a6e433debe003b7bc88b2ecad0bb6109f0cef7c55e692139a34"
|
||||
|
||||
[[package]]
|
||||
name = "oxc_mangler"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de5d581f96d85a192d1c6191ee3b476ba715f565a228c605b501be2456e49f51"
|
||||
checksum = "73839b44f57bdc707e63c3558d4da368e904960f13db1639c6d6c5e2e7e4ab8d"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"itertools",
|
||||
@@ -1823,9 +1821,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_minifier"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5137cabf5eeadec26708803c9676539cb418afab1403374d46c3ed0554d41c38"
|
||||
checksum = "ba22e1818002180efb9e023422a56d67b4baaa537deaf61d23d8ea6893ddcdd7"
|
||||
dependencies = [
|
||||
"cow-utils",
|
||||
"oxc_allocator",
|
||||
@@ -1844,9 +1842,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_parser"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd86fb907a291acf174889b545424abf2c11b3875203d363637c1835d9692176"
|
||||
checksum = "8b9d2ce3556a0ec6ee45da6936c5f04d97e6f00ef7ddca08dcc473094a21cfed"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
@@ -1867,9 +1865,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_regular_expression"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d77b2ad85545ca2f6a2d319230d35e6cbacc2734029079bc820b2176cea37dc4"
|
||||
checksum = "e0e6d43914ff6758f62cf150f12d068a737fd464735d6b83ecefc7136f65e685"
|
||||
dependencies = [
|
||||
"oxc_allocator",
|
||||
"oxc_ast_macros",
|
||||
@@ -1883,9 +1881,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_semantic"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f021acc396e980c533abc66ec3680c18dda8ff4329dc32c49b8666d48d4627a1"
|
||||
checksum = "1e08d8bb5afc64dfc05e9f3b573c9af8b23a9a24aceb2bb1634d1f131717b736"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"itertools",
|
||||
@@ -1919,9 +1917,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_span"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3492e9db0c166a6072f4da2003966f3c3dad5f6c39ba86589dc0ab85de94dbb6"
|
||||
checksum = "0ae8953d32ade84e814d154ee046f0f08b2b236e12f550545a690454ab2a102c"
|
||||
dependencies = [
|
||||
"compact_str",
|
||||
"oxc-miette",
|
||||
@@ -1932,9 +1930,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_syntax"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2945b8d56e1cf569c933812bc37277de784bba3fd92a565a640f2650801b2d5"
|
||||
checksum = "9f375deb505b7fad6ed2125fdb92ffa8d550d866ca2ab0dfd24e681b125883c1"
|
||||
dependencies = [
|
||||
"assert-unchecked",
|
||||
"bitflags",
|
||||
@@ -1953,9 +1951,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "oxc_traverse"
|
||||
version = "0.49.0"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ba02cbc4f6b98c1bb04c3a73c336657d999a75b464af91cf3abb93e610c0650"
|
||||
checksum = "2a4d049215b81048597d120851e1e797331be4f45ae2bb39ae2855c3be76fcc3"
|
||||
dependencies = [
|
||||
"compact_str",
|
||||
"itoa",
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ derive_deref = { workspace = true }
|
||||
indexer = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
logger = { workspace = true }
|
||||
oxc = { version = "0.49.0", features = ["codegen", "minifier"] }
|
||||
oxc = { version = "0.50.0", features = ["codegen", "minifier"] }
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.12", features = ["blocking", "json"] }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
use std::{fmt::Debug, path::PathBuf, time::Instant};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::{Path, Query, State},
|
||||
http::HeaderMap,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use bincode::Decode;
|
||||
use chrono::{DateTime, Utc};
|
||||
use color_eyre::eyre::{eyre, ContextCompat};
|
||||
use reqwest::StatusCode;
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
server::{
|
||||
api::{
|
||||
structs::{ChunkMetadata, DatasetRange, DatasetRangeChunk, Extension, Kind, Route, Routes},
|
||||
API_URL_PREFIX,
|
||||
},
|
||||
header_map::{HeaderMapExtended, Modified},
|
||||
log_result,
|
||||
response::ResponseExtended,
|
||||
AppState,
|
||||
},
|
||||
structs::{
|
||||
Date, GenericMap, Height, HeightMapChunkId, MapChunkId, MapKey, MapSerialized, MapValue, SerializedBTreeMap,
|
||||
SerializedDateMap, SerializedTimeMap, SerializedVec, Timestamp, OHLC,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DatasetParams {
|
||||
pub chunk: Option<usize>,
|
||||
pub all: Option<bool>,
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
pub async fn dataset_handler(
|
||||
headers: HeaderMap,
|
||||
path: Path<String>,
|
||||
query: Query<DatasetParams>,
|
||||
State(app_state): State<AppState>,
|
||||
) -> Response {
|
||||
let instant = Instant::now();
|
||||
|
||||
let ser_path = format!(
|
||||
"{API_URL_PREFIX}/{}?kind={}{}{}",
|
||||
path.0,
|
||||
query.kind,
|
||||
query.chunk.map_or("".to_string(), |chunk| format!("&chunk={chunk}")),
|
||||
query.all.map_or("".to_string(), |all| format!("&all={all}"))
|
||||
);
|
||||
|
||||
match result_handler(headers, &path, &query, app_state) {
|
||||
Ok(response) => {
|
||||
log_result(response.status(), &ser_path, instant);
|
||||
response
|
||||
}
|
||||
Err(error) => {
|
||||
let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response();
|
||||
log_result(response.status(), &ser_path, instant);
|
||||
response.headers_mut().insert_cors();
|
||||
response
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn result_handler(
|
||||
headers: HeaderMap,
|
||||
Path(path): &Path<String>,
|
||||
query: &Query<DatasetParams>,
|
||||
AppState { routes, .. }: AppState,
|
||||
) -> color_eyre::Result<Response> {
|
||||
let path_buf = PathBuf::from(&path);
|
||||
let id = path_buf.file_stem().unwrap().to_str().unwrap();
|
||||
let ext = Extension::from(&path_buf);
|
||||
|
||||
let route = routes.get(id);
|
||||
if route.is_none() {
|
||||
return Err(eyre!("Wrong path"));
|
||||
}
|
||||
let route = route.unwrap();
|
||||
|
||||
let type_name = route.type_name.as_str();
|
||||
Ok(match type_name {
|
||||
"u8" => typed_handler::<u8>(headers, id, ext, query, route, &routes)?,
|
||||
"u16" => typed_handler::<u16>(headers, id, ext, query, route, &routes)?,
|
||||
"u32" => typed_handler::<u32>(headers, id, ext, query, route, &routes)?,
|
||||
"u64" => typed_handler::<u64>(headers, id, ext, query, route, &routes)?,
|
||||
"usize" => typed_handler::<usize>(headers, id, ext, query, route, &routes)?,
|
||||
"f32" => typed_handler::<f32>(headers, id, ext, query, route, &routes)?,
|
||||
"f64" => typed_handler::<f64>(headers, id, ext, query, route, &routes)?,
|
||||
"OHLC" => typed_handler::<OHLC>(headers, id, ext, query, route, &routes)?,
|
||||
"Date" => typed_handler::<Date>(headers, id, ext, query, route, &routes)?,
|
||||
"Height" => typed_handler::<Height>(headers, id, ext, query, route, &routes)?,
|
||||
"Timestamp" => typed_handler::<Timestamp>(headers, id, ext, query, route, &routes)?,
|
||||
_ => panic!("Incompatible type: {type_name}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn typed_handler<T>(
|
||||
headers: HeaderMap,
|
||||
id: &str,
|
||||
ext: Option<Extension>,
|
||||
query: &Query<DatasetParams>,
|
||||
route: &Route,
|
||||
routes: &Routes,
|
||||
) -> color_eyre::Result<Response>
|
||||
where
|
||||
T: Serialize + Debug + DeserializeOwned + Decode + MapValue,
|
||||
{
|
||||
let kind = Kind::try_from(&query.kind)?;
|
||||
if !route.list.contains(&kind) {
|
||||
return Err(eyre!("{kind:?} not supported for this dataset"));
|
||||
}
|
||||
|
||||
let range = DatasetRange::try_from(query)?;
|
||||
|
||||
let (mut response, date_modified) = match kind {
|
||||
Kind::Last => {
|
||||
let last_value: T = route.serialization.import(&route.path.join("last"))?;
|
||||
return Ok(axum::response::Json(last_value).into_response());
|
||||
}
|
||||
Kind::Date => match read_serialized::<Date, T, _, SerializedDateMap<T>>(id, &headers, route, &range, query)? {
|
||||
ReadSerialized::DatasetAndDate((dataset, date, chunk_meta)) => {
|
||||
(serialized_to_response(dataset, id, chunk_meta, ext), date)
|
||||
}
|
||||
ReadSerialized::NotModified => return Ok(Response::new_not_modified()),
|
||||
ReadSerialized::_Phantom(_) => unreachable!(),
|
||||
},
|
||||
Kind::Height => match read_serialized::<Height, T, _, SerializedVec<T>>(id, &headers, route, &range, query)? {
|
||||
ReadSerialized::DatasetAndDate((dataset, date, chunk_meta)) => (
|
||||
serialized_to_response::<Height, T, _, SerializedVec<T>>(dataset, id, chunk_meta, ext),
|
||||
date,
|
||||
),
|
||||
ReadSerialized::NotModified => return Ok(Response::new_not_modified()),
|
||||
ReadSerialized::_Phantom(_) => unreachable!(),
|
||||
},
|
||||
Kind::Timestamp => {
|
||||
let (dataset, date, chunk_meta) =
|
||||
match read_serialized::<Height, T, _, SerializedVec<T>>(id, &headers, route, &range, query)? {
|
||||
ReadSerialized::DatasetAndDate(tuple) => tuple,
|
||||
ReadSerialized::NotModified => return Ok(Response::new_not_modified()),
|
||||
ReadSerialized::_Phantom(_) => unreachable!(),
|
||||
};
|
||||
|
||||
let (timestamp_dataset, _, _) = match read_serialized::<Height, Timestamp, _, SerializedVec<Timestamp>>(
|
||||
"timestamp",
|
||||
&headers,
|
||||
routes.get("timestamp").unwrap(),
|
||||
&range,
|
||||
query,
|
||||
)? {
|
||||
ReadSerialized::DatasetAndDate(tuple) => tuple,
|
||||
ReadSerialized::NotModified => return Ok(Response::new_not_modified()),
|
||||
ReadSerialized::_Phantom(_) => unreachable!(),
|
||||
};
|
||||
|
||||
let mut serialized_timemap: SerializedTimeMap<T> = SerializedBTreeMap::default();
|
||||
|
||||
dataset.map.into_iter().enumerate().for_each(|(index, value)| {
|
||||
serialized_timemap.map.insert(
|
||||
timestamp_dataset.get_index(index).cloned().unwrap_or(Timestamp::now()),
|
||||
value,
|
||||
);
|
||||
});
|
||||
|
||||
(
|
||||
serialized_to_response::<Timestamp, T, HeightMapChunkId, SerializedTimeMap<T>>(
|
||||
serialized_timemap,
|
||||
id,
|
||||
chunk_meta,
|
||||
ext,
|
||||
),
|
||||
date,
|
||||
)
|
||||
|
||||
// let m = read_serialized::<Height, T, _, SerializedVec<T>>(
|
||||
// id, &headers, route, &range, query,
|
||||
// )?;
|
||||
// let t = read_serialized::<Height, Timestamp, _, SerializedVec<Timestamp>>(
|
||||
// "timestamp",
|
||||
// &headers,
|
||||
// routes.get("timestamp").unwrap(),
|
||||
// &range,
|
||||
// query,
|
||||
// );
|
||||
// t
|
||||
}
|
||||
};
|
||||
|
||||
let headers = response.headers_mut();
|
||||
|
||||
headers.insert_cors();
|
||||
headers.insert_last_modified(date_modified);
|
||||
|
||||
match ext {
|
||||
Some(extension) => {
|
||||
headers.insert_content_disposition_attachment();
|
||||
match extension {
|
||||
Extension::CSV => headers.insert_content_type_text_csv(),
|
||||
Extension::JSON => headers.insert_content_type_application_json(),
|
||||
}
|
||||
}
|
||||
_ => headers.insert_content_type_application_json(),
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn serialized_to_response<Key, Value, ChunkId, Serialized>(
|
||||
dataset: Serialized,
|
||||
id: &str,
|
||||
chunk_meta: Option<ChunkMetadata>,
|
||||
ext: Option<Extension>,
|
||||
) -> Response<Body>
|
||||
where
|
||||
Key: MapKey<ChunkId>,
|
||||
Value: MapValue,
|
||||
ChunkId: MapChunkId,
|
||||
Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
{
|
||||
if ext == Some(Extension::CSV) {
|
||||
dataset.to_csv(id).into_response()
|
||||
} else if let Some(chunk) = chunk_meta {
|
||||
axum::response::Json(SerializedMapChunk {
|
||||
chunk,
|
||||
map: dataset.map(),
|
||||
version: dataset.version(),
|
||||
})
|
||||
.into_response()
|
||||
} else {
|
||||
axum::response::Json(dataset).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
enum ReadSerialized<Key, Value, ChunkId, Serialized>
|
||||
where
|
||||
Key: MapKey<ChunkId>,
|
||||
Value: MapValue,
|
||||
ChunkId: MapChunkId,
|
||||
Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
{
|
||||
DatasetAndDate((Serialized, DateTime<Utc>, Option<ChunkMetadata>)),
|
||||
NotModified,
|
||||
_Phantom((Key, Value, ChunkId)),
|
||||
}
|
||||
|
||||
fn read_serialized<Key, Value, ChunkId, Serialized>(
|
||||
id: &str,
|
||||
headers: &HeaderMap,
|
||||
route: &Route,
|
||||
range: &DatasetRange,
|
||||
query: &Query<DatasetParams>,
|
||||
) -> color_eyre::Result<ReadSerialized<Key, Value, ChunkId, Serialized>>
|
||||
where
|
||||
Key: MapKey<ChunkId>,
|
||||
Value: MapValue,
|
||||
ChunkId: MapChunkId,
|
||||
Serialized: MapSerialized<Key, Value, ChunkId>,
|
||||
{
|
||||
let folder_path = route.path.join(Key::map_name());
|
||||
let serialization = &route.serialization;
|
||||
|
||||
let date_modified;
|
||||
|
||||
let datasets = GenericMap::<Key, Value, ChunkId, Serialized>::_read_dir(&folder_path, serialization);
|
||||
|
||||
let mut chunk_meta = None;
|
||||
|
||||
let dataset = if let DatasetRange::Chunk(range_chunk) = range {
|
||||
let chunk_id = match range_chunk {
|
||||
DatasetRangeChunk::Last => *datasets.last_key_value().context("Last tuple of dataset directory")?.0,
|
||||
DatasetRangeChunk::Chunk(chunk) => ChunkId::from_usize(*chunk),
|
||||
};
|
||||
|
||||
let chunk_path = datasets.get(&chunk_id);
|
||||
if chunk_path.is_none() {
|
||||
return Err(eyre!("Couldn't find chunk"));
|
||||
}
|
||||
let chunk_path = chunk_path.unwrap();
|
||||
|
||||
let (modified, date) = headers.check_if_modified_since(chunk_path)?;
|
||||
if modified == Modified::NotModifiedSince {
|
||||
return Ok(ReadSerialized::NotModified);
|
||||
}
|
||||
date_modified = date;
|
||||
|
||||
let to_url = |chunk: Option<ChunkId>| {
|
||||
chunk.and_then(|chunk| {
|
||||
datasets.contains_key(&chunk).then(|| {
|
||||
let scheme = headers.get_scheme();
|
||||
let host = headers.get_host();
|
||||
format!(
|
||||
"{scheme}://{host}/api/{id}?kind={}&chunk={}",
|
||||
query.kind,
|
||||
chunk.to_usize()
|
||||
)
|
||||
})
|
||||
})
|
||||
};
|
||||
|
||||
chunk_meta.replace(ChunkMetadata {
|
||||
id: chunk_id.to_usize(),
|
||||
next: to_url(chunk_id.next()),
|
||||
previous: to_url(chunk_id.previous()),
|
||||
});
|
||||
|
||||
serialization.import::<Serialized>(chunk_path)?
|
||||
} else {
|
||||
let newest_file = datasets
|
||||
.values()
|
||||
.max_by(|a, b| {
|
||||
a.metadata()
|
||||
.unwrap()
|
||||
.modified()
|
||||
.unwrap()
|
||||
.cmp(&b.metadata().unwrap().modified().unwrap())
|
||||
})
|
||||
.context("Expect to find newest file")?;
|
||||
|
||||
let (modified, date) = headers.check_if_modified_since(newest_file)?;
|
||||
if modified == Modified::NotModifiedSince {
|
||||
return Ok(ReadSerialized::NotModified);
|
||||
}
|
||||
|
||||
date_modified = date;
|
||||
|
||||
Serialized::import_all(&folder_path, serialization)
|
||||
};
|
||||
|
||||
Ok(ReadSerialized::DatasetAndDate((dataset, date_modified, chunk_meta)))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SerializedMapChunk<T>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
version: u32,
|
||||
chunk: ChunkMetadata,
|
||||
map: T,
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
mod dataset;
|
||||
mod last_values;
|
||||
|
||||
pub use dataset::*;
|
||||
pub use last_values::*;
|
||||
+12
-4
@@ -110,8 +110,15 @@ fn req_to_response_res(
|
||||
let ids_last_i = ids.len() - 1;
|
||||
|
||||
let mut response = match format {
|
||||
Some(Format::CSV) => {
|
||||
let mut csv = ids.into_iter().map(|(id, _)| id).collect::<Vec<_>>().join(",");
|
||||
Some(Format::CSV) | Some(Format::TSV) => {
|
||||
let delimiter = if format == Some(Format::CSV) { ',' } else { '\t' };
|
||||
|
||||
let mut csv = ids
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect::<Vec<_>>()
|
||||
.join(&delimiter.to_string());
|
||||
|
||||
csv.push('\n');
|
||||
|
||||
let values_len = values.first().unwrap().len();
|
||||
@@ -123,7 +130,7 @@ fn req_to_response_res(
|
||||
if id_i == ids_last_i {
|
||||
line.push('\n');
|
||||
} else {
|
||||
line.push(',');
|
||||
line.push(delimiter);
|
||||
}
|
||||
});
|
||||
csv += &line;
|
||||
@@ -131,7 +138,7 @@ fn req_to_response_res(
|
||||
|
||||
csv.into_response()
|
||||
}
|
||||
_ => {
|
||||
Some(Format::JSON) | None => {
|
||||
if values.len() == 1 {
|
||||
let values = values.first().unwrap();
|
||||
if values.len() == 1 {
|
||||
@@ -156,6 +163,7 @@ fn req_to_response_res(
|
||||
headers.insert_content_disposition_attachment();
|
||||
match format {
|
||||
Format::CSV => headers.insert_content_type_text_csv(),
|
||||
Format::TSV => headers.insert_content_type_text_tsv(),
|
||||
Format::JSON => headers.insert_content_type_application_json(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use color_eyre::eyre::eyre;
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Format {
|
||||
CSV,
|
||||
TSV,
|
||||
JSON,
|
||||
}
|
||||
|
||||
@@ -15,6 +16,8 @@ impl TryFrom<Option<String>> for Format {
|
||||
let value = value.as_str();
|
||||
if value == "csv" {
|
||||
Ok(Self::CSV)
|
||||
} else if value == "tsv" {
|
||||
Ok(Self::TSV)
|
||||
} else if value == "json" {
|
||||
Ok(Self::JSON)
|
||||
} else {
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
use std::{collections::BTreeSet, path::PathBuf};
|
||||
|
||||
use crate::{io::Serialization, structs::AnyMap};
|
||||
|
||||
use super::Kind;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Route {
|
||||
pub type_name: String,
|
||||
pub list: BTreeSet<Kind>,
|
||||
pub path: PathBuf,
|
||||
pub serialization: Serialization,
|
||||
}
|
||||
|
||||
impl Route {
|
||||
pub fn update(&mut self, map: &(dyn AnyMap + Send + Sync)) {
|
||||
self.list.append(&mut BTreeSet::from(map));
|
||||
if self.serialization != map.serialization() {
|
||||
panic!("route.upate() different serialization")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&(dyn AnyMap + Send + Sync)> for Route {
|
||||
fn from(map: &(dyn AnyMap + Send + Sync)) -> Self {
|
||||
Self {
|
||||
list: BTreeSet::from(map),
|
||||
path: map.path_parent().to_owned(),
|
||||
type_name: map.type_name().split("::").last().unwrap().to_owned(),
|
||||
serialization: map.serialization(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
parser::{AnyDatasets, Datasets},
|
||||
structs::Config,
|
||||
};
|
||||
|
||||
use super::Route;
|
||||
|
||||
#[derive(Debug, Clone, Default, Deref, DerefMut)]
|
||||
pub struct Routes(BTreeMap<String, Route>);
|
||||
|
||||
const WEBSITE_TYPES_PATH: &str = "../website/scripts/types";
|
||||
|
||||
impl Routes {
|
||||
pub fn build(datasets: &Datasets, config: &Config) -> Self {
|
||||
datasets
|
||||
.to_any_dataset_vec()
|
||||
.into_iter()
|
||||
.flat_map(|dataset| dataset.to_all_map_vec())
|
||||
.fold(Self::default(), |mut routes, map| {
|
||||
routes
|
||||
.entry(map.id(config))
|
||||
.or_insert_with(|| Route::from(map))
|
||||
.update(map);
|
||||
routes
|
||||
})
|
||||
}
|
||||
|
||||
pub fn generate_dts_file(&self) {
|
||||
// let map_to_type = |name: &str, map: &HashMap<String, Route>| -> String {
|
||||
// let paths = map
|
||||
// .values()
|
||||
// .map(|route| format!("\"{}\"", route.url_path))
|
||||
// .join(" | ");
|
||||
|
||||
// format!("export type {}Path = {};\n", name, paths)
|
||||
// };
|
||||
|
||||
// let date_type = map_to_type("Date", &self.date);
|
||||
|
||||
// let height_type = map_to_type("Height", &self.height);
|
||||
|
||||
// let last_type = map_to_type("Last", &self.last);
|
||||
|
||||
// fs::write(
|
||||
// format!("{WEBSITE_TYPES_PATH}/paths.d.ts"),
|
||||
// format!("// This file is auto generated by the server\n// Manual changes are forbidden\n\n{date_type}\n{height_type}\n{last_type}"),
|
||||
// )
|
||||
// .unwrap();
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -1,13 +1,12 @@
|
||||
use std::{collections::BTreeMap, time::Instant};
|
||||
|
||||
use api::{structs::Index, ApiRoutes};
|
||||
use axum::{routing::get, serve, Router};
|
||||
use axum::{routing::get, serve, Json, Router};
|
||||
use color_eyre::owo_colors::OwoColorize;
|
||||
use computer::Computer;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use indexer::Indexer;
|
||||
use logger::{error, info};
|
||||
use oxc::syntax::identifier::LF;
|
||||
use reqwest::StatusCode;
|
||||
use storable_vec::{AnyJsonStorableVec, STATELESS};
|
||||
use tokio::net::TcpListener;
|
||||
@@ -29,6 +28,7 @@ pub struct AppState {
|
||||
pub struct VecIdToIndexToVec(BTreeMap<String, IndexToVec>);
|
||||
|
||||
impl VecIdToIndexToVec {
|
||||
// Not the most performant or type safe but only built once so that's okay
|
||||
pub fn insert(&mut self, vec: &'static dyn AnyJsonStorableVec) {
|
||||
let file_name = vec.file_name();
|
||||
let split = file_name.split("_to_").collect::<Vec<_>>();
|
||||
@@ -83,7 +83,7 @@ pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) ->
|
||||
let router = Router::new()
|
||||
.add_api_routes()
|
||||
.add_website_routes()
|
||||
.route("/version", get(env!("CARGO_PKG_VERSION")))
|
||||
.route("/version", get(Json(env!("CARGO_PKG_VERSION"))))
|
||||
.with_state(state)
|
||||
.layer(compression_layer);
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ pub trait HeaderMapExtended {
|
||||
fn insert_content_type_application_pdf(&mut self);
|
||||
fn insert_content_type_text_css(&mut self);
|
||||
fn insert_content_type_text_csv(&mut self);
|
||||
fn insert_content_type_text_tsv(&mut self);
|
||||
fn insert_content_type_text_html(&mut self);
|
||||
fn insert_content_type_text_plain(&mut self);
|
||||
fn insert_content_type_font_woff2(&mut self);
|
||||
@@ -197,6 +198,10 @@ impl HeaderMapExtended for HeaderMap {
|
||||
self.insert(header::CONTENT_TYPE, "text/csv".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_tsv(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/tab-separated-values".parse().unwrap());
|
||||
}
|
||||
|
||||
fn insert_content_type_text_html(&mut self) {
|
||||
self.insert(header::CONTENT_TYPE, "text/html".parse().unwrap());
|
||||
}
|
||||
|
||||
@@ -10,17 +10,13 @@ use oxc::{
|
||||
span::SourceType,
|
||||
};
|
||||
|
||||
//
|
||||
pub fn minify_js(path: &Path) -> String {
|
||||
let allocator = Allocator::default();
|
||||
|
||||
let source_text = fs::read_to_string(path).unwrap();
|
||||
let source_type = SourceType::from_path(path).unwrap();
|
||||
|
||||
let source_text = fs::read_to_string(path).unwrap();
|
||||
let allocator = Allocator::default();
|
||||
|
||||
let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
|
||||
|
||||
let mut program = parser_return.program;
|
||||
let mut program = Parser::new(&allocator, &source_text, source_type).parse().program;
|
||||
|
||||
let minifier_return = Minifier::new(MinifierOptions {
|
||||
mangle: Some(MangleOptions::default()),
|
||||
|
||||
@@ -58,7 +58,7 @@ where
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
} else {
|
||||
unreachable!()
|
||||
todo!("todo ?")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user