diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index a3e464b8a..d6af28b26 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -12,7 +12,7 @@ use super::AppState; mod explorer; mod query; -pub use query::DTS; +pub use query::Bridge; pub trait ApiRoutes { fn add_api_routes(self) -> Self; diff --git a/crates/brk_server/src/api/query/dts.rs b/crates/brk_server/src/api/query/bridge.rs similarity index 89% rename from crates/brk_server/src/api/query/dts.rs rename to crates/brk_server/src/api/query/bridge.rs index 7e3bdce11..b5c28e683 100644 --- a/crates/brk_server/src/api/query/dts.rs +++ b/crates/brk_server/src/api/query/bridge.rs @@ -7,12 +7,12 @@ use crate::{VERSION, Website}; const SCRIPTS: &str = "scripts"; #[allow(clippy::upper_case_acronyms)] -pub trait DTS { - fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()>; +pub trait Bridge { + fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()>; } -impl DTS for Query<'static> { - fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()> { +impl Bridge for Query<'static> { + fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()> { if website.is_none() { return Ok(()); } diff --git a/crates/brk_server/src/api/query/mod.rs b/crates/brk_server/src/api/query/mod.rs index 4eb129dc0..6fe308af3 100644 --- a/crates/brk_server/src/api/query/mod.rs +++ b/crates/brk_server/src/api/query/mod.rs @@ -5,14 +5,18 @@ use axum::{ response::{IntoResponse, Response}, }; use brk_query::{Format, Index, Output, Params}; +use brk_vec::{CollectableVec, StoredVec}; +use color_eyre::eyre::eyre; use crate::traits::{HeaderMapExtended, ModifiedState, ResponseExtended}; use super::AppState; -mod dts; +mod bridge; -pub use dts::*; +pub use bridge::*; + +const MAX_WEIGHT: usize = 320_000; pub async fn handler( headers: HeaderMap, @@ -48,6 +52,23 @@ fn req_to_response_res( &values.iter().map(|v| v.as_str()).collect::>(), ); + if vecs.is_empty() { + return Ok(Json(vec![] as Vec).into_response()); + } + + let weight = vecs + .iter() + .map(|(_, v)| { + let len = v.len(); + let count = StoredVec::::range_count(from, to, len); + count * v.value_type_to_size_of() + }) + .sum::(); + + if weight > MAX_WEIGHT { + return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}")); + } + let mut date_modified_opt = None; if to.is_none() { @@ -75,8 +96,8 @@ fn req_to_response_res( Output::TSV(s) => s.into_response(), Output::Json(v) => match v { brk_query::Value::Single(v) => Json(v).into_response(), - brk_query::Value::List(l) => Json(l).into_response(), - brk_query::Value::Matrix(m) => Json(m).into_response(), + brk_query::Value::List(v) => Json(v).into_response(), + brk_query::Value::Matrix(v) => Json(v).into_response(), }, Output::MD(s) => s.into_response(), }; diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index e45096f00..ffddfe728 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -10,7 +10,7 @@ use std::{ time::Duration, }; -use api::{ApiRoutes, DTS}; +use api::{ApiRoutes, Bridge}; use axum::{ Json, Router, body::Body, @@ -89,7 +89,7 @@ impl Server { downloaded_websites_path }; - query.generate_dts_file(website, websites_path.as_path())?; + query.generate_bridge_file(website, websites_path.as_path())?; Some(websites_path) } else { diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs index 2ee5f5a06..0ec1fe409 100644 --- a/crates/brk_vec/src/traits/any.rs +++ b/crates/brk_vec/src/traits/any.rs @@ -13,6 +13,7 @@ pub trait AnyVec: Send + Sync { } fn modified_time(&self) -> Result; fn index_type_to_string(&self) -> String; + fn value_type_to_size_of(&self) -> usize; } pub trait AnyIterableVec: AnyVec { diff --git a/crates/brk_vec/src/traits/collectable.rs b/crates/brk_vec/src/traits/collectable.rs index 59b1e7529..22d7ddfa9 100644 --- a/crates/brk_vec/src/traits/collectable.rs +++ b/crates/brk_vec/src/traits/collectable.rs @@ -34,6 +34,12 @@ where } } + fn range_count(from: Option, to: Option, len: usize) -> usize { + let from = from.map(|i| Self::i64_to_usize(i, len)); + let to = to.map(|i| Self::i64_to_usize(i, len)); + (from.unwrap_or_default()..to.unwrap_or(len)).count() + } + #[doc(hidden)] fn collect_signed_range(&self, from: Option, to: Option) -> Result> { let len = self.len(); diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index c29111e8c..d1d2ed766 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -361,6 +361,11 @@ where fn index_type_to_string(&self) -> String { I::to_string() } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl Clone for CompressedVec { diff --git a/crates/brk_vec/src/variants/computed.rs b/crates/brk_vec/src/variants/computed.rs index 83ecdd685..7ad899488 100644 --- a/crates/brk_vec/src/variants/computed.rs +++ b/crates/brk_vec/src/variants/computed.rs @@ -255,6 +255,11 @@ where ComputedVec::LazyFrom3(v) => v.modified_time(), } } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> { diff --git a/crates/brk_vec/src/variants/eager.rs b/crates/brk_vec/src/variants/eager.rs index c5fbc7c7a..8c21141e8 100644 --- a/crates/brk_vec/src/variants/eager.rs +++ b/crates/brk_vec/src/variants/eager.rs @@ -1298,6 +1298,11 @@ where fn index_type_to_string(&self) -> String { I::to_string() } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl AnyIterableVec for EagerVec diff --git a/crates/brk_vec/src/variants/indexed.rs b/crates/brk_vec/src/variants/indexed.rs index 63b8c69b1..66e691259 100644 --- a/crates/brk_vec/src/variants/indexed.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -127,6 +127,11 @@ where fn index_type_to_string(&self) -> String { I::to_string() } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } pub trait AnyIndexedVec: AnyVec { diff --git a/crates/brk_vec/src/variants/lazy1.rs b/crates/brk_vec/src/variants/lazy1.rs index f2c0db94e..b5f0c9cf1 100644 --- a/crates/brk_vec/src/variants/lazy1.rs +++ b/crates/brk_vec/src/variants/lazy1.rs @@ -146,6 +146,11 @@ where fn modified_time(&self) -> Result { self.source.modified_time() } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl AnyIterableVec for LazyVecFrom1 diff --git a/crates/brk_vec/src/variants/lazy2.rs b/crates/brk_vec/src/variants/lazy2.rs index 16b3a640e..212f18017 100644 --- a/crates/brk_vec/src/variants/lazy2.rs +++ b/crates/brk_vec/src/variants/lazy2.rs @@ -194,6 +194,11 @@ where .modified_time()? .min(self.source2.modified_time()?)) } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl AnyIterableVec for LazyVecFrom2 diff --git a/crates/brk_vec/src/variants/lazy3.rs b/crates/brk_vec/src/variants/lazy3.rs index c23f2932e..2cff02d97 100644 --- a/crates/brk_vec/src/variants/lazy3.rs +++ b/crates/brk_vec/src/variants/lazy3.rs @@ -229,6 +229,11 @@ where .min(self.source2.modified_time()?) .min(self.source3.modified_time()?)) } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl AnyIterableVec diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 8f79e8da1..02f023f8f 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -195,6 +195,11 @@ where fn index_type_to_string(&self) -> String { I::to_string() } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } impl Clone for RawVec { diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs index 30dd68dae..a9ca895ff 100644 --- a/crates/brk_vec/src/variants/stored.rs +++ b/crates/brk_vec/src/variants/stored.rs @@ -149,6 +149,11 @@ where StoredVec::Compressed(v) => v.name(), } } + + #[inline] + fn value_type_to_size_of(&self) -> usize { + size_of::() + } } #[derive(Debug)] diff --git a/websites/default/scripts/vecid-to-indexes.js b/websites/default/scripts/vecid-to-indexes.js index b2de0f376..bf892b346 100644 --- a/websites/default/scripts/vecid-to-indexes.js +++ b/websites/default/scripts/vecid-to-indexes.js @@ -2,7 +2,7 @@ // File auto-generated, any modifications will be overwritten // -export const VERSION = "v0.0.45"; +export const VERSION = "v0.0.46"; /** @typedef {0} DateIndex */ /** @typedef {1} DecadeIndex */