release: v0.0.47

Cargo.lock

@@ -388,7 +388,7 @@ dependencies = [
 
 [[package]]
 name = "brk"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "brk_cli",
  "brk_computer",
@@ -407,7 +407,7 @@ dependencies = [
 
 [[package]]
 name = "brk_cli"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bitcoincore-rpc",
  "brk_computer",
@@ -432,7 +432,7 @@ dependencies = [
 
 [[package]]
 name = "brk_computer"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -453,7 +453,7 @@ dependencies = [
 
 [[package]]
 name = "brk_core"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bincode",
  "bitcoin",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "brk_exit"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "brk_logger",
  "ctrlc",
@@ -483,7 +483,7 @@ dependencies = [
 
 [[package]]
 name = "brk_fetcher"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -496,7 +496,7 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -514,7 +514,7 @@ dependencies = [
 
 [[package]]
 name = "brk_logger"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -524,7 +524,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -539,7 +539,7 @@ dependencies = [
 
 [[package]]
 name = "brk_query"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "brk_computer",
  "brk_core",
@@ -557,7 +557,7 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "axum",
  "bitcoincore-rpc",
@@ -586,7 +586,7 @@ dependencies = [
 
 [[package]]
 name = "brk_state"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "bincode",
  "brk_core",
@@ -600,7 +600,7 @@ dependencies = [
 
 [[package]]
 name = "brk_store"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "arc-swap",
  "brk_core",
@@ -610,7 +610,7 @@ dependencies = [
 
 [[package]]
 name = "brk_vec"
-version = "0.0.46"
+version = "0.0.47"
 dependencies = [
  "arc-swap",
  "brk_core",

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.46"
+package.version = "0.0.47"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 

crates/brk_cli/src/lib.rs

@@ -1,4 +1,4 @@
-use std::fs;
+use std::{fs, thread};
 
 use brk_core::{dot_brk_log_path, dot_brk_path};
 use brk_query::Params as QueryArgs;
@@ -35,8 +35,12 @@ pub fn main() -> color_eyre::Result<()> {
 
     let cli = Cli::parse();
 
-    match cli.command {
-        Commands::Run(args) => run(args),
-        Commands::Query(args) => query(args),
-    }
+    thread::Builder::new()
+        .stack_size(128 * 1024 * 1024)
+        .spawn(|| match cli.command {
+            Commands::Run(args) => run(args),
+            Commands::Query(args) => query(args),
+        })?
+        .join()
+        .unwrap()
 }

crates/brk_cli/src/query.rs

@@ -19,12 +19,14 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
     let query = Query::build(&indexer, &computer);
 
     let index = Index::try_from(params.index.as_str())?;
-
     let ids = params.values.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    let from = params.from();
+    let to = params.to();
+    let format = params.format();
 
-    let res = query.search_and_format(index, &ids, params.from, params.to, params.format)?;
+    let res = query.search_and_format(index, &ids, from, to, format)?;
 
-    if params.format.is_some() {
+    if format.is_some() {
         println!("{}", res);
     } else {
         println!(

crates/brk_cli/src/run.rs

@@ -1,7 +1,7 @@
 use std::{
     fs,
     path::{Path, PathBuf},
-    thread::{self, sleep},
+    thread::sleep,
     time::Duration,
 };
 
@@ -49,69 +49,61 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
         Ok(())
     };
 
-    let f = move || -> color_eyre::Result<()> {
-        let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
-        computer.import_stores(&indexer)?;
-        computer.import_vecs(&indexer, config.computation())?;
+    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
+    computer.import_stores(&indexer)?;
+    computer.import_vecs(&indexer, config.computation())?;
 
-        tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()?
-            .block_on(async {
-                let server = if config.serve() {
-                    let served_indexer = indexer.clone();
-                    let served_computer = computer.clone();
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()?
+        .block_on(async {
+            let server = if config.serve() {
+                let served_indexer = indexer.clone();
+                let served_computer = computer.clone();
 
-                    let server = Server::new(served_indexer, served_computer, config.website())?;
+                let server = Server::new(served_indexer, served_computer, config.website())?;
 
-                    let opt = Some(tokio::spawn(async move {
-                        server.serve().await.unwrap();
-                    }));
+                let opt = Some(tokio::spawn(async move {
+                    server.serve().await.unwrap();
+                }));
 
-                    sleep(Duration::from_secs(1));
+                sleep(Duration::from_secs(1));
 
-                    opt
-                } else {
-                    None
-                };
+                opt
+            } else {
+                None
+            };
 
-                if config.process() {
-                    loop {
-                        wait_for_synced_node()?;
+            if config.process() {
+                loop {
+                    wait_for_synced_node()?;
 
-                        let block_count = rpc.get_block_count()?;
+                    let block_count = rpc.get_block_count()?;
 
-                        info!("{} blocks found.", block_count + 1);
+                    info!("{} blocks found.", block_count + 1);
 
-                        let starting_indexes = indexer.index(&parser, rpc, &exit)?;
+                    let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
-                        computer.compute(&mut indexer, starting_indexes, &exit)?;
+                    computer.compute(&mut indexer, starting_indexes, &exit)?;
 
-                        if let Some(delay) = config.delay() {
-                            sleep(Duration::from_secs(delay))
-                        }
+                    if let Some(delay) = config.delay() {
+                        sleep(Duration::from_secs(delay))
+                    }
 
-                        info!("Waiting for new blocks...");
+                    info!("Waiting for new blocks...");
 
-                        while block_count == rpc.get_block_count()? {
-                            sleep(Duration::from_secs(1))
-                        }
+                    while block_count == rpc.get_block_count()? {
+                        sleep(Duration::from_secs(1))
                     }
                 }
+            }
 
-                if let Some(handle) = server {
-                    handle.await.unwrap();
-                }
+            if let Some(handle) = server {
+                handle.await.unwrap();
+            }
 
-                Ok(())
-            })
-    };
-
-    thread::Builder::new()
-        .stack_size(128 * 1024 * 1024)
-        .spawn(f)?
-        .join()
-        .unwrap()
+            Ok(())
+        })
 }
 
 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]

crates/brk_computer/src/vecs/mod.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::Path};
+use std::{fs, path::Path, thread};
 
 use brk_core::Version;
 use brk_exit::Exit;
@@ -44,22 +44,31 @@ impl Vecs {
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
-        let indexes = indexes::Vecs::forced_import(
-            path,
-            version + VERSION + Version::ZERO,
-            indexer,
-            computation,
-            format,
-        )?;
+        let (indexes, fetched) = thread::scope(|s| {
+            let indexes_handle = s.spawn(|| {
+                indexes::Vecs::forced_import(
+                    path,
+                    version + VERSION + Version::ZERO,
+                    indexer,
+                    computation,
+                    format,
+                )
+                .unwrap()
+            });
 
-        let fetched = fetch.then(|| {
-            fetched::Vecs::forced_import(
-                path,
-                version + VERSION + Version::ZERO,
-                computation,
-                format,
-            )
-            .unwrap()
+            let fetch_handle = s.spawn(|| {
+                fetch.then(|| {
+                    fetched::Vecs::forced_import(
+                        path,
+                        version + VERSION + Version::ZERO,
+                        computation,
+                        format,
+                    )
+                    .unwrap()
+                })
+            });
+
+            (indexes_handle.join().unwrap(), fetch_handle.join().unwrap())
         });
 
         Ok(Self {

crates/brk_core/src/structs/version.rs

@@ -33,7 +33,9 @@ impl Version {
         Self(self.0.swap_bytes())
     }
 
-    pub fn validate(&self, path: &Path) -> Result<()> {
+    /// Ok(true) if existed and is same
+    /// Ok(false) if didn't exist
+    pub fn validate(&self, path: &Path) -> Result<bool> {
         if let Ok(prev_version) = Version::try_from(path) {
             if prev_version != *self {
                 if prev_version.swap_bytes() == *self {
@@ -44,9 +46,11 @@ impl Version {
                     expected: *self,
                 });
             }
-        }
 
-        Ok(())
+            Ok(true)
+        } else {
+            Ok(false)
+        }
     }
 }
 

crates/brk_query/src/lib.rs

@@ -19,7 +19,7 @@ mod vec_trees;
 pub use format::Format;
 pub use index::Index;
 pub use output::{Output, Value};
-pub use params::Params;
+pub use params::{Params, ParamsOpt};
 pub use table::Tabled;
 use vec_trees::VecTrees;
 

crates/brk_query/src/params.rs

@@ -1,6 +1,8 @@
+use std::{fmt::Display, ops::Deref, str::FromStr};
+
 use clap::builder::PossibleValuesParser;
 use clap_derive::Parser;
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 use serde_with::{OneOrMany, formats::PreferOne, serde_as};
 
 use crate::{Format, Index};
@@ -17,15 +19,111 @@ pub struct Params {
     #[serde_as(as = "OneOrMany<_, PreferOne>")]
     /// Names of the values requested
     pub values: Vec<String>,
+
+    #[clap(flatten)]
+    #[serde(flatten)]
+    pub rest: ParamsOpt,
+}
+
+// The macro creates custom deserialization code.
+// You need to specify a function name and the field name of the flattened field.
+serde_with::flattened_maybe!(deserialize_rest, "rest");
+
+impl Deref for Params {
+    type Target = ParamsOpt;
+    fn deref(&self) -> &Self::Target {
+        &self.rest
+    }
+}
+
+impl From<((String, String), ParamsOpt)> for Params {
+    fn from(((index, id), rest): ((String, String), ParamsOpt)) -> Self {
+        Self {
+            index,
+            values: vec![id],
+            rest,
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize, Parser)]
+pub struct ParamsOpt {
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(alias = "f")]
+    #[serde(default, alias = "f", deserialize_with = "de_unquote_i64")]
     /// Inclusive starting index, if negative will be from the end
-    pub from: Option<i64>,
+    from: Option<i64>,
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(default, alias = "t")]
-    /// Inclusive ending index, if negative will be from the end
-    pub to: Option<i64>,
+    #[serde(default, alias = "t", deserialize_with = "de_unquote_i64")]
+    /// Exclusive ending index, if negative will be from the end, overrides 'count'
+    to: Option<i64>,
+    #[clap(short, long, allow_hyphen_values = true)]
+    #[serde(default, alias = "c", deserialize_with = "de_unquote_usize")]
+    /// Number of values
+    count: Option<usize>,
     #[clap(short = 'F', long)]
     /// Format of the output
-    pub format: Option<Format>,
+    format: Option<Format>,
+}
+
+impl ParamsOpt {
+    pub fn from(&self) -> Option<i64> {
+        self.from
+    }
+
+    pub fn to(&self) -> Option<i64> {
+        if self.to.is_none() {
+            if let Some(c) = self.count {
+                let c = c as i64;
+                if let Some(f) = self.from {
+                    if f.is_positive() || f.abs() > c {
+                        return Some(f + c);
+                    }
+                } else {
+                    return Some(c);
+                }
+            }
+        }
+        self.to
+    }
+
+    pub fn format(&self) -> Option<Format> {
+        self.format
+    }
+}
+
+fn de_unquote_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote_usize<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote<'de, D, F>(deserializer: D) -> Result<Option<F>, D::Error>
+where
+    D: Deserializer<'de>,
+    F: FromStr + Display,
+    <F as std::str::FromStr>::Err: Display,
+{
+    let opt: Option<String> = Option::deserialize(deserializer)?;
+    let s = match opt {
+        None => return Ok(None),
+        Some(mut s) => {
+            // strip any leading/trailing quotes
+            if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
+                s = s[1..s.len() - 1].to_string();
+            }
+            s
+        }
+    };
+    s.parse::<F>()
+        .map(Some)
+        .map_err(|e| serde::de::Error::custom(format!("cannot parse `{}` as type: {}", s, e)))
 }

crates/brk_server/README.md

@@ -41,22 +41,59 @@ The API uses `brk_query` and so inherites all of its features including formats.
 
 ### API
 
-#### `GET /api/vecs/indexes`
+#### [`GET /api/vecs/index-count`](https://bitcoinresearchkit.org/api/vecs/index-count)
+
+Count of all possible indexes
+
+#### [`GET /api/vecs/id-count`](https://bitcoinresearchkit.org/api/vecs/id-count)
+
+Count of all possible ids
+
+#### [`GET /api/vecs/variant-count`](https://bitcoinresearchkit.org/api/vecs/variant-count)
+
+Count of all possible variants
+
+#### [`GET /api/vecs/indexes`](https://bitcoinresearchkit.org/api/vecs/indexes)
 
 A list of all possible vec indexes and their accepted variants
 
-#### `GET /api/vecs/ids`
+#### [`GET /api/vecs/ids`](https://bitcoinresearchkit.org/api/vecs/ids)
 
 A list of all possible vec ids
 
-#### `GET /api/vecs/id-to-indexes`
+#### [`GET /api/vecs/variants`](https://bitcoinresearchkit.org/api/vecs/variants)
+
+A list of all possible variants
+
+#### [`GET /api/vecs/id-to-indexes`](https://bitcoinresearchkit.org/api/vecs/id-to-indexes)
 
 A list of all possible vec ids and their supported vec indexes
 
-#### `GET /api/vecs/index-to-ids`
+#### [`GET /api/vecs/index-to-ids`](https://bitcoinresearchkit.org/api/vecs/index-to-ids)
 
 A list of all possible vec indexes and their supported vec ids
 
+#### `GET /api/{INDEX}-to-{VALUE}`
+
+This endpoint retrieves data based on the specified vector index and id.
+
+**Parameters:**
+
+| Parameter | Type | Required | Description |
+| --- | --- | --- | --- |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
+| `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
+
+**Examples:**
+
+```
+GET /api/date-to-close
+GET /date-to-close?from=-100
+GET /date-to-close?count=100&format=csv
+```
+
 #### `GET /api/query`
 
 This endpoint retrieves data based on the specified vector index and values.
@@ -67,8 +104,9 @@ This endpoint retrieves data based on the specified vector index and values.
 | --- | --- | --- | --- |
 | `index` | `VecIndex` | Yes | The vector index to query. |
 | `values` | `VecId[]` | Yes | A comma or space-separated list of vector IDs to retrieve. |
-| `from` | `unsigned int` | No | The starting index for pagination (default is 0). |
-| `to` | `unsigned int` | No | The ending index for pagination (default is the total number of results). |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
 | `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
 
 **Examples:**
@@ -80,7 +118,7 @@ GET /api/query?index=week&values=ohlc,block-interval-average&from=0&to=20&format
 
 ### Meta
 
-#### `GET /version`
+#### [`GET /version`](https://bitcoinresearchkit.org/version)
 
 The version of the server and thus BRK.
 

crates/brk_server/src/api/mod.rs

@@ -1,18 +1,20 @@
 use std::collections::BTreeMap;
 
 use axum::{
-    Router,
-    extract::State,
+    Json, Router,
+    extract::{Path, Query, State},
+    http::HeaderMap,
     response::{IntoResponse, Redirect, Response},
     routing::get,
 };
+use brk_query::{Params, ParamsOpt};
 
 use super::AppState;
 
 mod explorer;
 mod query;
 
-pub use query::DTS;
+pub use query::Bridge;
 
 pub trait ApiRoutes {
     fn add_api_routes(self) -> Self;
@@ -20,24 +22,29 @@ pub trait ApiRoutes {
 
 impl ApiRoutes for Router<AppState> {
     fn add_api_routes(self) -> Self {
-        self.route(
-            "/api",
-            get(|| async {
-                Redirect::permanent(
-                    "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
-                )
-            }),
-        )
-        .route("/api/query", get(query::handler))
-        .route("/api/vecs/ids", get(vecids_handler))
-        .route("/api/vecs/indexes", get(vecindexes_handler))
-        .route("/api/vecs/id-to-indexes", get(vecid_to_vecindexes_handler))
-        .route("/api/vecs/index-to-ids", get(vecindex_to_vecids_handler))
+        self.route("/api/query", get(query::handler))
+            .route("/api/vecs/id-count", get(id_count_handler))
+            .route("/api/vecs/index-count", get(index_count_handler))
+            .route("/api/vecs/variant-count", get(variant_count_handler))
+            .route("/api/vecs/ids", get(ids_handler))
+            .route("/api/vecs/indexes", get(indexes_handler))
+            .route("/api/vecs/variants", get(variants_handler))
+            .route("/api/vecs/id-to-indexes", get(id_to_indexes_handler))
+            .route("/api/vecs/index-to-ids", get(index_to_ids_handler))
+            .route("/api/{variant}", get(variant_handler))
+            .route(
+                "/api",
+                get(|| async {
+                    Redirect::temporary(
+                        "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
+                    )
+                }),
+            )
     }
 }
 
-pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -48,8 +55,29 @@ pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn variant_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .values()
+            .map(|tree| tree.len())
+            .sum::<usize>(),
+    )
+    .into_response()
+}
+
+pub async fn id_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.id_to_index_to_vec.keys().count()).into_response()
+}
+
+pub async fn index_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.index_to_id_to_vec.keys().count()).into_response()
+}
+
+pub async fn indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -61,10 +89,48 @@ pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecid_to_vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+pub async fn variants_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .iter()
+            .flat_map(|(index, id_to_vec)| {
+                let index_ser = index.serialize_long();
+                id_to_vec
+                    .keys()
+                    .map(|id| format!("{}-to-{}", index_ser, id))
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>(),
+    )
+    .into_response()
 }
 
-pub async fn vecindex_to_vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+pub async fn id_to_indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+}
+
+pub async fn index_to_ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+}
+
+const TO_SEPARATOR: &str = "-to-";
+
+pub async fn variant_handler(
+    headers: HeaderMap,
+    Path(variant): Path<String>,
+    Query(params_opt): Query<ParamsOpt>,
+    state: State<AppState>,
+) -> Response {
+    let mut split = variant.split(TO_SEPARATOR);
+    let params = Params::from((
+        (
+            split.next().unwrap().to_string(),
+            split.collect::<Vec<_>>().join(TO_SEPARATOR),
+        ),
+        params_opt,
+    ));
+    query::handler(headers, Query(params), state).await
 }

crates/brk_server/src/api/query/bridge.rs

@@ -7,12 +7,12 @@ use crate::{VERSION, Website};
 const SCRIPTS: &str = "scripts";
 
 #[allow(clippy::upper_case_acronyms)]
-pub trait DTS {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
+pub trait Bridge {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
 }
 
-impl DTS for Query<'static> {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
+impl Bridge for Query<'static> {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
         if website.is_none() {
             return Ok(());
         }

crates/brk_server/src/api/query/mod.rs

@@ -5,14 +5,18 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use brk_query::{Format, Index, Output, Params};
+use brk_vec::{CollectableVec, StoredVec};
+use color_eyre::eyre::eyre;
 
 use crate::traits::{HeaderMapExtended, ModifiedState, ResponseExtended};
 
 use super::AppState;
 
-mod dts;
+mod bridge;
 
-pub use dts::*;
+pub use bridge::*;
+
+const MAX_WEIGHT: usize = 320_000;
 
 pub async fn handler(
     headers: HeaderMap,
@@ -33,11 +37,9 @@ pub async fn handler(
 fn req_to_response_res(
     headers: HeaderMap,
     AxumQuery(Params {
-        format,
-        from,
         index,
-        to,
         values,
+        rest,
     }): AxumQuery<Params>,
     AppState { query, .. }: AppState,
 ) -> color_eyre::Result<Response> {
@@ -48,6 +50,27 @@ fn req_to_response_res(
         &values.iter().map(|v| v.as_str()).collect::<Vec<_>>(),
     );
 
+    if vecs.is_empty() {
+        return Ok(Json(vec![] as Vec<usize>).into_response());
+    }
+
+    let from = rest.from();
+    let to = rest.to();
+    let format = rest.format();
+
+    let weight = vecs
+        .iter()
+        .map(|(_, v)| {
+            let len = v.len();
+            let count = StoredVec::<usize, usize>::range_count(from, to, len);
+            count * v.value_type_to_size_of()
+        })
+        .sum::<usize>();
+
+    if weight > MAX_WEIGHT {
+        return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}"));
+    }
+
     let mut date_modified_opt = None;
 
     if to.is_none() {
@@ -75,8 +98,8 @@ fn req_to_response_res(
         Output::TSV(s) => s.into_response(),
         Output::Json(v) => match v {
             brk_query::Value::Single(v) => Json(v).into_response(),
-            brk_query::Value::List(l) => Json(l).into_response(),
-            brk_query::Value::Matrix(m) => Json(m).into_response(),
+            brk_query::Value::List(v) => Json(v).into_response(),
+            brk_query::Value::Matrix(v) => Json(v).into_response(),
         },
         Output::MD(s) => s.into_response(),
     };

crates/brk_server/src/lib.rs

@@ -10,7 +10,7 @@ use std::{
     time::Duration,
 };
 
-use api::{ApiRoutes, DTS};
+use api::{ApiRoutes, Bridge};
 use axum::{
     Json, Router,
     body::Body,
@@ -89,7 +89,7 @@ impl Server {
                 downloaded_websites_path
             };
 
-            query.generate_dts_file(website, websites_path.as_path())?;
+            query.generate_bridge_file(website, websites_path.as_path())?;
 
             Some(websites_path)
         } else {

crates/brk_vec/src/traits/any.rs

@@ -13,6 +13,7 @@ pub trait AnyVec: Send + Sync {
     }
     fn modified_time(&self) -> Result<Duration>;
     fn index_type_to_string(&self) -> String;
+    fn value_type_to_size_of(&self) -> usize;
 }
 
 pub trait AnyIterableVec<I, T>: AnyVec {

crates/brk_vec/src/traits/collectable.rs

@@ -27,13 +27,19 @@ where
     #[inline]
     fn i64_to_usize(i: i64, len: usize) -> usize {
         if i >= 0 {
-            i as usize
+            (i as usize).min(len)
         } else {
             let v = len as i64 + i;
             if v < 0 { 0 } else { v as usize }
         }
     }
 
+    fn range_count(from: Option<i64>, to: Option<i64>, len: usize) -> usize {
+        let from = from.map(|i| Self::i64_to_usize(i, len));
+        let to = to.map(|i| Self::i64_to_usize(i, len));
+        (from.unwrap_or_default()..to.unwrap_or(len)).count()
+    }
+
     #[doc(hidden)]
     fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
         let len = self.len();

crates/brk_vec/src/variants/compressed.rs

@@ -361,6 +361,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for CompressedVec<I, T> {

crates/brk_vec/src/variants/computed.rs

@@ -255,6 +255,11 @@ where
             ComputedVec::LazyFrom3(v) => v.modified_time(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {

crates/brk_vec/src/variants/eager.rs

@@ -1298,6 +1298,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> AnyIterableVec<I, T> for EagerVec<I, T>

crates/brk_vec/src/variants/indexed.rs

@@ -127,6 +127,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub trait AnyIndexedVec: AnyVec {

crates/brk_vec/src/variants/lazy1.rs

@@ -146,6 +146,11 @@ where
     fn modified_time(&self) -> Result<std::time::Duration> {
         self.source.modified_time()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T> AnyIterableVec<I, T> for LazyVecFrom1<I, T, S1I, S1T>

crates/brk_vec/src/variants/lazy2.rs

@@ -194,6 +194,11 @@ where
             .modified_time()?
             .min(self.source2.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T> AnyIterableVec<I, T> for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>

crates/brk_vec/src/variants/lazy3.rs

@@ -229,6 +229,11 @@ where
             .min(self.source2.modified_time()?)
             .min(self.source3.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>

crates/brk_vec/src/variants/raw.rs

@@ -48,8 +48,10 @@ where
         fs::create_dir_all(path)?;
 
         let version_path = Self::path_version_(path);
-        version.validate(version_path.as_ref())?;
-        version.write(version_path.as_ref())?;
+
+        if !version.validate(version_path.as_ref())? {
+            version.write(version_path.as_ref())?;
+        }
 
         let file = Self::open_file_(Self::path_vec_(path).as_path())?;
         let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
@@ -195,6 +197,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for RawVec<I, T> {

crates/brk_vec/src/variants/stored.rs

@@ -149,6 +149,11 @@ where
             StoredVec::Compressed(v) => v.name(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 #[derive(Debug)]

websites/default/scripts/vecid-to-indexes.js

@@ -2,7 +2,7 @@
 // File auto-generated, any modifications will be overwritten
 //
 
-export const VERSION = "v0.0.45";
+export const VERSION = "v0.0.46";
 
 /** @typedef {0} DateIndex */
 /** @typedef {1} DecadeIndex */