release: v0.0.47

.cargo/config.toml

@@ -1,2 +0,0 @@
-[build]
-rustflags = ["-C", "target-cpu=native"]

Cargo.lock

@@ -388,7 +388,7 @@ dependencies = [
 
 [[package]]
 name = "brk"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "brk_cli",
  "brk_computer",
@@ -407,7 +407,7 @@ dependencies = [
 
 [[package]]
 name = "brk_cli"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bitcoincore-rpc",
  "brk_computer",
@@ -432,7 +432,7 @@ dependencies = [
 
 [[package]]
 name = "brk_computer"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -453,7 +453,7 @@ dependencies = [
 
 [[package]]
 name = "brk_core"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bincode",
  "bitcoin",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "brk_exit"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "brk_logger",
  "ctrlc",
@@ -483,7 +483,7 @@ dependencies = [
 
 [[package]]
 name = "brk_fetcher"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -496,7 +496,7 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -514,7 +514,7 @@ dependencies = [
 
 [[package]]
 name = "brk_logger"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -524,7 +524,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -539,7 +539,7 @@ dependencies = [
 
 [[package]]
 name = "brk_query"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "brk_computer",
  "brk_core",
@@ -557,7 +557,7 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "axum",
  "bitcoincore-rpc",
@@ -586,7 +586,7 @@ dependencies = [
 
 [[package]]
 name = "brk_state"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "bincode",
  "brk_core",
@@ -600,7 +600,7 @@ dependencies = [
 
 [[package]]
 name = "brk_store"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "arc-swap",
  "brk_core",
@@ -610,7 +610,7 @@ dependencies = [
 
 [[package]]
 name = "brk_vec"
-version = "0.0.45"
+version = "0.0.47"
 dependencies = [
  "arc-swap",
  "brk_core",
@@ -1171,9 +1171,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.1.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
+checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d"
 dependencies = [
  "crc32fast",
  "libz-rs-sys",
@@ -1301,9 +1301,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 
 [[package]]
 name = "hashbrown"
-version = "0.15.3"
+version = "0.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
+checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
 dependencies = [
  "allocator-api2",
  "equivalent",
@@ -1481,7 +1481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
  "serde",
 ]
 
@@ -1887,7 +1887,7 @@ checksum = "92e50218e74886659d1d13de8e6a4ff13c7e96924ed0017bc193a1feb8001b18"
 dependencies = [
  "allocator-api2",
  "bumpalo",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
  "oxc_data_structures",
  "rustc-hash",
 ]
@@ -2245,7 +2245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
 dependencies = [
  "fixedbitset",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
  "indexmap 2.9.0",
  "serde",
 ]
@@ -2378,7 +2378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
 ]
 
 [[package]]
@@ -2769,9 +2769,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "smallvec"
-version = "1.15.0"
+version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
 [[package]]
 name = "smawk"

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.45"
+package.version = "0.0.47"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 

README.md

@@ -31,17 +31,9 @@
   </a>
 </p>
 
-> **WARNING**
->
-> This project is still a work in progress and while it's much better in many ways than its previous version ([kibo v0.5](https://github.com/kibo-money/kibo)), it doesn't yet include all of those datasets. If you're interested in having everything right now, please use the latter until feature parity is achieved.
->
-> The explorer part (mempool.space/electrs) is also not viable just yet.
->
-> Stay tuned and please be patient, it's a lot of work !
-
 The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin Core node, enabling users to gain deeper insights into the Bitcoin network.
 
-In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) and [electrs](https://github.com/romanz/electrs) all in one package with a particular focus on simplicity and the self-hosting experience.
+In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and the self-hosting experience.
 
 The toolkit can be used in various ways to accommodate as many needs as possible:
 

crates/brk_cli/src/lib.rs

@@ -1,4 +1,4 @@
-use std::fs;
+use std::{fs, thread};
 
 use brk_core::{dot_brk_log_path, dot_brk_path};
 use brk_query::Params as QueryArgs;
@@ -35,8 +35,12 @@ pub fn main() -> color_eyre::Result<()> {
 
     let cli = Cli::parse();
 
-    match cli.command {
-        Commands::Run(args) => run(args),
-        Commands::Query(args) => query(args),
-    }
+    thread::Builder::new()
+        .stack_size(128 * 1024 * 1024)
+        .spawn(|| match cli.command {
+            Commands::Run(args) => run(args),
+            Commands::Query(args) => query(args),
+        })?
+        .join()
+        .unwrap()
 }

crates/brk_cli/src/query.rs

@@ -19,12 +19,14 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
     let query = Query::build(&indexer, &computer);
 
     let index = Index::try_from(params.index.as_str())?;
-
     let ids = params.values.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    let from = params.from();
+    let to = params.to();
+    let format = params.format();
 
-    let res = query.search_and_format(index, &ids, params.from, params.to, params.format)?;
+    let res = query.search_and_format(index, &ids, from, to, format)?;
 
-    if params.format.is_some() {
+    if format.is_some() {
         println!("{}", res);
     } else {
         println!(

crates/brk_cli/src/run.rs

@@ -1,7 +1,7 @@
 use std::{
     fs,
     path::{Path, PathBuf},
-    thread::{self, sleep},
+    thread::sleep,
     time::Duration,
 };
 
@@ -49,69 +49,61 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
         Ok(())
     };
 
-    let f = move || -> color_eyre::Result<()> {
-        let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
-        computer.import_stores(&indexer)?;
-        computer.import_vecs(&indexer, config.computation())?;
+    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
+    computer.import_stores(&indexer)?;
+    computer.import_vecs(&indexer, config.computation())?;
 
-        tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()?
-            .block_on(async {
-                let server = if config.serve() {
-                    let served_indexer = indexer.clone();
-                    let served_computer = computer.clone();
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()?
+        .block_on(async {
+            let server = if config.serve() {
+                let served_indexer = indexer.clone();
+                let served_computer = computer.clone();
 
-                    let server = Server::new(served_indexer, served_computer, config.website())?;
+                let server = Server::new(served_indexer, served_computer, config.website())?;
 
-                    let opt = Some(tokio::spawn(async move {
-                        server.serve().await.unwrap();
-                    }));
+                let opt = Some(tokio::spawn(async move {
+                    server.serve().await.unwrap();
+                }));
 
-                    sleep(Duration::from_secs(1));
+                sleep(Duration::from_secs(1));
 
-                    opt
-                } else {
-                    None
-                };
+                opt
+            } else {
+                None
+            };
 
-                if config.process() {
-                    loop {
-                        wait_for_synced_node()?;
+            if config.process() {
+                loop {
+                    wait_for_synced_node()?;
 
-                        let block_count = rpc.get_block_count()?;
+                    let block_count = rpc.get_block_count()?;
 
-                        info!("{} blocks found.", block_count + 1);
+                    info!("{} blocks found.", block_count + 1);
 
-                        let starting_indexes = indexer.index(&parser, rpc, &exit)?;
+                    let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
-                        computer.compute(&mut indexer, starting_indexes, &exit)?;
+                    computer.compute(&mut indexer, starting_indexes, &exit)?;
 
-                        if let Some(delay) = config.delay() {
-                            sleep(Duration::from_secs(delay))
-                        }
+                    if let Some(delay) = config.delay() {
+                        sleep(Duration::from_secs(delay))
+                    }
 
-                        info!("Waiting for new blocks...");
+                    info!("Waiting for new blocks...");
 
-                        while block_count == rpc.get_block_count()? {
-                            sleep(Duration::from_secs(1))
-                        }
+                    while block_count == rpc.get_block_count()? {
+                        sleep(Duration::from_secs(1))
                     }
                 }
+            }
 
-                if let Some(handle) = server {
-                    handle.await.unwrap();
-                }
+            if let Some(handle) = server {
+                handle.await.unwrap();
+            }
 
-                Ok(())
-            })
-    };
-
-    thread::Builder::new()
-        .stack_size(128 * 1024 * 1024)
-        .spawn(f)?
-        .join()
-        .unwrap()
+            Ok(())
+        })
 }
 
 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]

crates/brk_computer/src/vecs/mod.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::Path};
+use std::{fs, path::Path, thread};
 
 use brk_core::Version;
 use brk_exit::Exit;
@@ -44,22 +44,31 @@ impl Vecs {
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
-        let indexes = indexes::Vecs::forced_import(
-            path,
-            version + VERSION + Version::ZERO,
-            indexer,
-            computation,
-            format,
-        )?;
+        let (indexes, fetched) = thread::scope(|s| {
+            let indexes_handle = s.spawn(|| {
+                indexes::Vecs::forced_import(
+                    path,
+                    version + VERSION + Version::ZERO,
+                    indexer,
+                    computation,
+                    format,
+                )
+                .unwrap()
+            });
 
-        let fetched = fetch.then(|| {
-            fetched::Vecs::forced_import(
-                path,
-                version + VERSION + Version::ZERO,
-                computation,
-                format,
-            )
-            .unwrap()
+            let fetch_handle = s.spawn(|| {
+                fetch.then(|| {
+                    fetched::Vecs::forced_import(
+                        path,
+                        version + VERSION + Version::ZERO,
+                        computation,
+                        format,
+                    )
+                    .unwrap()
+                })
+            });
+
+            (indexes_handle.join().unwrap(), fetch_handle.join().unwrap())
         });
 
         Ok(Self {

crates/brk_core/src/structs/version.rs

@@ -33,7 +33,9 @@ impl Version {
         Self(self.0.swap_bytes())
     }
 
-    pub fn validate(&self, path: &Path) -> Result<()> {
+    /// Ok(true) if existed and is same
+    /// Ok(false) if didn't exist
+    pub fn validate(&self, path: &Path) -> Result<bool> {
         if let Ok(prev_version) = Version::try_from(path) {
             if prev_version != *self {
                 if prev_version.swap_bytes() == *self {
@@ -44,9 +46,11 @@ impl Version {
                     expected: *self,
                 });
             }
-        }
 
-        Ok(())
+            Ok(true)
+        } else {
+            Ok(false)
+        }
     }
 }
 

crates/brk_query/src/lib.rs

@@ -19,7 +19,7 @@ mod vec_trees;
 pub use format::Format;
 pub use index::Index;
 pub use output::{Output, Value};
-pub use params::Params;
+pub use params::{Params, ParamsOpt};
 pub use table::Tabled;
 use vec_trees::VecTrees;
 

crates/brk_query/src/params.rs

@@ -1,6 +1,8 @@
+use std::{fmt::Display, ops::Deref, str::FromStr};
+
 use clap::builder::PossibleValuesParser;
 use clap_derive::Parser;
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 use serde_with::{OneOrMany, formats::PreferOne, serde_as};
 
 use crate::{Format, Index};
@@ -17,15 +19,111 @@ pub struct Params {
     #[serde_as(as = "OneOrMany<_, PreferOne>")]
     /// Names of the values requested
     pub values: Vec<String>,
+
+    #[clap(flatten)]
+    #[serde(flatten)]
+    pub rest: ParamsOpt,
+}
+
+// The macro creates custom deserialization code.
+// You need to specify a function name and the field name of the flattened field.
+serde_with::flattened_maybe!(deserialize_rest, "rest");
+
+impl Deref for Params {
+    type Target = ParamsOpt;
+    fn deref(&self) -> &Self::Target {
+        &self.rest
+    }
+}
+
+impl From<((String, String), ParamsOpt)> for Params {
+    fn from(((index, id), rest): ((String, String), ParamsOpt)) -> Self {
+        Self {
+            index,
+            values: vec![id],
+            rest,
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize, Parser)]
+pub struct ParamsOpt {
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(alias = "f")]
+    #[serde(default, alias = "f", deserialize_with = "de_unquote_i64")]
     /// Inclusive starting index, if negative will be from the end
-    pub from: Option<i64>,
+    from: Option<i64>,
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(default, alias = "t")]
-    /// Inclusive ending index, if negative will be from the end
-    pub to: Option<i64>,
+    #[serde(default, alias = "t", deserialize_with = "de_unquote_i64")]
+    /// Exclusive ending index, if negative will be from the end, overrides 'count'
+    to: Option<i64>,
+    #[clap(short, long, allow_hyphen_values = true)]
+    #[serde(default, alias = "c", deserialize_with = "de_unquote_usize")]
+    /// Number of values
+    count: Option<usize>,
     #[clap(short = 'F', long)]
     /// Format of the output
-    pub format: Option<Format>,
+    format: Option<Format>,
+}
+
+impl ParamsOpt {
+    pub fn from(&self) -> Option<i64> {
+        self.from
+    }
+
+    pub fn to(&self) -> Option<i64> {
+        if self.to.is_none() {
+            if let Some(c) = self.count {
+                let c = c as i64;
+                if let Some(f) = self.from {
+                    if f.is_positive() || f.abs() > c {
+                        return Some(f + c);
+                    }
+                } else {
+                    return Some(c);
+                }
+            }
+        }
+        self.to
+    }
+
+    pub fn format(&self) -> Option<Format> {
+        self.format
+    }
+}
+
+fn de_unquote_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote_usize<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote<'de, D, F>(deserializer: D) -> Result<Option<F>, D::Error>
+where
+    D: Deserializer<'de>,
+    F: FromStr + Display,
+    <F as std::str::FromStr>::Err: Display,
+{
+    let opt: Option<String> = Option::deserialize(deserializer)?;
+    let s = match opt {
+        None => return Ok(None),
+        Some(mut s) => {
+            // strip any leading/trailing quotes
+            if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
+                s = s[1..s.len() - 1].to_string();
+            }
+            s
+        }
+    };
+    s.parse::<F>()
+        .map(Some)
+        .map_err(|e| serde::de::Error::custom(format!("cannot parse `{}` as type: {}", s, e)))
 }

crates/brk_server/README.md

@@ -41,22 +41,59 @@ The API uses `brk_query` and so inherites all of its features including formats.
 
 ### API
 
-#### `GET /api/vecs/indexes`
+#### [`GET /api/vecs/index-count`](https://bitcoinresearchkit.org/api/vecs/index-count)
+
+Count of all possible indexes
+
+#### [`GET /api/vecs/id-count`](https://bitcoinresearchkit.org/api/vecs/id-count)
+
+Count of all possible ids
+
+#### [`GET /api/vecs/variant-count`](https://bitcoinresearchkit.org/api/vecs/variant-count)
+
+Count of all possible variants
+
+#### [`GET /api/vecs/indexes`](https://bitcoinresearchkit.org/api/vecs/indexes)
 
 A list of all possible vec indexes and their accepted variants
 
-#### `GET /api/vecs/ids`
+#### [`GET /api/vecs/ids`](https://bitcoinresearchkit.org/api/vecs/ids)
 
 A list of all possible vec ids
 
-#### `GET /api/vecs/id-to-indexes`
+#### [`GET /api/vecs/variants`](https://bitcoinresearchkit.org/api/vecs/variants)
+
+A list of all possible variants
+
+#### [`GET /api/vecs/id-to-indexes`](https://bitcoinresearchkit.org/api/vecs/id-to-indexes)
 
 A list of all possible vec ids and their supported vec indexes
 
-#### `GET /api/vecs/index-to-ids`
+#### [`GET /api/vecs/index-to-ids`](https://bitcoinresearchkit.org/api/vecs/index-to-ids)
 
 A list of all possible vec indexes and their supported vec ids
 
+#### `GET /api/{INDEX}-to-{VALUE}`
+
+This endpoint retrieves data based on the specified vector index and id.
+
+**Parameters:**
+
+| Parameter | Type | Required | Description |
+| --- | --- | --- | --- |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
+| `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
+
+**Examples:**
+
+```
+GET /api/date-to-close
+GET /date-to-close?from=-100
+GET /date-to-close?count=100&format=csv
+```
+
 #### `GET /api/query`
 
 This endpoint retrieves data based on the specified vector index and values.
@@ -67,8 +104,9 @@ This endpoint retrieves data based on the specified vector index and values.
 | --- | --- | --- | --- |
 | `index` | `VecIndex` | Yes | The vector index to query. |
 | `values` | `VecId[]` | Yes | A comma or space-separated list of vector IDs to retrieve. |
-| `from` | `unsigned int` | No | The starting index for pagination (default is 0). |
-| `to` | `unsigned int` | No | The ending index for pagination (default is the total number of results). |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
 | `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
 
 **Examples:**
@@ -80,7 +118,7 @@ GET /api/query?index=week&values=ohlc,block-interval-average&from=0&to=20&format
 
 ### Meta
 
-#### `GET /version`
+#### [`GET /version`](https://bitcoinresearchkit.org/version)
 
 The version of the server and thus BRK.
 

crates/brk_server/src/api/mod.rs

@@ -1,18 +1,20 @@
 use std::collections::BTreeMap;
 
 use axum::{
-    Router,
-    extract::State,
+    Json, Router,
+    extract::{Path, Query, State},
+    http::HeaderMap,
     response::{IntoResponse, Redirect, Response},
     routing::get,
 };
+use brk_query::{Params, ParamsOpt};
 
 use super::AppState;
 
 mod explorer;
 mod query;
 
-pub use query::DTS;
+pub use query::Bridge;
 
 pub trait ApiRoutes {
     fn add_api_routes(self) -> Self;
@@ -20,24 +22,29 @@ pub trait ApiRoutes {
 
 impl ApiRoutes for Router<AppState> {
     fn add_api_routes(self) -> Self {
-        self.route(
-            "/api",
-            get(|| async {
-                Redirect::permanent(
-                    "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
-                )
-            }),
-        )
-        .route("/api/query", get(query::handler))
-        .route("/api/vecs/ids", get(vecids_handler))
-        .route("/api/vecs/indexes", get(vecindexes_handler))
-        .route("/api/vecs/id-to-indexes", get(vecid_to_vecindexes_handler))
-        .route("/api/vecs/index-to-ids", get(vecindex_to_vecids_handler))
+        self.route("/api/query", get(query::handler))
+            .route("/api/vecs/id-count", get(id_count_handler))
+            .route("/api/vecs/index-count", get(index_count_handler))
+            .route("/api/vecs/variant-count", get(variant_count_handler))
+            .route("/api/vecs/ids", get(ids_handler))
+            .route("/api/vecs/indexes", get(indexes_handler))
+            .route("/api/vecs/variants", get(variants_handler))
+            .route("/api/vecs/id-to-indexes", get(id_to_indexes_handler))
+            .route("/api/vecs/index-to-ids", get(index_to_ids_handler))
+            .route("/api/{variant}", get(variant_handler))
+            .route(
+                "/api",
+                get(|| async {
+                    Redirect::temporary(
+                        "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
+                    )
+                }),
+            )
     }
 }
 
-pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -48,8 +55,29 @@ pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn variant_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .values()
+            .map(|tree| tree.len())
+            .sum::<usize>(),
+    )
+    .into_response()
+}
+
+pub async fn id_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.id_to_index_to_vec.keys().count()).into_response()
+}
+
+pub async fn index_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.index_to_id_to_vec.keys().count()).into_response()
+}
+
+pub async fn indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -61,10 +89,48 @@ pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecid_to_vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+pub async fn variants_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .iter()
+            .flat_map(|(index, id_to_vec)| {
+                let index_ser = index.serialize_long();
+                id_to_vec
+                    .keys()
+                    .map(|id| format!("{}-to-{}", index_ser, id))
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>(),
+    )
+    .into_response()
 }
 
-pub async fn vecindex_to_vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+pub async fn id_to_indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+}
+
+pub async fn index_to_ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+}
+
+const TO_SEPARATOR: &str = "-to-";
+
+pub async fn variant_handler(
+    headers: HeaderMap,
+    Path(variant): Path<String>,
+    Query(params_opt): Query<ParamsOpt>,
+    state: State<AppState>,
+) -> Response {
+    let mut split = variant.split(TO_SEPARATOR);
+    let params = Params::from((
+        (
+            split.next().unwrap().to_string(),
+            split.collect::<Vec<_>>().join(TO_SEPARATOR),
+        ),
+        params_opt,
+    ));
+    query::handler(headers, Query(params), state).await
 }

crates/brk_server/src/api/query/bridge.rs

@@ -2,17 +2,17 @@ use std::{fs, io, path::Path};
 
 use brk_query::{Index, Query};
 
-use crate::Website;
+use crate::{VERSION, Website};
 
 const SCRIPTS: &str = "scripts";
 
 #[allow(clippy::upper_case_acronyms)]
-pub trait DTS {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
+pub trait Bridge {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
 }
 
-impl DTS for Query<'static> {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
+impl Bridge for Query<'static> {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
         if website.is_none() {
             return Ok(());
         }
@@ -31,10 +31,16 @@ impl DTS for Query<'static> {
 
         let indexes = Index::all();
 
-        let mut contents = "//
+        let mut contents = format!(
+            "//
 // File auto-generated, any modifications will be overwritten
-//\n\n"
-            .to_string();
+//
+
+export const VERSION = \"v{}\";
+
+",
+            VERSION
+        );
 
         contents += &indexes
             .iter()
@@ -57,7 +63,7 @@ impl DTS for Query<'static> {
 
         contents += "\n\nexport function createVecIdToIndexes() {\n";
 
-        contents += "\n\n  return /** @type {const} */ ({\n";
+        contents += "  return /** @type {const} */ ({\n";
 
         self.vec_trees
             .id_to_index_to_vec

crates/brk_server/src/api/query/mod.rs

@@ -5,14 +5,18 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use brk_query::{Format, Index, Output, Params};
+use brk_vec::{CollectableVec, StoredVec};
+use color_eyre::eyre::eyre;
 
 use crate::traits::{HeaderMapExtended, ModifiedState, ResponseExtended};
 
 use super::AppState;
 
-mod dts;
+mod bridge;
 
-pub use dts::*;
+pub use bridge::*;
+
+const MAX_WEIGHT: usize = 320_000;
 
 pub async fn handler(
     headers: HeaderMap,
@@ -33,11 +37,9 @@ pub async fn handler(
 fn req_to_response_res(
     headers: HeaderMap,
     AxumQuery(Params {
-        format,
-        from,
         index,
-        to,
         values,
+        rest,
     }): AxumQuery<Params>,
     AppState { query, .. }: AppState,
 ) -> color_eyre::Result<Response> {
@@ -48,6 +50,27 @@ fn req_to_response_res(
         &values.iter().map(|v| v.as_str()).collect::<Vec<_>>(),
     );
 
+    if vecs.is_empty() {
+        return Ok(Json(vec![] as Vec<usize>).into_response());
+    }
+
+    let from = rest.from();
+    let to = rest.to();
+    let format = rest.format();
+
+    let weight = vecs
+        .iter()
+        .map(|(_, v)| {
+            let len = v.len();
+            let count = StoredVec::<usize, usize>::range_count(from, to, len);
+            count * v.value_type_to_size_of()
+        })
+        .sum::<usize>();
+
+    if weight > MAX_WEIGHT {
+        return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}"));
+    }
+
     let mut date_modified_opt = None;
 
     if to.is_none() {
@@ -75,8 +98,8 @@ fn req_to_response_res(
         Output::TSV(s) => s.into_response(),
         Output::Json(v) => match v {
             brk_query::Value::Single(v) => Json(v).into_response(),
-            brk_query::Value::List(l) => Json(l).into_response(),
-            brk_query::Value::Matrix(m) => Json(m).into_response(),
+            brk_query::Value::List(v) => Json(v).into_response(),
+            brk_query::Value::Matrix(v) => Json(v).into_response(),
         },
         Output::MD(s) => s.into_response(),
     };

crates/brk_server/src/lib.rs

@@ -10,7 +10,7 @@ use std::{
     time::Duration,
 };
 
-use api::{ApiRoutes, DTS};
+use api::{ApiRoutes, Bridge};
 use axum::{
     Json, Router,
     body::Body,
@@ -45,10 +45,11 @@ pub struct AppState {
     websites_path: Option<PathBuf>,
 }
 
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+
 const DEV_PATH: &str = "../..";
 const DOWNLOADS: &str = "downloads";
 const WEBSITES: &str = "websites";
-const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 pub struct Server(AppState);
 
@@ -88,7 +89,7 @@ impl Server {
                 downloaded_websites_path
             };
 
-            query.generate_dts_file(website, websites_path.as_path())?;
+            query.generate_bridge_file(website, websites_path.as_path())?;
 
             Some(websites_path)
         } else {

crates/brk_vec/src/traits/any.rs

@@ -13,6 +13,7 @@ pub trait AnyVec: Send + Sync {
     }
     fn modified_time(&self) -> Result<Duration>;
     fn index_type_to_string(&self) -> String;
+    fn value_type_to_size_of(&self) -> usize;
 }
 
 pub trait AnyIterableVec<I, T>: AnyVec {

crates/brk_vec/src/traits/collectable.rs

@@ -27,13 +27,19 @@ where
     #[inline]
     fn i64_to_usize(i: i64, len: usize) -> usize {
         if i >= 0 {
-            i as usize
+            (i as usize).min(len)
         } else {
             let v = len as i64 + i;
             if v < 0 { 0 } else { v as usize }
         }
     }
 
+    fn range_count(from: Option<i64>, to: Option<i64>, len: usize) -> usize {
+        let from = from.map(|i| Self::i64_to_usize(i, len));
+        let to = to.map(|i| Self::i64_to_usize(i, len));
+        (from.unwrap_or_default()..to.unwrap_or(len)).count()
+    }
+
     #[doc(hidden)]
     fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
         let len = self.len();

crates/brk_vec/src/variants/compressed.rs

@@ -361,6 +361,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for CompressedVec<I, T> {

crates/brk_vec/src/variants/computed.rs

@@ -255,6 +255,11 @@ where
             ComputedVec::LazyFrom3(v) => v.modified_time(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {

crates/brk_vec/src/variants/eager.rs

@@ -1298,6 +1298,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> AnyIterableVec<I, T> for EagerVec<I, T>

crates/brk_vec/src/variants/indexed.rs

@@ -127,6 +127,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub trait AnyIndexedVec: AnyVec {

crates/brk_vec/src/variants/lazy1.rs

@@ -146,6 +146,11 @@ where
     fn modified_time(&self) -> Result<std::time::Duration> {
         self.source.modified_time()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T> AnyIterableVec<I, T> for LazyVecFrom1<I, T, S1I, S1T>

crates/brk_vec/src/variants/lazy2.rs

@@ -194,6 +194,11 @@ where
             .modified_time()?
             .min(self.source2.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T> AnyIterableVec<I, T> for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>

crates/brk_vec/src/variants/lazy3.rs

@@ -229,6 +229,11 @@ where
             .min(self.source2.modified_time()?)
             .min(self.source3.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>

crates/brk_vec/src/variants/raw.rs

@@ -48,8 +48,10 @@ where
         fs::create_dir_all(path)?;
 
         let version_path = Self::path_version_(path);
-        version.validate(version_path.as_ref())?;
-        version.write(version_path.as_ref())?;
+
+        if !version.validate(version_path.as_ref())? {
+            version.write(version_path.as_ref())?;
+        }
 
         let file = Self::open_file_(Self::path_vec_(path).as_path())?;
         let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
@@ -195,6 +197,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for RawVec<I, T> {

crates/brk_vec/src/variants/stored.rs

@@ -149,6 +149,11 @@ where
             StoredVec::Compressed(v) => v.name(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 #[derive(Debug)]

websites/default/index.html

@@ -1161,7 +1161,7 @@
       }
 
       if ("serviceWorker" in navigator) {
-        navigator.serviceWorker.register("/scripts/service-worker.js");
+        navigator.serviceWorker.register("/service-worker.js");
       }
     </script>
 

websites/default/scripts/main.js

@@ -2189,8 +2189,10 @@ function main() {
   createKeyDownEventListener();
 
   packages.signals().then((signals) =>
-    vecidToIndexesPromise.then(({ createVecIdToIndexes }) =>
+    vecidToIndexesPromise.then(({ createVecIdToIndexes, VERSION }) =>
       optionsPromise.then(async ({ initOptions }) => {
+        console.log(`VERSION = ${VERSION}`);
+
         const vecIdToIndexes = createVecIdToIndexes();
 
         if (env.localhost) {

websites/default/scripts/options.js

@@ -2795,10 +2795,6 @@ function createPartialOptions(colors) {
     {
       name: "Social",
       tree: [
-        {
-          name: "Github",
-          url: () => "https://github.com/bitcoinresearchkit/brk",
-        },
         {
           name: "Nostr",
           url: () =>
@@ -2867,6 +2863,10 @@ function createPartialOptions(colors) {
           url: () =>
             "lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4",
         },
+        {
+          name: "Geyser",
+          url: () => "https://geyser.fund/project/brk",
+        },
       ],
     },
     {

websites/default/scripts/service-worker.js

@@ -1,76 +0,0 @@
-const version = "v1";
-
-/** @type {ServiceWorkerGlobalScope} */
-const sw = /** @type {any} */ (self);
-
-sw.addEventListener("install", (_event) => {
-  console.log("sw: install");
-  sw.skipWaiting();
-});
-
-sw.addEventListener("activate", (event) => {
-  console.log("sw: active");
-  event.waitUntil(sw.clients.claim());
-});
-
-sw.addEventListener("fetch", (event) => {
-  let request = event.request;
-  const method = request.method;
-  let url = request.url;
-
-  const { pathname, origin } = new URL(url);
-
-  const slashMatches = url.match(/\//g);
-  const dotMatches = pathname.split("/").at(-1)?.match(/./g);
-  const endsWithDotHtml = pathname.endsWith(".html");
-  const slashApiSlashMatches = url.match(/\/api\//g);
-
-  if (
-    slashMatches &&
-    slashMatches.length <= 3 &&
-    !slashApiSlashMatches &&
-    (!dotMatches || endsWithDotHtml)
-  ) {
-    url = `${origin}/`;
-  }
-  request = new Request(url, request.mode !== "navigate" ? request : undefined);
-
-  console.log(request);
-
-  console.log(`service-worker: fetch ${url}`);
-
-  event.respondWith(
-    caches.match(request).then(async (cachedResponse) => {
-      return fetch(request)
-        .then((response) => {
-          const { status, type } = response;
-
-          if (method !== "GET" || slashApiSlashMatches) {
-            // API calls are cached in script.js
-            return response;
-          } else if ((status === 200 || status === 304) && type === "basic") {
-            if (status === 200) {
-              const clonedResponse = response.clone();
-              caches.open(version).then((cache) => {
-                cache.put(request, clonedResponse);
-              });
-            }
-            return response;
-          } else {
-            return cachedResponse || response;
-          }
-        })
-        .catch(() => {
-          console.log("service-worker: offline");
-
-          return (
-            cachedResponse ||
-            new Response("Offline", {
-              status: 503,
-              statusText: "Service Unavailable",
-            })
-          );
-        });
-    }),
-  );
-});

websites/default/scripts/vecid-to-indexes.js

@@ -2,6 +2,8 @@
 // File auto-generated, any modifications will be overwritten
 //
 
+export const VERSION = "v0.0.46";
+
 /** @typedef {0} DateIndex */
 /** @typedef {1} DecadeIndex */
 /** @typedef {2} DifficultyEpoch */
@@ -30,8 +32,6 @@
 /** @typedef {DateIndex | DecadeIndex | DifficultyEpoch | EmptyOutputIndex | HalvingEpoch | Height | InputIndex | MonthIndex | OpReturnIndex | OutputIndex | P2AIndex | P2MSIndex | P2PK33Index | P2PK65Index | P2PKHIndex | P2SHIndex | P2TRIndex | P2WPKHIndex | P2WSHIndex | QuarterIndex | TxIndex | UnknownOutputIndex | WeekIndex | YearIndex} Index */
 
 export function createVecIdToIndexes() {
-
-
   return /** @type {const} */ ({
     "0": [0, 1, 2, 5, 7, 19, 22, 23],
     "0sats-adjusted-spent-output-profit-ratio": [0],

websites/default/service-worker.js

@@ -0,0 +1,81 @@
+const CACHE_NAME = "v1";
+
+/** @type {ServiceWorkerGlobalScope} */
+const sw = /** @type {any} */ (self);
+
+sw.addEventListener("install", (event) => {
+  console.log("sw: install");
+  event.waitUntil(sw.skipWaiting());
+});
+
+sw.addEventListener("activate", (event) => {
+  console.log("sw: active");
+  event.waitUntil(sw.clients.claim());
+  event.waitUntil(caches.delete(CACHE_NAME));
+});
+
+async function indexHTMLOrOffline() {
+  return caches.match("/index.html").then((cached) => {
+    if (cached) return cached;
+    return new Response("Offline and no cached version", {
+      status: 503,
+      statusText: "Service Unavailable",
+      headers: { "Content-Type": "text/plain" },
+    });
+  });
+}
+
+sw.addEventListener("fetch", (event) => {
+  const req = event.request;
+  const url = new URL(req.url);
+
+  // 1) Bypass API calls & non-GETs
+  if (req.method !== "GET" || url.pathname.startsWith("/api")) {
+    return; // let the browser handle it
+  }
+
+  // 2) NAVIGATION: network‐first on your shell
+  if (req.mode === "navigate") {
+    event.respondWith(
+      // Always fetch index.html
+      fetch("/index.html")
+        .then((response) => {
+          // If we got a valid 2xx back, cache it (optional) and return it
+          if (response.ok || response.status === 304) {
+            if (response.ok) {
+              const clone = response.clone();
+              caches
+                .open(CACHE_NAME)
+                .then((cache) => cache.put("/index.html", clone));
+            }
+            return response;
+          }
+          throw new Error("Non-2xx on shell");
+        })
+        // On any failure, fall back to the cached shell
+        .catch(indexHTMLOrOffline),
+    );
+    return;
+  }
+
+  // 3) For all other GETs: network-first, fallback to cache
+  event.respondWith(
+    fetch(req)
+      .then((response) => {
+        if (response.ok) {
+          const clone = response.clone();
+          caches.open(CACHE_NAME).then((cache) => cache.put(req, clone));
+        }
+        return response;
+      })
+      .catch(async () => {
+        return caches
+          .match(req)
+          .then((cached) => {
+            return cached || indexHTMLOrOffline();
+          })
+          .catch(indexHTMLOrOffline);
+      })
+      .catch(indexHTMLOrOffline),
+  );
+});