release: v0.0.65

.cargo/config.toml

@@ -1,2 +0,0 @@
-[build]
-rustflags = ["-C", "target-cpu=native"]

.gitignore

@@ -3,6 +3,7 @@
 
 # Builds
 target
+dist
 
 # Copies
 *\ copy*

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.45"
+package.version = "0.0.65"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 
@@ -22,26 +22,27 @@ axum = "0.8.4"
 bincode = { version = "2.0.1", features = ["serde"] }
 bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_cli = { version = "0", path = "crates/brk_cli" }
-brk_computer = { version = "0", path = "crates/brk_computer" }
-brk_core = { version = "0", path = "crates/brk_core" }
-brk_exit = { version = "0", path = "crates/brk_exit" }
-brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0", path = "crates/brk_indexer" }
-brk_logger = { version = "0", path = "crates/brk_logger" }
-brk_parser = { version = "0", path = "crates/brk_parser" }
-brk_query = { version = "0", path = "crates/brk_query" }
-brk_server = { version = "0", path = "crates/brk_server" }
-brk_state = { version = "0", path = "crates/brk_state" }
-brk_store = { version = "0", path = "crates/brk_store" }
-brk_vec = { version = "0", path = "crates/brk_vec" }
+brk_bundler = { version = "0.0.65", path = "crates/brk_bundler" }
+brk_cli = { version = "0.0.65", path = "crates/brk_cli" }
+brk_computer = { version = "0.0.65", path = "crates/brk_computer" }
+brk_core = { version = "0.0.65", path = "crates/brk_core" }
+brk_exit = { version = "0.0.65", path = "crates/brk_exit" }
+brk_fetcher = { version = "0.0.65", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0.0.65", path = "crates/brk_indexer" }
+brk_logger = { version = "0.0.65", path = "crates/brk_logger" }
+brk_parser = { version = "0.0.65", path = "crates/brk_parser" }
+brk_query = { version = "0.0.65", path = "crates/brk_query" }
+brk_server = { version = "0.0.65", path = "crates/brk_server" }
+brk_state = { version = "0.0.65", path = "crates/brk_state" }
+brk_store = { version = "0.0.65", path = "crates/brk_store" }
+brk_vec = { version = "0.0.65", path = "crates/brk_vec" }
 byteview = "=0.6.1"
-clap = { version = "4.5.39", features = ["string"] }
-clap_derive = "4.5.32"
+clap = { version = "4.5.40", features = ["string"] }
+clap_derive = "4.5.40"
 color-eyre = "0.6.5"
 derive_deref = "1.1.1"
 fjall = "2.11.0"
-jiff = "0.2.14"
+jiff = "0.2.15"
 log = { version = "0.4.27" }
 minreq = { version = "2.13.4", features = ["https", "serde_json"] }
 rayon = "1.10.0"

README.md

@@ -31,28 +31,22 @@
   </a>
 </p>
 
-> **WARNING**
->
-> This project is still a work in progress and while it's much better in many ways than its previous version ([kibo v0.5](https://github.com/kibo-money/kibo)), it doesn't yet include all of those datasets. If you're interested in having everything right now, please use the latter until feature parity is achieved.
->
-> The explorer part (mempool.space/electrs) is also not viable just yet.
->
-> Stay tuned and please be patient, it's a lot of work !
+The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
 
-The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin Core node, enabling users to gain deeper insights into the Bitcoin network.
-
-In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) and [electrs](https://github.com/romanz/electrs) all in one package with a particular focus on simplicity and the self-hosting experience.
+In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and ease of use.
 
 The toolkit can be used in various ways to accommodate as many needs as possible:
 
-- **[Website](https://kibo.money)** \
-  Everyone is welcome to visit [kibo.money](https://kibo.money) which is the official showcase of the suite's capabilities and served by default when running BRK. \
-  Researchers and developers are free to use the API which endpoints documentation can be found [here](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints). \
-  As a token of gratitude to the community and to stimulate curiosity, both the website and the API are entirely free, allowing anyone to use them.
+- **[Website](https://bitcoinresearchkit.org)** \
+  Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
+  It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
+  Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
+- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#endpoints)** \
+  Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvariant-count&query=%24&style=flat&label=%20&color=white) dataset variants at your disposal. \
+  Just like the website, it's entirely free, with no authentication or rate-limiting.
 - **[CLI](https://crates.io/crates/brk_cli)** \
-  Node runners are strongly encouraged to try out and self-host their own instance. \
-  A lot of effort has gone into making this as easy as possible. \
-  For more information visit: [`brk_cli`](https://crates.io/crates/brk_cli)
+  Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
+  The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
 - **[Crates](https://crates.io/crates/brk)** \
   Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
   PRs are welcome, especially if their goal is to introduce additional datasets.
@@ -77,22 +71,15 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
 - [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
 - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
 - [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
-
-## Acknowledgments
-
-Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
-
-Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
+- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
 
 ## Hosting as a service
 
-*Soon™*
-
 If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
 
 - 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
 - 99.99% SLA
-- Configurated for speed (`raw + eager`)
+- Configured for speed
 - Updates delivered at your convenience
 - Direct communication for feature requests and support
 - Bitcoin Core or Knots with desired version
@@ -101,6 +88,12 @@ If you'd like to have your own instance hosted for you please contact [hosting@b
 
 Pricing: `0.01 BTC / month` *or* `0.1 BTC / year`
 
+## Acknowledgments
+
+Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
+
+Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [kibo.money](https://kibo.money) public instance.
+
 ## Donate
 
 [`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve)

crates/brk/Cargo.toml

@@ -10,6 +10,7 @@ version.workspace = true
 
 [features]
 full = [
+    "bundler",
     "core",
     "computer",
     "exit",
@@ -23,6 +24,7 @@ full = [
     "store",
     "vec",
 ]
+bundler = ["brk_bundler"]
 core = ["brk_core"]
 computer = ["brk_computer"]
 exit = ["brk_exit"]
@@ -37,6 +39,7 @@ store = ["brk_store"]
 vec = ["brk_vec"]
 
 [dependencies]
+brk_bundler = { workspace = true, optional = true }
 brk_cli = { workspace = true }
 brk_core = { workspace = true, optional = true }
 brk_computer = { workspace = true, optional = true }

crates/brk/examples/main.rs

@@ -0,0 +1 @@
+fn main() {}

crates/brk/src/lib.rs

@@ -1,5 +1,12 @@
 #![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
 
+#[cfg(feature = "bundler")]
+#[doc(inline)]
+pub use brk_bundler as bundler;
+
+#[doc(inline)]
+pub use brk_cli as cli;
+
 #[cfg(feature = "core")]
 #[doc(inline)]
 pub use brk_core as core;

crates/brk_bundler/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "brk_bundler"
+description = "A thin wrapper around rolldown"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[dependencies]
+log = { workspace = true }
+notify = "8.0.0"
+brk_rolldown = "0.0.1"
+sugar_path = "1.2.0"
+tokio = { workspace = true }

crates/brk_bundler/src/lib.rs

@@ -0,0 +1,145 @@
+use std::{fs, io, path::Path, sync::Arc};
+
+use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
+use log::error;
+use notify::{EventKind, RecursiveMode, Watcher};
+use sugar_path::SugarPath;
+use tokio::sync::Mutex;
+
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+
+pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
+    let source_path = websites_path.join(source_folder);
+    let dist_path = websites_path.join("dist");
+
+    let _ = fs::remove_dir_all(&dist_path);
+    copy_dir_all(&source_path, &dist_path)?;
+
+    let source_scripts = format!("./{source_folder}/scripts");
+    let source_entry = format!("{source_scripts}/entry.js");
+
+    let absolute_websites_path = websites_path.absolutize();
+
+    let mut bundler = Bundler::new(BundlerOptions {
+        input: Some(vec![source_entry.into()]),
+        dir: Some("./dist/scripts".to_string()),
+        cwd: Some(absolute_websites_path),
+        minify: Some(RawMinifyOptions::Bool(true)),
+        sourcemap: Some(SourceMapType::File),
+        ..Default::default()
+    });
+
+    bundler.write().await.unwrap();
+
+    let absolute_source_index_path = source_path.join("index.html").absolutize();
+    let absolute_source_index_path_clone = absolute_source_index_path.clone();
+    let absolute_source_path = source_path.absolutize();
+    let absolute_source_path_clone = absolute_source_path.clone();
+    let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
+    let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
+    let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
+
+    let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
+    let absolute_dist_index_path = dist_path.join("index.html").absolutize();
+    let absolute_dist_path = dist_path.absolutize();
+    let absolute_dist_path_clone = absolute_dist_path.clone();
+    let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
+
+    let write_index = move || {
+        let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
+
+        if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
+            if let Some(start) = entry.find("main") {
+                if let Some(end) = entry.find(".js") {
+                    let main_hashed = &entry[start..end];
+                    contents =
+                        contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
+                }
+            }
+        }
+
+        let _ = fs::write(&absolute_dist_index_path, contents);
+    };
+
+    let write_sw = move || {
+        let contents = fs::read_to_string(&absolute_source_sw_path)
+            .unwrap()
+            .replace("__VERSION__", &format!("v{VERSION}"));
+        let _ = fs::write(&absolute_dist_sw_path, contents);
+    };
+
+    write_index();
+    write_sw();
+
+    if !watch {
+        return Ok(());
+    }
+
+    tokio::spawn(async move {
+        let write_index_clone = write_index.clone();
+
+        let mut entry_watcher = notify::recommended_watcher(
+            move |res: Result<notify::Event, notify::Error>| match res {
+                Ok(_) => write_index_clone(),
+                Err(e) => error!("watch error: {:?}", e),
+            },
+        )
+        .unwrap();
+
+        entry_watcher
+            .watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
+            .unwrap();
+
+        let mut source_watcher = notify::recommended_watcher(
+            move |res: Result<notify::Event, notify::Error>| match res {
+                Ok(event) => match event.kind {
+                    EventKind::Create(_) => event.paths,
+                    EventKind::Modify(_) => event.paths,
+                    _ => vec![],
+                }
+                .into_iter()
+                .filter(|path| path.starts_with(&absolute_source_path))
+                .filter(|path| !path.starts_with(&absolute_source_scripts_path))
+                .for_each(|source_path| {
+                    let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
+                    let dist_path = absolute_dist_path.join(suffix);
+
+                    if source_path == absolute_source_index_path_clone {
+                        write_index();
+                    } else if source_path == absolute_source_sw_path_clone {
+                        write_sw();
+                    } else {
+                        let _ = fs::copy(&source_path, &dist_path);
+                    }
+                }),
+                Err(e) => error!("watch error: {:?}", e),
+            },
+        )
+        .unwrap();
+
+        source_watcher
+            .watch(&absolute_source_path_clone, RecursiveMode::Recursive)
+            .unwrap();
+
+        let watcher =
+            brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
+
+        watcher.start().await;
+    });
+
+    Ok(())
+}
+
+fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir_all(&dst)?;
+    for entry in fs::read_dir(src)? {
+        let entry = entry?;
+        let ty = entry.file_type()?;
+        if ty.is_dir() {
+            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        } else {
+            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        }
+    }
+    Ok(())
+}

crates/brk_cli/src/lib.rs

@@ -1,4 +1,4 @@
-use std::fs;
+use std::{fs, thread};
 
 use brk_core::{dot_brk_log_path, dot_brk_path};
 use brk_query::Params as QueryArgs;
@@ -20,9 +20,9 @@ struct Cli {
 
 #[derive(Subcommand, Debug)]
 enum Commands {
-    /// Run the indexer, computer and server
+    /// Run the indexer, computer and server, use `run -h` for more information
     Run(RunConfig),
-    /// Query generated datasets via the `run` command in a similar fashion as the server's API
+    /// Query generated datasets via the `run` command in a similar fashion as the server's API, use `query -h` for more information
     Query(QueryArgs),
 }
 
@@ -35,8 +35,12 @@ pub fn main() -> color_eyre::Result<()> {
 
     let cli = Cli::parse();
 
-    match cli.command {
-        Commands::Run(args) => run(args),
-        Commands::Query(args) => query(args),
-    }
+    thread::Builder::new()
+        .stack_size(128 * 1024 * 1024)
+        .spawn(|| match cli.command {
+            Commands::Run(args) => run(args),
+            Commands::Query(args) => query(args),
+        })?
+        .join()
+        .unwrap()
 }

crates/brk_cli/src/query.rs

@@ -10,7 +10,7 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
 
     let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
     indexer.import_vecs()?;
 
     let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
@@ -19,12 +19,14 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> {
     let query = Query::build(&indexer, &computer);
 
     let index = Index::try_from(params.index.as_str())?;
-
     let ids = params.values.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    let from = params.from();
+    let to = params.to();
+    let format = params.format();
 
-    let res = query.search_and_format(index, &ids, params.from, params.to, params.format)?;
+    let res = query.search_and_format(index, &ids, from, to, format)?;
 
-    if params.format.is_some() {
+    if format.is_some() {
         println!("{}", res);
     } else {
         println!(

crates/brk_cli/src/run.rs

@@ -1,13 +1,13 @@
 use std::{
     fs,
     path::{Path, PathBuf},
-    thread::{self, sleep},
+    thread::sleep,
     time::Duration,
 };
 
 use bitcoincore_rpc::{self, Auth, Client, RpcApi};
 use brk_computer::Computer;
-use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
+use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -29,7 +29,7 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
 
     let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), config.check_collisions())?;
     indexer.import_stores()?;
     indexer.import_vecs()?;
 
@@ -49,130 +49,143 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
         Ok(())
     };
 
-    let f = move || -> color_eyre::Result<()> {
-        let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
-        computer.import_stores(&indexer)?;
-        computer.import_vecs(&indexer, config.computation())?;
+    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
+    computer.import_stores(&indexer)?;
+    computer.import_vecs(&indexer, config.computation())?;
 
-        tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()?
-            .block_on(async {
-                let server = if config.serve() {
-                    let served_indexer = indexer.clone();
-                    let served_computer = computer.clone();
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()?
+        .block_on(async {
+            let server = if config.serve() {
+                let served_indexer = indexer.clone();
+                let served_computer = computer.clone();
 
-                    let server = Server::new(served_indexer, served_computer, config.website())?;
+                let server = Server::new(served_indexer, served_computer, config.website())?;
 
-                    let opt = Some(tokio::spawn(async move {
-                        server.serve().await.unwrap();
-                    }));
+                let watch = config.watch();
+                let opt = Some(tokio::spawn(async move {
+                    server.serve(watch).await.unwrap();
+                }));
 
-                    sleep(Duration::from_secs(1));
+                sleep(Duration::from_secs(1));
 
-                    opt
-                } else {
-                    None
-                };
+                opt
+            } else {
+                None
+            };
 
-                if config.process() {
-                    loop {
-                        wait_for_synced_node()?;
+            if config.process() {
+                loop {
+                    wait_for_synced_node()?;
 
-                        let block_count = rpc.get_block_count()?;
+                    let block_count = rpc.get_block_count()?;
 
-                        info!("{} blocks found.", block_count + 1);
+                    info!("{} blocks found.", block_count + 1);
 
-                        let starting_indexes = indexer.index(&parser, rpc, &exit)?;
+                    let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
-                        computer.compute(&mut indexer, starting_indexes, &exit)?;
+                    computer.compute(&mut indexer, starting_indexes, &exit)?;
 
-                        if let Some(delay) = config.delay() {
-                            sleep(Duration::from_secs(delay))
-                        }
+                    if let Some(delay) = config.delay() {
+                        sleep(Duration::from_secs(delay))
+                    }
 
-                        info!("Waiting for new blocks...");
+                    info!("Waiting for new blocks...");
 
-                        while block_count == rpc.get_block_count()? {
-                            sleep(Duration::from_secs(1))
-                        }
+                    while block_count == rpc.get_block_count()? {
+                        sleep(Duration::from_secs(1))
                     }
                 }
+            }
 
-                if let Some(handle) = server {
-                    handle.await.unwrap();
-                }
+            if let Some(handle) = server {
+                handle.await.unwrap();
+            }
 
-                Ok(())
-            })
-    };
-
-    thread::Builder::new()
-        .stack_size(128 * 1024 * 1024)
-        .spawn(f)?
-        .join()
-        .unwrap()
+            Ok(())
+        })
 }
 
 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
 pub struct RunConfig {
     /// Bitcoin main directory path, defaults: ~/.bitcoin, ~/Library/Application\ Support/Bitcoin, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     bitcoindir: Option<String>,
 
     /// Bitcoin blocks directory path, default: --bitcoindir/blocks, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     blocksdir: Option<String>,
 
     /// Bitcoin Research Kit outputs directory path, default: ~/.brk, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     brkdir: Option<String>,
 
-    /// Executed by the runner, default: all, saved
+    /// Activated services, default: all, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
-    mode: Option<Mode>,
+    services: Option<Services>,
 
-    /// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
+    /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
     computation: Option<Computation>,
 
-    /// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
-    #[arg(short, long, value_name = "FORMAT")]
+    /// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
+    #[serde(default, deserialize_with = "default_on_error")]
+    #[arg(short, long)]
     format: Option<Format>,
 
     /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short = 'F', long, value_name = "BOOL")]
     fetch: Option<bool>,
 
     /// Website served by the server (if active), default: default, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(short, long)]
     website: Option<Website>,
 
     /// Bitcoin RPC ip, default: localhost, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "IP")]
     rpcconnect: Option<String>,
 
     /// Bitcoin RPC port, default: 8332, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PORT")]
     rpcport: Option<u16>,
 
     /// Bitcoin RPC cookie file, default: --bitcoindir/.cookie, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PATH")]
     rpccookiefile: Option<String>,
 
     /// Bitcoin RPC username, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "USERNAME")]
     rpcuser: Option<String>,
 
     /// Bitcoin RPC password, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "PASSWORD")]
     rpcpassword: Option<String>,
 
     /// Delay between runs, default: 0, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "SECONDS")]
     delay: Option<u64>,
 
+    /// DEV: Activate to watch the selected website's folder for changes, default: false, saved
+    #[serde(default, deserialize_with = "default_on_error")]
+    #[arg(long, value_name = "BOOL")]
+    watch: Option<bool>,
+
     /// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
+    #[serde(default, deserialize_with = "default_on_error")]
     #[arg(long, value_name = "BOOL")]
     check_collisions: Option<bool>,
 }
@@ -200,8 +213,8 @@ impl RunConfig {
                 config_saved.brkdir = Some(brkdir);
             }
 
-            if let Some(mode) = config_args.mode.take() {
-                config_saved.mode = Some(mode);
+            if let Some(services) = config_args.services.take() {
+                config_saved.services = Some(services);
             }
 
             if let Some(computation) = config_args.computation.take() {
@@ -248,6 +261,10 @@ impl RunConfig {
                 config_saved.check_collisions = Some(check_collisions);
             }
 
+            if let Some(watch) = config_args.watch.take() {
+                config_saved.watch = Some(watch);
+            }
+
             if config_args != RunConfig::default() {
                 dbg!(config_args);
                 panic!("Didn't consume the full config")
@@ -260,19 +277,6 @@ impl RunConfig {
 
         config.write(&path)?;
 
-        // info!("Configuration {{");
-        // info!("  bitcoindir: {:?}", config.bitcoindir);
-        // info!("  brkdir: {:?}", config.brkdir);
-        // info!("  mode: {:?}", config.mode);
-        // info!("  website: {:?}", config.website);
-        // info!("  rpcconnect: {:?}", config.rpcconnect);
-        // info!("  rpcport: {:?}", config.rpcport);
-        // info!("  rpccookiefile: {:?}", config.rpccookiefile);
-        // info!("  rpcuser: {:?}", config.rpcuser);
-        // info!("  rpcpassword: {:?}", config.rpcpassword);
-        // info!("  delay: {:?}", config.delay);
-        // info!("}}");
-
         Ok(config)
     }
 
@@ -383,13 +387,13 @@ impl RunConfig {
     }
 
     pub fn process(&self) -> bool {
-        self.mode
-            .is_none_or(|m| m == Mode::All || m == Mode::Processor)
+        self.services
+            .is_none_or(|m| m == Services::All || m == Services::Processor)
     }
 
     pub fn serve(&self) -> bool {
-        self.mode
-            .is_none_or(|m| m == Mode::All || m == Mode::Server)
+        self.services
+            .is_none_or(|m| m == Services::All || m == Services::Server)
     }
 
     fn path_cookiefile(&self) -> PathBuf {
@@ -441,6 +445,10 @@ impl RunConfig {
     pub fn check_collisions(&self) -> bool {
         self.check_collisions.is_some_and(|b| b)
     }
+
+    pub fn watch(&self) -> bool {
+        self.watch.is_some_and(|b| b)
+    }
 }
 
 #[derive(
@@ -457,7 +465,7 @@ impl RunConfig {
     PartialOrd,
     Ord,
 )]
-pub enum Mode {
+pub enum Services {
     #[default]
     All,
     Processor,

crates/brk_computer/examples/main.rs

@@ -33,7 +33,7 @@ pub fn main() -> color_eyre::Result<()> {
 
             let format = Format::Raw;
 
-            let mut indexer = Indexer::new(outputs_dir, format, true)?;
+            let mut indexer = Indexer::new(outputs_dir, true)?;
             indexer.import_stores()?;
             indexer.import_vecs()?;
 

crates/brk_computer/src/vecs/fetched.rs

@@ -7,7 +7,7 @@ use brk_core::{
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
+use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format, StoredIndex};
 
 use super::{
     Indexes,
@@ -429,8 +429,18 @@ impl Vecs {
         self.dateindex_to_ohlc_in_cents.compute_transform(
             starting_indexes.dateindex,
             &indexes.dateindex_to_date,
-            |(di, d, ..)| {
-                let ohlc = fetcher.get_date(d).unwrap();
+            |(di, d, this)| {
+                let mut ohlc = fetcher.get_date(d).unwrap();
+                if let Some(prev) = di.decremented() {
+                    let prev_open = *this
+                        .get_or_read(prev, &this.mmap().load())
+                        .unwrap()
+                        .unwrap()
+                        .close;
+                    *ohlc.open = prev_open;
+                    *ohlc.high = (*ohlc.high).max(prev_open);
+                    *ohlc.low = (*ohlc.low).min(prev_open);
+                }
                 (di, ohlc)
             },
             exit,

crates/brk_computer/src/vecs/grouped/builder.rs

@@ -130,7 +130,11 @@ where
                 Box::new(
                     EagerVec::forced_import(
                         path,
-                        &maybe_suffix("sum"),
+                        &(if !options.last {
+                            name.to_string()
+                        } else {
+                            maybe_suffix("sum")
+                        }),
                         version + VERSION + Version::ZERO,
                         format,
                     )

crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs

@@ -24,8 +24,10 @@ pub struct ComputedRatioVecsFromDateIndex {
     pub ratio_1w_sma: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_1m_sma: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_1y_sma: ComputedVecsFromDateIndex<StoredF32>,
+    pub ratio_4y_sma: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex<StoredF32>,
-    pub ratio_standard_deviation: ComputedVecsFromDateIndex<StoredF32>,
+    pub ratio_sd: ComputedVecsFromDateIndex<StoredF32>,
+    pub ratio_4y_sd: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_p99_9: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_p99_5: ComputedVecsFromDateIndex<StoredF32>,
     pub ratio_p99: ComputedVecsFromDateIndex<StoredF32>,
@@ -51,6 +53,7 @@ pub struct ComputedRatioVecsFromDateIndex {
     pub ratio_m2sd_as_price: ComputedVecsFromDateIndex<Dollars>,
     pub ratio_m3sd_as_price: ComputedVecsFromDateIndex<Dollars>,
     pub ratio_zscore: ComputedVecsFromDateIndex<StoredF32>,
+    pub ratio_4y_zscore: ComputedVecsFromDateIndex<StoredF32>,
 }
 
 const VERSION: Version = Version::ZERO;
@@ -116,6 +119,14 @@ impl ComputedRatioVecsFromDateIndex {
                 format,
                 options,
             )?,
+            ratio_4y_sma: ComputedVecsFromDateIndex::forced_import(
+                path,
+                &format!("{name}_ratio_4y_sma"),
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
             ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
                 path,
                 &format!("{name}_ratio_1y_sma_momentum_oscillator"),
@@ -124,9 +135,17 @@ impl ComputedRatioVecsFromDateIndex {
                 format,
                 options,
             )?,
-            ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
+            ratio_sd: ComputedVecsFromDateIndex::forced_import(
                 path,
-                &format!("{name}_ratio_standard_deviation"),
+                &format!("{name}_ratio_sd"),
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            ratio_4y_sd: ComputedVecsFromDateIndex::forced_import(
+                path,
+                &format!("{name}_ratio_4y_sd"),
                 true,
                 version + VERSION + Version::ZERO,
                 format,
@@ -332,6 +351,14 @@ impl ComputedRatioVecsFromDateIndex {
                 format,
                 options,
             )?,
+            ratio_4y_zscore: ComputedVecsFromDateIndex::forced_import(
+                path,
+                &format!("{name}_ratio_4y_zscore"),
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
         })
     }
 
@@ -476,6 +503,22 @@ impl ComputedRatioVecsFromDateIndex {
             },
         )?;
 
+        self.ratio_4y_sma.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_sma_(
+                    starting_indexes.dateindex,
+                    self.ratio.dateindex.as_ref().unwrap(),
+                    4 * 365,
+                    exit,
+                    Some(min_ratio_date),
+                )
+            },
+        )?;
+
         self.ratio_1y_sma_momentum_oscillator.compute_all(
             indexer,
             indexes,
@@ -528,6 +571,8 @@ impl ComputedRatioVecsFromDateIndex {
 
         let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
 
+        let mut _4y_sma_iter = self.ratio_4y_sma.dateindex.as_ref().unwrap().into_iter();
+
         let nan = StoredF32::from(f32::NAN);
         self.ratio
             .dateindex
@@ -566,7 +611,12 @@ impl ComputedRatioVecsFromDateIndex {
                         .as_mut()
                         .unwrap()
                         .forced_push_at(index, nan, exit)?;
-                    self.ratio_standard_deviation
+                    self.ratio_sd
+                        .dateindex
+                        .as_mut()
+                        .unwrap()
+                        .forced_push_at(index, nan, exit)?;
+                    self.ratio_4y_sd
                         .dateindex
                         .as_mut()
                         .unwrap()
@@ -645,12 +695,26 @@ impl ComputedRatioVecsFromDateIndex {
                             .sqrt(),
                     );
 
-                    self.ratio_standard_deviation
+                    self.ratio_sd
                         .dateindex
                         .as_mut()
                         .unwrap()
                         .forced_push_at(index, sd, exit)?;
 
+                    let _4y_avg = _4y_sma_iter.unwrap_get_inner(index);
+
+                    let _4y_sd = StoredF32::from(
+                        (sorted.iter().map(|v| (**v - *_4y_avg).powi(2)).sum::<f32>()
+                            / (index.unwrap_to_usize() + 1) as f32)
+                            .sqrt(),
+                    );
+
+                    self.ratio_4y_sd
+                        .dateindex
+                        .as_mut()
+                        .unwrap()
+                        .forced_push_at(index, _4y_sd, exit)?;
+
                     self.ratio_p1sd.dateindex.as_mut().unwrap().forced_push_at(
                         index,
                         avg + sd,
@@ -726,7 +790,13 @@ impl ComputedRatioVecsFromDateIndex {
             exit,
             None as Option<&EagerVec<_, _>>,
         )?;
-        self.ratio_standard_deviation.compute_rest(
+        self.ratio_sd.compute_rest(
+            indexes,
+            starting_indexes,
+            exit,
+            None as Option<&EagerVec<_, _>>,
+        )?;
+        self.ratio_4y_sd.compute_rest(
             indexes,
             starting_indexes,
             exit,
@@ -1007,21 +1077,27 @@ impl ComputedRatioVecsFromDateIndex {
             starting_indexes,
             exit,
             |vec, _, _, starting_indexes, exit| {
-                let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
-                let mut sd_iter = self
-                    .ratio_standard_deviation
-                    .dateindex
-                    .as_ref()
-                    .unwrap()
-                    .into_iter();
-                vec.compute_transform(
+                vec.compute_zscore(
                     starting_indexes.dateindex,
                     self.ratio.dateindex.as_ref().unwrap(),
-                    |(i, ratio, ..)| {
-                        let sma = sma_iter.unwrap_get_inner(i);
-                        let sd = sd_iter.unwrap_get_inner(i);
-                        (i, (ratio - sma) / sd)
-                    },
+                    self.ratio_sma.dateindex.as_ref().unwrap(),
+                    self.ratio_sd.dateindex.as_ref().unwrap(),
+                    exit,
+                )
+            },
+        )?;
+
+        self.ratio_4y_zscore.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, _, starting_indexes, exit| {
+                vec.compute_zscore(
+                    starting_indexes.dateindex,
+                    self.ratio.dateindex.as_ref().unwrap(),
+                    self.ratio_4y_sma.dateindex.as_ref().unwrap(),
+                    self.ratio_4y_sd.dateindex.as_ref().unwrap(),
                     exit,
                 )
             },
@@ -1032,7 +1108,8 @@ impl ComputedRatioVecsFromDateIndex {
 
     fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec<DateIndex, StoredF32>> {
         vec![
-            self.ratio_standard_deviation.dateindex.as_mut().unwrap(),
+            self.ratio_sd.dateindex.as_mut().unwrap(),
+            self.ratio_4y_sd.dateindex.as_mut().unwrap(),
             self.ratio_p99_9.dateindex.as_mut().unwrap(),
             self.ratio_p99_5.dateindex.as_mut().unwrap(),
             self.ratio_p99.dateindex.as_mut().unwrap(),
@@ -1056,8 +1133,10 @@ impl ComputedRatioVecsFromDateIndex {
             self.ratio_1w_sma.vecs(),
             self.ratio_1m_sma.vecs(),
             self.ratio_1y_sma.vecs(),
+            self.ratio_4y_sma.vecs(),
             self.ratio_1y_sma_momentum_oscillator.vecs(),
-            self.ratio_standard_deviation.vecs(),
+            self.ratio_sd.vecs(),
+            self.ratio_4y_sd.vecs(),
             self.ratio_p99_9.vecs(),
             self.ratio_p99_5.vecs(),
             self.ratio_p99.vecs(),
@@ -1083,6 +1162,7 @@ impl ComputedRatioVecsFromDateIndex {
             self.ratio_m2sd_as_price.vecs(),
             self.ratio_m3sd_as_price.vecs(),
             self.ratio_zscore.vecs(),
+            self.ratio_4y_zscore.vecs(),
         ]
         .into_iter()
         .flatten()

crates/brk_computer/src/vecs/grouped/value_from_txindex.rs

@@ -226,7 +226,11 @@ impl ComputedValueVecsFromTxindex {
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
         [
             self.sats.vecs(),
+            vec![&self.bitcoin_txindex as &dyn AnyCollectableVec],
             self.bitcoin.vecs(),
+            self.dollars_txindex
+                .as_ref()
+                .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
             self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
         ]
         .into_iter()

crates/brk_computer/src/vecs/market.rs

@@ -36,11 +36,15 @@ pub struct Vecs {
     pub indexes_to_55d_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_89d_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_144d_sma: ComputedRatioVecsFromDateIndex,
+    pub indexes_to_200d_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_1y_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_2y_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_200w_sma: ComputedRatioVecsFromDateIndex,
     pub indexes_to_4y_sma: ComputedRatioVecsFromDateIndex,
 
+    pub indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex<Dollars>,
+    pub indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex<Dollars>,
+
     pub price_1d_ago: ComputedVecsFromDateIndex<Dollars>,
     pub price_1w_ago: ComputedVecsFromDateIndex<Dollars>,
     pub price_1m_ago: ComputedVecsFromDateIndex<Dollars>,
@@ -306,6 +310,14 @@ impl Vecs {
                 format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
+            indexes_to_200d_sma: ComputedRatioVecsFromDateIndex::forced_import(
+                path,
+                "200d_sma",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
             indexes_to_1y_sma: ComputedRatioVecsFromDateIndex::forced_import(
                 path,
                 "1y_sma",
@@ -1215,6 +1227,23 @@ impl Vecs {
                 format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
+
+            indexes_to_200d_sma_x2_4: ComputedVecsFromDateIndex::forced_import(
+                path,
+                "200d_sma_x2_4",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
+            indexes_to_200d_sma_x0_8: ComputedVecsFromDateIndex::forced_import(
+                path,
+                "200d_sma_x0_8",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
         })
     }
 
@@ -1771,6 +1800,7 @@ impl Vecs {
                 (&mut self.indexes_to_55d_sma, 55),
                 (&mut self.indexes_to_89d_sma, 89),
                 (&mut self.indexes_to_144d_sma, 144),
+                (&mut self.indexes_to_200d_sma, 200),
                 (&mut self.indexes_to_1y_sma, 365),
                 (&mut self.indexes_to_2y_sma, 2 * 365),
                 (&mut self.indexes_to_200w_sma, 200 * 7),
@@ -1797,7 +1827,51 @@ impl Vecs {
                 });
             });
             Ok(())
-        })
+        })?;
+
+        self.indexes_to_200d_sma_x0_8.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.indexes_to_200d_sma
+                        .price
+                        .as_ref()
+                        .unwrap()
+                        .dateindex
+                        .as_ref()
+                        .unwrap(),
+                    |(i, v, ..)| (i, v * 0.8),
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_200d_sma_x2_4.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.indexes_to_200d_sma
+                        .price
+                        .as_ref()
+                        .unwrap()
+                        .dateindex
+                        .as_ref()
+                        .unwrap(),
+                    |(i, v, ..)| (i, v * 2.4),
+                    exit,
+                )
+            },
+        )?;
+
+        Ok(())
     }
 
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -1817,10 +1891,13 @@ impl Vecs {
             self.indexes_to_55d_sma.vecs(),
             self.indexes_to_89d_sma.vecs(),
             self.indexes_to_144d_sma.vecs(),
+            self.indexes_to_200d_sma.vecs(),
             self.indexes_to_1y_sma.vecs(),
             self.indexes_to_2y_sma.vecs(),
             self.indexes_to_200w_sma.vecs(),
             self.indexes_to_4y_sma.vecs(),
+            self.indexes_to_200d_sma_x0_8.vecs(),
+            self.indexes_to_200d_sma_x2_4.vecs(),
             self.price_1d_ago.vecs(),
             self.price_1w_ago.vecs(),
             self.price_1m_ago.vecs(),

crates/brk_computer/src/vecs/mod.rs

@@ -1,4 +1,4 @@
-use std::{fs, path::Path};
+use std::{fs, path::Path, thread};
 
 use brk_core::Version;
 use brk_exit::Exit;
@@ -44,22 +44,31 @@ impl Vecs {
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
-        let indexes = indexes::Vecs::forced_import(
-            path,
-            version + VERSION + Version::ZERO,
-            indexer,
-            computation,
-            format,
-        )?;
+        let (indexes, fetched) = thread::scope(|s| {
+            let indexes_handle = s.spawn(|| {
+                indexes::Vecs::forced_import(
+                    path,
+                    version + VERSION + Version::ZERO,
+                    indexer,
+                    computation,
+                    format,
+                )
+                .unwrap()
+            });
 
-        let fetched = fetch.then(|| {
-            fetched::Vecs::forced_import(
-                path,
-                version + VERSION + Version::ZERO,
-                computation,
-                format,
-            )
-            .unwrap()
+            let fetch_handle = s.spawn(|| {
+                fetch.then(|| {
+                    fetched::Vecs::forced_import(
+                        path,
+                        version + VERSION + Version::ZERO,
+                        computation,
+                        format,
+                    )
+                    .unwrap()
+                })
+            });
+
+            (indexes_handle.join().unwrap(), fetch_handle.join().unwrap())
         });
 
         Ok(Self {
@@ -115,6 +124,7 @@ impl Vecs {
         fetcher: Option<&mut Fetcher>,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
+        info!("Computing indexes...");
         let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
 
         info!("Computing constants...");

crates/brk_computer/src/vecs/stateful/cohort.rs

@@ -57,7 +57,7 @@ pub struct Vecs {
     pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
     pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
     pub dateindex_to_adjusted_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
-    pub dateindex_to_realized_cap_30d_change: Option<EagerVec<DateIndex, Dollars>>,
+    pub indexes_to_realized_cap_30d_change: Option<ComputedVecsFromDateIndex<Dollars>>,
     pub dateindex_to_sell_side_risk_ratio: Option<EagerVec<DateIndex, StoredF32>>,
     pub dateindex_to_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
     pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
@@ -89,6 +89,10 @@ pub struct Vecs {
         Option<EagerVec<Height, StoredF32>>,
     pub indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap:
         Option<ComputedVecsFromDateIndex<StoredF32>>,
+    pub indexes_to_realized_profit_relative_to_realized_cap:
+        Option<ComputedVecsFromHeight<StoredF32>>,
+    pub indexes_to_realized_loss_relative_to_realized_cap:
+        Option<ComputedVecsFromHeight<StoredF32>>,
     pub indexes_to_net_realized_profit_and_loss_relative_to_realized_cap:
         Option<ComputedVecsFromHeight<StoredF32>>,
     pub height_to_supply_even_value: Option<ComputedHeightValueVecs>,
@@ -117,6 +121,12 @@ pub struct Vecs {
         Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_in_profit_relative_to_circulating_supply:
         Option<ComputedVecsFromDateIndex<StoredF64>>,
+    pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change:
+        Option<ComputedVecsFromDateIndex<Dollars>>,
+    pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap:
+        Option<ComputedVecsFromDateIndex<StoredF32>>,
+    pub indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap:
+        Option<ComputedVecsFromDateIndex<StoredF32>>,
 }
 
 impl Vecs {
@@ -432,7 +442,9 @@ impl Vecs {
                     false,
                     version + VERSION + Version::ZERO,
                     format,
-                    StorableVecGeneatorOptions::default().add_sum(),
+                    StorableVecGeneatorOptions::default()
+                        .add_sum()
+                        .add_cumulative(),
                 )
                 .unwrap()
             }),
@@ -452,7 +464,9 @@ impl Vecs {
                     false,
                     version + VERSION + Version::ZERO,
                     format,
-                    StorableVecGeneatorOptions::default().add_sum(),
+                    StorableVecGeneatorOptions::default()
+                        .add_sum()
+                        .add_cumulative(),
                 )
                 .unwrap()
             }),
@@ -463,7 +477,7 @@ impl Vecs {
                     true,
                     version + VERSION + Version::ONE,
                     format,
-                    StorableVecGeneatorOptions::default().add_sum(),
+                    StorableVecGeneatorOptions::default().add_sum().add_cumulative(),
                 )
                 .unwrap()
             }),
@@ -558,12 +572,14 @@ impl Vecs {
                 )
                 .unwrap()
             }),
-            dateindex_to_realized_cap_30d_change: compute_dollars.then(|| {
-                EagerVec::forced_import(
+            indexes_to_realized_cap_30d_change: compute_dollars.then(|| {
+                ComputedVecsFromDateIndex::forced_import(
                     path,
                     &suffix("realized_cap_30d_change"),
+                    true,
                     version + VERSION + Version::ZERO,
                     format,
+                    StorableVecGeneatorOptions::default().add_last(),
                 )
                 .unwrap()
             }),
@@ -574,7 +590,9 @@ impl Vecs {
                     true,
                     version + VERSION + Version::ZERO,
                     format,
-                    StorableVecGeneatorOptions::default().add_sum(),
+                    StorableVecGeneatorOptions::default()
+                        .add_sum()
+                        .add_cumulative(),
                 )
                 .unwrap()
             }),
@@ -686,15 +704,37 @@ impl Vecs {
                     .unwrap()
                 },
             ),
+            indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| {
+                ComputedVecsFromHeight::forced_import(
+                    path,
+                    &suffix("realized_profit_relative_to_realized_cap"),
+                    true,
+                    version + VERSION + Version::ZERO,
+                    format,
+                    StorableVecGeneatorOptions::default().add_sum(),
+                )
+                .unwrap()
+            }),
+            indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| {
+                ComputedVecsFromHeight::forced_import(
+                    path,
+                    &suffix("realized_loss_relative_to_realized_cap"),
+                    true,
+                    version + VERSION + Version::ZERO,
+                    format,
+                    StorableVecGeneatorOptions::default().add_sum(),
+                )
+                .unwrap()
+            }),
             indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then(
                 || {
                     ComputedVecsFromHeight::forced_import(
                         path,
                         &suffix("net_realized_profit_and_loss_relative_to_realized_cap"),
                         true,
-                        version + VERSION + Version::ZERO,
+                        version + VERSION + Version::ONE,
                         format,
-                        StorableVecGeneatorOptions::default().add_last(),
+                        StorableVecGeneatorOptions::default().add_sum(),
                     )
                     .unwrap()
                 },
@@ -903,6 +943,39 @@ impl Vecs {
                 format,
                 StorableVecGeneatorOptions::default().add_sum(),
             )?,
+            indexes_to_cumulative_net_realized_profit_and_loss_30d_change: compute_dollars.then(|| {
+                ComputedVecsFromDateIndex::forced_import(
+                    path,
+                    &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change")),
+                    true,
+                    version + VERSION + Version::new(3),
+                    format,
+                    StorableVecGeneatorOptions::default().add_last()
+                )
+                .unwrap()
+            }),
+            indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap: compute_dollars.then(|| {
+                ComputedVecsFromDateIndex::forced_import(
+                    path,
+                    &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_realized_cap")),
+                    true,
+                    version + VERSION + Version::new(3),
+                    format,
+                    StorableVecGeneatorOptions::default().add_last()
+                )
+                .unwrap()
+            }),
+            indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap: compute_dollars.then(|| {
+                ComputedVecsFromDateIndex::forced_import(
+                    path,
+                    &format!("cumulative_{}", suffix("net_realized_profit_and_loss_30d_change_relative_to_market_cap")),
+                    true,
+                    version + VERSION + Version::new(3),
+                    format,
+                    StorableVecGeneatorOptions::default().add_last()
+                )
+                .unwrap()
+            }),
         })
     }
 
@@ -1877,6 +1950,7 @@ impl Vecs {
         height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
         dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
         height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
+        dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
         if let Some(v) = self
@@ -2021,18 +2095,26 @@ impl Vecs {
                     Some(self.height_to_adjusted_value_destroyed.as_ref().unwrap()),
                 )?;
 
-            self.dateindex_to_realized_cap_30d_change
+            self.indexes_to_realized_cap_30d_change
                 .as_mut()
                 .unwrap()
-                .compute_change(
-                    starting_indexes.dateindex,
-                    self.indexes_to_realized_cap
-                        .as_ref()
-                        .unwrap()
-                        .dateindex
-                        .unwrap_last(),
-                    30,
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
                     exit,
+                    |vec, _, _, starting_indexes, exit| {
+                        vec.compute_change(
+                            starting_indexes.dateindex,
+                            self.indexes_to_realized_cap
+                                .as_ref()
+                                .unwrap()
+                                .dateindex
+                                .unwrap_last(),
+                            30,
+                            exit,
+                        )
+                    },
                 )?;
 
             self.indexes_to_net_realized_profit_and_loss
@@ -2280,6 +2362,42 @@ impl Vecs {
                     },
                 )?;
 
+            self.indexes_to_realized_profit_relative_to_realized_cap
+                .as_mut()
+                .unwrap()
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |vec, _, _, starting_indexes, exit| {
+                        vec.compute_percentage(
+                            starting_indexes.height,
+                            self.height_to_realized_profit.as_ref().unwrap(),
+                            *height_to_realized_cap.as_ref().unwrap(),
+                            exit,
+                        )
+                    },
+                )?;
+
+            self.indexes_to_realized_loss_relative_to_realized_cap
+                .as_mut()
+                .unwrap()
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |vec, _, _, starting_indexes, exit| {
+                        vec.compute_percentage(
+                            starting_indexes.height,
+                            self.height_to_realized_loss.as_ref().unwrap(),
+                            *height_to_realized_cap.as_ref().unwrap(),
+                            exit,
+                        )
+                    },
+                )?;
+
             self.indexes_to_net_realized_profit_and_loss_relative_to_realized_cap
                 .as_mut()
                 .unwrap()
@@ -2435,6 +2553,64 @@ impl Vecs {
                     },
                 )?;
 
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change
+                .as_mut()
+                .unwrap()
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |v, _, _, starting_indexes, exit| {
+                        v.compute_change(
+                            starting_indexes.dateindex,
+                            self.indexes_to_net_realized_profit_and_loss
+                                .as_ref()
+                                .unwrap()
+                                .dateindex
+                                .unwrap_cumulative(),
+                            30,
+                            exit,
+                        )
+                    },
+                )?;
+
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap.
+                as_mut()
+                .unwrap()
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |v, _, _, starting_indexes, exit| {
+                        v.compute_percentage(
+                            starting_indexes.dateindex,
+                            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
+                            *dateindex_to_realized_cap.as_ref().unwrap(),
+                            exit,
+                        )
+                    },
+                )?;
+
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap.
+                as_mut()
+                .unwrap()
+                .compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |v, _, _, starting_indexes, exit| {
+                        v.compute_percentage(
+                            starting_indexes.dateindex,
+                            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref().unwrap().dateindex.as_ref().unwrap(),
+                            market.indexes_to_marketcap.dateindex.as_ref().unwrap(),
+                            exit,
+                        )
+                    },
+                )?;
+
             if let Some(height_to_supply_even_relative_to_circulating_supply) = self
                 .height_to_supply_even_relative_to_circulating_supply
                 .as_mut()
@@ -2618,9 +2794,9 @@ impl Vecs {
             self.indexes_to_adjusted_value_destroyed
                 .as_ref()
                 .map_or(vec![], |v| v.vecs()),
-            self.dateindex_to_realized_cap_30d_change
+            self.indexes_to_realized_cap_30d_change
                 .as_ref()
-                .map_or(vec![], |v| vec![v]),
+                .map_or(vec![], |v| v.vecs()),
             self.indexes_to_net_realized_profit_and_loss
                 .as_ref()
                 .map_or(vec![], |v| v.vecs()),
@@ -2703,6 +2879,12 @@ impl Vecs {
             self.indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap
                 .as_ref()
                 .map_or(vec![], |v| v.vecs()),
+            self.indexes_to_realized_profit_relative_to_realized_cap
+                .as_ref()
+                .map_or(vec![], |v| v.vecs()),
+            self.indexes_to_realized_loss_relative_to_realized_cap
+                .as_ref()
+                .map_or(vec![], |v| v.vecs()),
             self.indexes_to_net_realized_profit_and_loss_relative_to_realized_cap
                 .as_ref()
                 .map_or(vec![], |v| v.vecs()),
@@ -2756,6 +2938,12 @@ impl Vecs {
                 .map_or(vec![], |v| v.vecs()),
             self.indexes_to_coinblocks_destroyed.vecs(),
             self.indexes_to_coindays_destroyed.vecs(),
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change.as_ref()
+            .map_or(vec![], |v| v.vecs()),
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_realized_cap.as_ref()
+            .map_or(vec![], |v| v.vecs()),
+            self.indexes_to_cumulative_net_realized_profit_and_loss_30d_change_relative_to_market_cap.as_ref()
+            .map_or(vec![], |v| v.vecs()),
         ]
         .into_iter()
         .flatten()

crates/brk_computer/src/vecs/stateful/mod.rs

@@ -4,8 +4,8 @@ use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Result, S
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{
-    AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Computation, EagerVec, Format,
-    GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator,
+    AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec,
+    StoredIndex, StoredVec, UnsafeSlice, VecIterator,
 };
 use log::info;
 use outputs::OutputCohorts;
@@ -1289,7 +1289,7 @@ impl Vecs {
                 base_version + self.height_to_opreturn_supply.inner_version(),
             )?;
 
-        let mut chain_state: Vec<BlockState>;
+        let mut chain_state: Vec<BlockState> = vec![];
         let mut chain_state_starting_height = Height::from(self.chain_state.len());
 
         let stateful_starting_height = match separate_utxo_vecs
@@ -1322,287 +1322,286 @@ impl Vecs {
                     .collect::<Vec<_>>();
                 chain_state_starting_height
             }
-            Ordering::Less => {
-                // todo!("rollback instead");
-                chain_state = vec![];
-                chain_state_starting_height = Height::ZERO;
-                Height::ZERO
-            }
+            Ordering::Less => Height::ZERO,
         };
-        if stateful_starting_height.is_zero() {
-            info!("Starting processing utxos from the start");
-            separate_utxo_vecs
-                .par_iter_mut()
-                .try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
-        }
+
         let starting_height = starting_indexes
             .height
             .min(stateful_starting_height)
             .min(Height::from(self.height_to_unspendable_supply.len()))
             .min(Height::from(self.height_to_opreturn_supply.len()));
 
-        if starting_height == Height::from(height_to_date_fixed.len()) {
-            return Ok(());
+        if starting_height.is_zero() {
+            info!("Starting processing utxos from the start");
+
+            // todo!("rollback instead");
+            chain_state = vec![];
+            chain_state_starting_height = Height::ZERO;
+
+            separate_utxo_vecs
+                .par_iter_mut()
+                .try_for_each(|(_, v)| v.state.price_to_amount.reset())?;
         }
 
-        // ---
-        // INIT
-        // ---
+        if starting_height < Height::from(height_to_date_fixed.len()) {
+            starting_indexes.update_from_height(starting_height, indexes);
 
-        separate_utxo_vecs
-            .par_iter_mut()
-            .for_each(|(_, v)| v.init(starting_height));
+            separate_utxo_vecs
+                .par_iter_mut()
+                .for_each(|(_, v)| v.init(starting_height));
 
-        let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
-            self.height_to_unspendable_supply
-                .into_iter()
-                .unwrap_get_inner(prev_height)
-        } else {
-            Sats::ZERO
-        };
-        let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
-            self.height_to_opreturn_supply
-                .into_iter()
-                .unwrap_get_inner(prev_height)
-        } else {
-            Sats::ZERO
-        };
-
-        let mut height = starting_height;
-        starting_indexes.update_from_height(height, indexes);
-
-        (height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
-            .map(Height::from)
-            .try_for_each(|_height| -> color_eyre::Result<()> {
-                height = _height;
-
-                self.utxos_vecs
-                    .as_mut_separate_vecs()
-                    .iter_mut()
-                    .for_each(|(_, v)| v.state.reset_single_iteration_values());
-
-                info!("Processing chain at {height}...");
-
-                let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
-                let price = height_to_close_iter
-                    .as_mut()
-                    .map(|i| *i.unwrap_get_inner(height));
-                let first_outputindex = height_to_first_outputindex_iter
-                    .unwrap_get_inner(height)
-                    .unwrap_to_usize();
-                let first_inputindex = height_to_first_inputindex_iter
-                    .unwrap_get_inner(height)
-                    .unwrap_to_usize();
-                let output_count = height_to_output_count_iter.unwrap_get_inner(height);
-                let input_count = height_to_input_count_iter.unwrap_get_inner(height);
-
-                let (mut height_to_sent, mut received) = thread::scope(|s| {
-                    if chain_state_starting_height <= height {
-                        s.spawn(|| {
-                            self.utxos_vecs
-                                .tick_tock_next_block(&chain_state, timestamp);
-                        });
-                    }
-
-                    let sent_handle = s.spawn(|| {
-                        // Skip coinbase
-                        (first_inputindex + 1..first_inputindex + *input_count)
-                            .into_par_iter()
-                            .map(InputIndex::from)
-                            .map(|inputindex| {
-                                let outputindex = inputindex_to_outputindex
-                                    .get_or_read(inputindex, &inputindex_to_outputindex_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                let value = outputindex_to_value
-                                    .get_or_read(outputindex, &outputindex_to_value_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                let input_type = outputindex_to_outputtype
-                                    .get_or_read(outputindex, &outputindex_to_outputtype_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                // dbg!(input_type);
-
-                                if input_type.is_unspendable() {
-                                    unreachable!()
-                                }
-
-                                let input_txindex = outputindex_to_txindex
-                                    .get_or_read(outputindex, &outputindex_to_txindex_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                let height = txindex_to_height
-                                    .get_or_read(input_txindex, &txindex_to_height_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                (height, value, input_type)
-                            })
-                            .fold(
-                                BTreeMap::<Height, Transacted>::default,
-                                |mut tree, (height, value, input_type)| {
-                                    tree.entry(height).or_default().iterate(value, input_type);
-                                    tree
-                                },
-                            )
-                            .reduce(BTreeMap::<Height, Transacted>::default, |first, second| {
-                                let (mut source, to_consume) = if first.len() > second.len() {
-                                    (first, second)
-                                } else {
-                                    (second, first)
-                                };
-                                to_consume.into_iter().for_each(|(k, v)| {
-                                    *source.entry(k).or_default() += v;
-                                });
-                                source
-                            })
-                    });
-
-                    let received_handle = s.spawn(|| {
-                        (first_outputindex..first_outputindex + *output_count)
-                            .into_par_iter()
-                            .map(OutputIndex::from)
-                            .map(|outputindex| {
-                                let value = outputindex_to_value
-                                    .get_or_read(outputindex, &outputindex_to_value_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                let output_type = outputindex_to_outputtype
-                                    .get_or_read(outputindex, &outputindex_to_outputtype_mmap)
-                                    .unwrap()
-                                    .unwrap()
-                                    .into_inner();
-
-                                (value, output_type)
-                            })
-                            .fold(
-                                Transacted::default,
-                                |mut transacted, (value, output_type)| {
-                                    transacted.iterate(value, output_type);
-                                    transacted
-                                },
-                            )
-                            .reduce(Transacted::default, |acc, transacted| acc + transacted)
-                    });
-
-                    (sent_handle.join().unwrap(), received_handle.join().unwrap())
-                });
-
-                unspendable_supply += received
-                    .by_type
-                    .unspendable
-                    .as_vec()
+            let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
+                self.height_to_unspendable_supply
                     .into_iter()
-                    .map(|state| state.value)
-                    .sum::<Sats>()
-                    + height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
-
-                opreturn_supply += received.by_type.unspendable.opreturn.value;
-
-                if height == Height::new(0) {
-                    received = Transacted::default();
-                    unspendable_supply += Sats::FIFTY_BTC;
-                } else if height == Height::new(91_842) || height == Height::new(91_880) {
-                    // Need to destroy invalid coinbases due to duplicate txids
-                    if height == Height::new(91_842) {
-                        height_to_sent.entry(Height::new(91_812)).or_default()
-                    } else {
-                        height_to_sent.entry(Height::new(91_722)).or_default()
-                    }
-                    .iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
-                };
-
-                if chain_state_starting_height <= height {
-                    // Push current block state before processing sends and receives
-                    chain_state.push(BlockState {
-                        supply: received.spendable_supply.clone(),
-                        price,
-                        timestamp,
-                    });
-
-                    self.utxos_vecs.receive(received, height, price);
-
-                    let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
-
-                    height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
-                        (*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
-                            &sent.spendable_supply;
-                    });
-
-                    self.utxos_vecs.send(height_to_sent, chain_state.as_slice());
-                } else {
-                    dbg!(chain_state_starting_height, height);
-                    panic!("temp, just making sure")
-                }
-
-                let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs();
-
-                separate_utxo_vecs
-                    .iter_mut()
-                    .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
-
-                self.height_to_unspendable_supply.forced_push_at(
-                    height,
-                    unspendable_supply,
-                    exit,
-                )?;
-
+                    .unwrap_get_inner(prev_height)
+            } else {
+                Sats::ZERO
+            };
+            let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
                 self.height_to_opreturn_supply
-                    .forced_push_at(height, opreturn_supply, exit)?;
+                    .into_iter()
+                    .unwrap_get_inner(prev_height)
+            } else {
+                Sats::ZERO
+            };
 
-                let date = height_to_date_fixed_iter.unwrap_get_inner(height);
-                let dateindex = DateIndex::try_from(date).unwrap();
-                let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
-                let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
-                let is_date_last_height = date_first_height
-                    + Height::from(date_height_count).decremented().unwrap()
-                    == height;
-                let date_price = dateindex_to_close_iter
-                    .as_mut()
-                    .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
+            let mut height = starting_height;
 
-                separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| {
-                    v.compute_then_force_push_unrealized_states(
+            (height.unwrap_to_usize()..height_to_date_fixed.len())
+                .map(Height::from)
+                .try_for_each(|_height| -> color_eyre::Result<()> {
+                    height = _height;
+
+                    self.utxos_vecs
+                        .as_mut_separate_vecs()
+                        .iter_mut()
+                        .for_each(|(_, v)| v.state.reset_single_iteration_values());
+
+                    info!("Processing chain at {height}...");
+
+                    let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
+                    let price = height_to_close_iter
+                        .as_mut()
+                        .map(|i| *i.unwrap_get_inner(height));
+                    let first_outputindex = height_to_first_outputindex_iter
+                        .unwrap_get_inner(height)
+                        .unwrap_to_usize();
+                    let first_inputindex = height_to_first_inputindex_iter
+                        .unwrap_get_inner(height)
+                        .unwrap_to_usize();
+                    let output_count = height_to_output_count_iter.unwrap_get_inner(height);
+                    let input_count = height_to_input_count_iter.unwrap_get_inner(height);
+
+                    let (mut height_to_sent, mut received) = thread::scope(|s| {
+                        if chain_state_starting_height <= height {
+                            s.spawn(|| {
+                                self.utxos_vecs
+                                    .tick_tock_next_block(&chain_state, timestamp);
+                            });
+                        }
+
+                        let sent_handle = s.spawn(|| {
+                            // Skip coinbase
+                            (first_inputindex + 1..first_inputindex + *input_count)
+                                .into_par_iter()
+                                .map(InputIndex::from)
+                                .map(|inputindex| {
+                                    let outputindex = inputindex_to_outputindex
+                                        .get_or_read(inputindex, &inputindex_to_outputindex_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    let value = outputindex_to_value
+                                        .get_or_read(outputindex, &outputindex_to_value_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    let input_type = outputindex_to_outputtype
+                                        .get_or_read(outputindex, &outputindex_to_outputtype_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    // dbg!(input_type);
+
+                                    if input_type.is_unspendable() {
+                                        unreachable!()
+                                    }
+
+                                    let input_txindex = outputindex_to_txindex
+                                        .get_or_read(outputindex, &outputindex_to_txindex_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    let height = txindex_to_height
+                                        .get_or_read(input_txindex, &txindex_to_height_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    (height, value, input_type)
+                                })
+                                .fold(
+                                    BTreeMap::<Height, Transacted>::default,
+                                    |mut tree, (height, value, input_type)| {
+                                        tree.entry(height).or_default().iterate(value, input_type);
+                                        tree
+                                    },
+                                )
+                                .reduce(BTreeMap::<Height, Transacted>::default, |first, second| {
+                                    let (mut source, to_consume) = if first.len() > second.len() {
+                                        (first, second)
+                                    } else {
+                                        (second, first)
+                                    };
+                                    to_consume.into_iter().for_each(|(k, v)| {
+                                        *source.entry(k).or_default() += v;
+                                    });
+                                    source
+                                })
+                        });
+
+                        let received_handle = s.spawn(|| {
+                            (first_outputindex..first_outputindex + *output_count)
+                                .into_par_iter()
+                                .map(OutputIndex::from)
+                                .map(|outputindex| {
+                                    let value = outputindex_to_value
+                                        .get_or_read(outputindex, &outputindex_to_value_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    let output_type = outputindex_to_outputtype
+                                        .get_or_read(outputindex, &outputindex_to_outputtype_mmap)
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner();
+
+                                    (value, output_type)
+                                })
+                                .fold(
+                                    Transacted::default,
+                                    |mut transacted, (value, output_type)| {
+                                        transacted.iterate(value, output_type);
+                                        transacted
+                                    },
+                                )
+                                .reduce(Transacted::default, |acc, transacted| acc + transacted)
+                        });
+
+                        (sent_handle.join().unwrap(), received_handle.join().unwrap())
+                    });
+
+                    unspendable_supply += received
+                        .by_type
+                        .unspendable
+                        .as_vec()
+                        .into_iter()
+                        .map(|state| state.value)
+                        .sum::<Sats>()
+                        + height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
+
+                    opreturn_supply += received.by_type.unspendable.opreturn.value;
+
+                    if height == Height::new(0) {
+                        received = Transacted::default();
+                        unspendable_supply += Sats::FIFTY_BTC;
+                    } else if height == Height::new(91_842) || height == Height::new(91_880) {
+                        // Need to destroy invalid coinbases due to duplicate txids
+                        if height == Height::new(91_842) {
+                            height_to_sent.entry(Height::new(91_812)).or_default()
+                        } else {
+                            height_to_sent.entry(Height::new(91_722)).or_default()
+                        }
+                        .iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
+                    };
+
+                    if chain_state_starting_height <= height {
+                        // Push current block state before processing sends and receives
+                        chain_state.push(BlockState {
+                            supply: received.spendable_supply.clone(),
+                            price,
+                            timestamp,
+                        });
+
+                        self.utxos_vecs.receive(received, height, price);
+
+                        let unsafe_chain_state = UnsafeSlice::new(&mut chain_state);
+
+                        height_to_sent.par_iter().for_each(|(height, sent)| unsafe {
+                            (*unsafe_chain_state.get(height.unwrap_to_usize())).supply -=
+                                &sent.spendable_supply;
+                        });
+
+                        self.utxos_vecs.send(height_to_sent, chain_state.as_slice());
+                    } else {
+                        dbg!(chain_state_starting_height, height);
+                        panic!("temp, just making sure")
+                    }
+
+                    let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs();
+
+                    separate_utxo_vecs
+                        .iter_mut()
+                        .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
+
+                    self.height_to_unspendable_supply.forced_push_at(
                         height,
-                        price,
-                        is_date_last_height.then_some(dateindex),
-                        date_price,
+                        unspendable_supply,
                         exit,
-                    )
+                    )?;
+
+                    self.height_to_opreturn_supply
+                        .forced_push_at(height, opreturn_supply, exit)?;
+
+                    let date = height_to_date_fixed_iter.unwrap_get_inner(height);
+                    let dateindex = DateIndex::try_from(date).unwrap();
+                    let date_first_height =
+                        dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
+                    let date_height_count =
+                        dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
+                    let is_date_last_height = date_first_height
+                        + Height::from(date_height_count).decremented().unwrap()
+                        == height;
+                    let date_price = dateindex_to_close_iter
+                        .as_mut()
+                        .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
+
+                    separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| {
+                        v.compute_then_force_push_unrealized_states(
+                            height,
+                            price,
+                            is_date_last_height.then_some(dateindex),
+                            date_price,
+                            exit,
+                        )
+                    })?;
+
+                    if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 {
+                        info!("Flushing...");
+                        exit.block();
+                        self.flush_states(height, &chain_state, exit)?;
+                        exit.release();
+                    }
+
+                    Ok(())
                 })?;
 
-                if height != Height::ZERO && height.unwrap_to_usize() % 20_000 == 0 {
-                    info!("Flushing...");
-                    exit.block();
-                    self.flush_states(height, &chain_state, exit)?;
-                    exit.release();
-                }
+            exit.block();
 
-                Ok(())
-            })?;
+            info!("Flushing...");
 
-        exit.block();
+            self.flush_states(height, &chain_state, exit)?;
+        }
 
-        info!("Flushing...");
-
-        self.flush_states(height, &chain_state, exit)?;
-
-        info!("Computing overlaping...");
+        info!("Computing overlapping...");
 
         self.utxos_vecs
-            .compute_overlaping_vecs(&starting_indexes, exit)?;
+            .compute_overlapping_vecs(&starting_indexes, exit)?;
 
         info!("Computing rest part 1...");
 
@@ -1625,6 +1624,13 @@ impl Vecs {
             .dateindex
             .clone();
         let height_to_realized_cap = self.utxos_vecs.all.1.height_to_realized_cap.clone();
+        let dateindex_to_realized_cap = self
+            .utxos_vecs
+            .all
+            .1
+            .indexes_to_realized_cap
+            .as_ref()
+            .map(|v| v.dateindex.unwrap_last().clone());
 
         self.utxos_vecs
             .as_mut_vecs()
@@ -1639,6 +1645,7 @@ impl Vecs {
                     &height_to_supply,
                     dateindex_to_supply.as_ref().unwrap(),
                     height_to_realized_cap.as_ref(),
+                    dateindex_to_realized_cap.as_ref(),
                     exit,
                 )
             })?;

crates/brk_computer/src/vecs/stateful/outputs.rs

@@ -14,7 +14,7 @@ pub trait OutputCohorts {
     fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp);
     fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]);
     fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>);
-    fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
+    fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
 }
 
 impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
@@ -172,7 +172,7 @@ impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
             });
     }
 
-    fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
+    fn compute_overlapping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
         let by_date_range = self.by_date_range.as_vec();
         let by_size_range = self.by_size_range.as_vec();
 

crates/brk_core/src/structs/dollars.rs

@@ -178,6 +178,17 @@ impl Mul<usize> for Close<Dollars> {
     }
 }
 
+impl Mul<f64> for Dollars {
+    type Output = Dollars;
+    fn mul(self, rhs: f64) -> Self::Output {
+        if rhs.fract() != 0.0 {
+            Self::from(self.0 * rhs)
+        } else {
+            self * rhs as i64
+        }
+    }
+}
+
 impl Mul<Bitcoin> for Dollars {
     type Output = Self;
     fn mul(self, rhs: Bitcoin) -> Self::Output {
@@ -208,11 +219,14 @@ impl Mul<Sats> for Dollars {
 impl Mul<StoredF32> for Dollars {
     type Output = Self;
     fn mul(self, rhs: StoredF32) -> Self::Output {
-        if rhs.fract() != 0.0 {
-            Self::from(self.0 * *rhs as f64)
-        } else {
-            self * *rhs as i64
-        }
+        self * *rhs as f64
+    }
+}
+
+impl Mul<StoredF64> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: StoredF64) -> Self::Output {
+        self * *rhs
     }
 }
 

crates/brk_core/src/structs/ohlc.rs

@@ -1,6 +1,6 @@
 use std::ops::{Add, Div};
 
-use derive_deref::Deref;
+use derive_deref::{Deref, DerefMut};
 use serde::{Serialize, Serializer, ser::SerializeTuple};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
@@ -172,6 +172,7 @@ impl From<Close<Sats>> for OHLCSats {
     IntoBytes,
     KnownLayout,
     Deref,
+    DerefMut,
     Serialize,
 )]
 #[repr(C)]
@@ -259,6 +260,7 @@ where
     IntoBytes,
     KnownLayout,
     Deref,
+    DerefMut,
     Serialize,
 )]
 #[repr(C)]
@@ -346,6 +348,7 @@ where
     IntoBytes,
     KnownLayout,
     Deref,
+    DerefMut,
     Serialize,
 )]
 #[repr(C)]
@@ -433,6 +436,7 @@ where
     IntoBytes,
     KnownLayout,
     Deref,
+    DerefMut,
     Serialize,
 )]
 #[repr(C)]

crates/brk_core/src/structs/version.rs

@@ -33,7 +33,9 @@ impl Version {
         Self(self.0.swap_bytes())
     }
 
-    pub fn validate(&self, path: &Path) -> Result<()> {
+    /// Ok(true) if existed and is same
+    /// Ok(false) if didn't exist
+    pub fn validate(&self, path: &Path) -> Result<bool> {
         if let Ok(prev_version) = Version::try_from(path) {
             if prev_version != *self {
                 if prev_version.swap_bytes() == *self {
@@ -44,9 +46,11 @@ impl Version {
                     expected: *self,
                 });
             }
-        }
 
-        Ok(())
+            Ok(true)
+        } else {
+            Ok(false)
+        }
     }
 }
 

crates/brk_core/src/utils/mod.rs

@@ -3,9 +3,11 @@ mod checked_sub;
 mod paths;
 mod pause;
 mod rlimit;
+mod serde;
 
 pub use bytes::*;
 pub use checked_sub::*;
 pub use paths::*;
 pub use pause::*;
 pub use rlimit::*;
+pub use serde::*;

crates/brk_core/src/utils/pause.rs

@@ -3,5 +3,7 @@ use log::info;
 pub fn pause() {
     info!("Press enter to continue...");
     let mut buffer = String::new();
-    std::io::stdin().read_line(&mut buffer).expect("Failed to read line");
+    std::io::stdin()
+        .read_line(&mut buffer)
+        .expect("Failed to read line");
 }

crates/brk_core/src/utils/serde.rs

@@ -0,0 +1,12 @@
+use serde::{Deserialize, Deserializer};
+
+pub fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: Deserializer<'de>,
+    T: Deserialize<'de> + Default,
+{
+    match T::deserialize(deserializer) {
+        Ok(v) => Ok(v),
+        Err(_) => Ok(T::default()),
+    }
+}

crates/brk_fetcher/examples/main.rs

@@ -1,5 +1,5 @@
 use brk_core::{Date, Height};
-use brk_fetcher::{BRK, Fetcher};
+use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -12,6 +12,19 @@ fn main() -> color_eyre::Result<()> {
 
     let mut fetcher = Fetcher::import(None)?;
 
+    Binance::fetch_1d().map(|b| {
+        dbg!(b.last_key_value());
+    })?;
+    Kraken::fetch_1d().map(|b| {
+        dbg!(b.last_key_value());
+    })?;
+    Binance::fetch_1mn().map(|b| {
+        dbg!(b.last_key_value());
+    })?;
+    Kraken::fetch_1mn().map(|b| {
+        dbg!(b.last_key_value());
+    })?;
+
     dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
     dbg!(fetcher.get_height(
         899911_u32.into(),

crates/brk_fetcher/src/fetchers/kraken.rs

@@ -32,7 +32,7 @@ impl Kraken {
         )
     }
 
-    fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
+    pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
         info!("Fetching 1mn prices from Kraken...");
 
         retry(
@@ -54,7 +54,7 @@ impl Kraken {
             .ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
     }
 
-    fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
+    pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
         info!("Fetching daily prices from Kraken...");
 
         retry(

crates/brk_fetcher/src/lib.rs

@@ -40,11 +40,11 @@ impl Fetcher {
     }
 
     fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
-        self.binance
+        self.kraken
             .get_from_1d(&date)
             .or_else(|_| {
                 // eprintln!("{e}");
-                self.kraken.get_from_1d(&date)
+                self.binance.get_from_1d(&date)
             })
             .or_else(|_| {
                 // eprintln!("{e}");
@@ -90,11 +90,11 @@ impl Fetcher {
         let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
 
         let ohlc = self
-            .binance
+            .kraken
             .get_from_1mn(timestamp, previous_timestamp)
             .unwrap_or_else(|_report| {
                 // eprintln!("{_report}");
-                self.kraken
+                self.binance
                     .get_from_1mn(timestamp, previous_timestamp)
                     .unwrap_or_else(|_report| {
                         //         // eprintln!("{_report}");
@@ -185,8 +185,8 @@ How to fix this:
     }
 
     pub fn clear(&mut self) {
+        self.kraken.clear();
         self.binance.clear();
         self.brk.clear();
-        self.kraken.clear();
     }
 }

crates/brk_indexer/examples/main.rs

@@ -4,7 +4,6 @@ use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_parser::Parser;
-use brk_vec::Format;
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -25,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
 
     let outputs = Path::new("../../_outputs");
 
-    let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
+    let mut indexer = Indexer::new(outputs, false)?;
 
     indexer.import_stores()?;
     indexer.import_vecs()?;

crates/brk_indexer/src/indexes.rs

@@ -110,13 +110,7 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
                 vecs.height_to_blockhash
                     .iter()
                     .get(*height)
-                    .is_none_or(|saved_blockhash| {
-                        let b = &rpc_blockhash != saved_blockhash.as_ref();
-                        if b {
-                            dbg!(rpc_blockhash, saved_blockhash.as_ref());
-                        }
-                        b
-                    })
+                    .is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref())
             })
             .unwrap_or(starting_height);
 

crates/brk_indexer/src/lib.rs

@@ -19,7 +19,7 @@ use brk_core::{
 use bitcoin::{Transaction, TxIn, TxOut};
 use brk_exit::Exit;
 use brk_parser::Parser;
-use brk_vec::{AnyVec, Format, VecIterator};
+use brk_vec::{AnyVec, VecIterator};
 use color_eyre::eyre::{ContextCompat, eyre};
 use fjall::TransactionalKeyspace;
 use log::{error, info};
@@ -42,21 +42,15 @@ pub struct Indexer {
     vecs: Option<Vecs>,
     stores: Option<Stores>,
     check_collisions: bool,
-    format: Format,
 }
 
 impl Indexer {
-    pub fn new(
-        outputs_dir: &Path,
-        format: Format,
-        check_collisions: bool,
-    ) -> color_eyre::Result<Self> {
+    pub fn new(outputs_dir: &Path, check_collisions: bool) -> color_eyre::Result<Self> {
         setrlimit()?;
         Ok(Self {
             path: outputs_dir.to_owned(),
             vecs: None,
             stores: None,
-            format,
             check_collisions,
         })
     }
@@ -65,7 +59,6 @@ impl Indexer {
         self.vecs = Some(Vecs::forced_import(
             &self.path.join("vecs/indexed"),
             VERSION + Version::ZERO,
-            self.format,
         )?);
         Ok(())
     }

crates/brk_indexer/src/vecs.rs

@@ -66,11 +66,7 @@ pub struct Vecs {
 }
 
 impl Vecs {
-    pub fn forced_import(
-        path: &Path,
-        version: Version,
-        format: Format,
-    ) -> color_eyre::Result<Self> {
+    pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
         Ok(Self {
@@ -78,7 +74,7 @@ impl Vecs {
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_blockhash: IndexedVec::forced_import(
                 path,
@@ -90,145 +86,145 @@ impl Vecs {
                 path,
                 "difficulty",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_emptyoutputindex: IndexedVec::forced_import(
                 path,
                 "first_emptyoutputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_inputindex: IndexedVec::forced_import(
                 path,
                 "first_inputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_opreturnindex: IndexedVec::forced_import(
                 path,
                 "first_opreturnindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_outputindex: IndexedVec::forced_import(
                 path,
                 "first_outputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2aindex: IndexedVec::forced_import(
                 path,
                 "first_p2aindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2msindex: IndexedVec::forced_import(
                 path,
                 "first_p2msindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pk33index: IndexedVec::forced_import(
                 path,
                 "first_p2pk33index",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pk65index: IndexedVec::forced_import(
                 path,
                 "first_p2pk65index",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2pkhindex: IndexedVec::forced_import(
                 path,
                 "first_p2pkhindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2shindex: IndexedVec::forced_import(
                 path,
                 "first_p2shindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2trindex: IndexedVec::forced_import(
                 path,
                 "first_p2trindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2wpkhindex: IndexedVec::forced_import(
                 path,
                 "first_p2wpkhindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_p2wshindex: IndexedVec::forced_import(
                 path,
                 "first_p2wshindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_txindex: IndexedVec::forced_import(
                 path,
                 "first_txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_first_unknownoutputindex: IndexedVec::forced_import(
                 path,
                 "first_unknownoutputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_timestamp: IndexedVec::forced_import(
                 path,
                 "timestamp",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_total_size: IndexedVec::forced_import(
                 path,
                 "total_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             height_to_weight: IndexedVec::forced_import(
                 path,
                 "weight",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             inputindex_to_outputindex: IndexedVec::forced_import(
                 path,
                 "outputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             opreturnindex_to_txindex: IndexedVec::forced_import(
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_outputtype: IndexedVec::forced_import(
                 path,
                 "outputtype",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_outputtypeindex: IndexedVec::forced_import(
                 path,
                 "outputtypeindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             outputindex_to_value: IndexedVec::forced_import(
                 path,
                 "value",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             p2aindex_to_p2abytes: IndexedVec::forced_import(
                 path,
@@ -240,7 +236,7 @@ impl Vecs {
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
                 path,
@@ -288,13 +284,13 @@ impl Vecs {
                 path,
                 "base_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_first_inputindex: IndexedVec::forced_import(
                 path,
                 "first_inputindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_first_outputindex: IndexedVec::forced_import(
                 path,
@@ -306,19 +302,19 @@ impl Vecs {
                 path,
                 "is_explicitly_rbf",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_rawlocktime: IndexedVec::forced_import(
                 path,
                 "rawlocktime",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_total_size: IndexedVec::forced_import(
                 path,
                 "total_size",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             txindex_to_txid: IndexedVec::forced_import(
                 path,
@@ -330,13 +326,13 @@ impl Vecs {
                 path,
                 "txversion",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
             unknownoutputindex_to_txindex: IndexedVec::forced_import(
                 path,
                 "txindex",
                 version + VERSION + Version::ZERO,
-                format,
+                Format::Raw,
             )?,
         })
     }

crates/brk_logger/src/lib.rs

@@ -25,50 +25,53 @@ pub fn init(path: Option<&Path>) {
             .unwrap()
     });
 
-    Builder::from_env(Env::default().default_filter_or("info,fjall=off,lsm_tree=off"))
-        .format(move |buf, record| {
-            let date_time = Timestamp::now()
-                .to_zoned(tz::TimeZone::system())
-                .strftime("%Y-%m-%d %H:%M:%S")
-                .to_string();
-            let level = record.level().as_str().to_lowercase();
-            let level = format!("{:5}", level);
-            let target = record.target();
-            let dash = "-";
-            let args = record.args();
+    Builder::from_env(
+        Env::default()
+            .default_filter_or("info,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off"),
+    )
+    .format(move |buf, record| {
+        let date_time = Timestamp::now()
+            .to_zoned(tz::TimeZone::system())
+            .strftime("%Y-%m-%d %H:%M:%S")
+            .to_string();
+        let level = record.level().as_str().to_lowercase();
+        let level = format!("{:5}", level);
+        let target = record.target();
+        let dash = "-";
+        let args = record.args();
 
-            if let Some(file) = file.as_ref() {
-                let _ = write(
-                    file.try_clone().unwrap(),
-                    &date_time,
-                    target,
-                    &level,
-                    dash,
-                    args,
-                );
-            }
-
-            let colored_date_time = date_time.bright_black();
-            let colored_level = match level.chars().next().unwrap() {
-                'e' => level.red().to_string(),
-                'w' => level.yellow().to_string(),
-                'i' => level.green().to_string(),
-                'd' => level.blue().to_string(),
-                't' => level.cyan().to_string(),
-                _ => panic!(),
-            };
-            let colored_dash = dash.bright_black();
-
-            write(
-                buf,
-                colored_date_time,
+        if let Some(file) = file.as_ref() {
+            let _ = write(
+                file.try_clone().unwrap(),
+                &date_time,
                 target,
-                colored_level,
-                colored_dash,
+                &level,
+                dash,
                 args,
-            )
-        })
-        .init();
+            );
+        }
+
+        let colored_date_time = date_time.bright_black();
+        let colored_level = match level.chars().next().unwrap() {
+            'e' => level.red().to_string(),
+            'w' => level.yellow().to_string(),
+            'i' => level.green().to_string(),
+            'd' => level.blue().to_string(),
+            't' => level.cyan().to_string(),
+            _ => panic!(),
+        };
+        let colored_dash = dash.bright_black();
+
+        write(
+            buf,
+            colored_date_time,
+            target,
+            colored_level,
+            colored_dash,
+            args,
+        )
+    })
+    .init();
 }
 
 fn write(
@@ -80,5 +83,9 @@ fn write(
     args: impl Display,
 ) -> Result<(), std::io::Error> {
     writeln!(buf, "{} {} {} {}", date_time, dash, level, args)
-    // writeln!(buf, "{} {} {} {}  {}", date_time, _target, level, dash, args)
+    // writeln!(
+    //     buf,
+    //     "{} {} {} {}  {}",
+    //     date_time, _target, level, dash, args
+    // )
 }

crates/brk_parser/src/blk_index_to_blk_recap.rs

@@ -96,9 +96,10 @@ impl BlkIndexToBlkRecap {
     }
 
     pub fn export(&self) {
-        let file = File::create(&self.path).unwrap_or_else(|_| {
+        let file = File::create(&self.path).unwrap_or_else(|e| {
+            dbg!(e);
             dbg!(&self.path);
-            panic!("No such file or directory")
+            panic!("Cannot write file");
         });
 
         serde_json::to_writer(&mut BufWriter::new(file), &self.tree).unwrap();

crates/brk_query/Cargo.toml

@@ -18,5 +18,5 @@ color-eyre = { workspace = true }
 derive_deref = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-serde_with = "3.12.0"
+serde_with = "3.13.0"
 tabled = { workspace = true }

crates/brk_query/examples/main.rs

@@ -12,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {
 
     let format = Format::Compressed;
 
-    let mut indexer = Indexer::new(outputs_dir, format, true)?;
+    let mut indexer = Indexer::new(outputs_dir, true)?;
     indexer.import_vecs()?;
 
     let mut computer = Computer::new(outputs_dir, None, format);

crates/brk_query/src/lib.rs

@@ -19,7 +19,7 @@ mod vec_trees;
 pub use format::Format;
 pub use index::Index;
 pub use output::{Output, Value};
-pub use params::Params;
+pub use params::{Params, ParamsOpt};
 pub use table::Tabled;
 use vec_trees::VecTrees;
 

crates/brk_query/src/params.rs

@@ -1,6 +1,8 @@
+use std::{fmt::Display, ops::Deref, str::FromStr};
+
 use clap::builder::PossibleValuesParser;
 use clap_derive::Parser;
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 use serde_with::{OneOrMany, formats::PreferOne, serde_as};
 
 use crate::{Format, Index};
@@ -17,15 +19,111 @@ pub struct Params {
     #[serde_as(as = "OneOrMany<_, PreferOne>")]
     /// Names of the values requested
     pub values: Vec<String>,
+
+    #[clap(flatten)]
+    #[serde(flatten)]
+    pub rest: ParamsOpt,
+}
+
+// The macro creates custom deserialization code.
+// You need to specify a function name and the field name of the flattened field.
+serde_with::flattened_maybe!(deserialize_rest, "rest");
+
+impl Deref for Params {
+    type Target = ParamsOpt;
+    fn deref(&self) -> &Self::Target {
+        &self.rest
+    }
+}
+
+impl From<((String, String), ParamsOpt)> for Params {
+    fn from(((index, id), rest): ((String, String), ParamsOpt)) -> Self {
+        Self {
+            index,
+            values: vec![id],
+            rest,
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize, Parser)]
+pub struct ParamsOpt {
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(alias = "f")]
+    #[serde(default, alias = "f", deserialize_with = "de_unquote_i64")]
     /// Inclusive starting index, if negative will be from the end
-    pub from: Option<i64>,
+    from: Option<i64>,
     #[clap(short, long, allow_hyphen_values = true)]
-    #[serde(default, alias = "t")]
-    /// Inclusive ending index, if negative will be from the end
-    pub to: Option<i64>,
+    #[serde(default, alias = "t", deserialize_with = "de_unquote_i64")]
+    /// Exclusive ending index, if negative will be from the end, overrides 'count'
+    to: Option<i64>,
+    #[clap(short, long, allow_hyphen_values = true)]
+    #[serde(default, alias = "c", deserialize_with = "de_unquote_usize")]
+    /// Number of values
+    count: Option<usize>,
     #[clap(short = 'F', long)]
     /// Format of the output
-    pub format: Option<Format>,
+    format: Option<Format>,
+}
+
+impl ParamsOpt {
+    pub fn from(&self) -> Option<i64> {
+        self.from
+    }
+
+    pub fn to(&self) -> Option<i64> {
+        if self.to.is_none() {
+            if let Some(c) = self.count {
+                let c = c as i64;
+                if let Some(f) = self.from {
+                    if f.is_positive() || f.abs() > c {
+                        return Some(f + c);
+                    }
+                } else {
+                    return Some(c);
+                }
+            }
+        }
+        self.to
+    }
+
+    pub fn format(&self) -> Option<Format> {
+        self.format
+    }
+}
+
+fn de_unquote_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote_usize<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    de_unquote(deserializer)
+}
+
+fn de_unquote<'de, D, F>(deserializer: D) -> Result<Option<F>, D::Error>
+where
+    D: Deserializer<'de>,
+    F: FromStr + Display,
+    <F as std::str::FromStr>::Err: Display,
+{
+    let opt: Option<String> = Option::deserialize(deserializer)?;
+    let s = match opt {
+        None => return Ok(None),
+        Some(mut s) => {
+            // strip any leading/trailing quotes
+            if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
+                s = s[1..s.len() - 1].to_string();
+            }
+            s
+        }
+    };
+    s.parse::<F>()
+        .map(Some)
+        .map_err(|e| serde::de::Error::custom(format!("cannot parse `{}` as type: {}", s, e)))
 }

crates/brk_query/src/vec_trees.rs

@@ -28,9 +28,14 @@ impl<'a> VecTrees<'a> {
                         || s.starts_with("cumulative_from")
                 }))
             && !(split.len() == 4
-                && split
-                    .get(1)
-                    .is_some_and(|s| s == &"up" || s == &"start" || s.starts_with("from"))
+                && split.get(1).is_some_and(|s| {
+                    s == &"up"
+                        || s == &"start"
+                        || s.starts_with("from")
+                        || s == &"cumulative_up"
+                        || s == &"cumulative_start"
+                        || s.starts_with("cumulative_from")
+                })
                 && split.get(2).is_some_and(|s| s.ends_with("relative")))
         {
             dbg!(&name, &split);

crates/brk_server/Cargo.toml

@@ -10,6 +10,7 @@ repository.workspace = true
 [dependencies]
 axum = { workspace = true }
 bitcoincore-rpc = { workspace = true }
+brk_bundler = { workspace = true }
 brk_computer = { workspace = true }
 brk_exit = { workspace = true }
 brk_core = { workspace = true }
@@ -25,11 +26,11 @@ color-eyre = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
-oxc = { version = "0.72.3", features = ["codegen", "minifier"] }
+oxc = { version = "0.73.0", features = ["codegen", "minifier"] }
 serde = { workspace = true }
 tokio = { workspace = true }
 tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }
-zip = "4.0.0"
+zip = "4.1.0"
 tracing = "0.1.41"
 
 [package.metadata.cargo-machete]

crates/brk_server/README.md

@@ -41,22 +41,59 @@ The API uses `brk_query` and so inherites all of its features including formats.
 
 ### API
 
-#### `GET /api/vecs/indexes`
+#### [`GET /api/vecs/index-count`](https://bitcoinresearchkit.org/api/vecs/index-count)
+
+Count of all possible indexes
+
+#### [`GET /api/vecs/id-count`](https://bitcoinresearchkit.org/api/vecs/id-count)
+
+Count of all possible ids
+
+#### [`GET /api/vecs/variant-count`](https://bitcoinresearchkit.org/api/vecs/variant-count)
+
+Count of all possible variants
+
+#### [`GET /api/vecs/indexes`](https://bitcoinresearchkit.org/api/vecs/indexes)
 
 A list of all possible vec indexes and their accepted variants
 
-#### `GET /api/vecs/ids`
+#### [`GET /api/vecs/ids`](https://bitcoinresearchkit.org/api/vecs/ids)
 
 A list of all possible vec ids
 
-#### `GET /api/vecs/id-to-indexes`
+#### [`GET /api/vecs/variants`](https://bitcoinresearchkit.org/api/vecs/variants)
+
+A list of all possible variants
+
+#### [`GET /api/vecs/id-to-indexes`](https://bitcoinresearchkit.org/api/vecs/id-to-indexes)
 
 A list of all possible vec ids and their supported vec indexes
 
-#### `GET /api/vecs/index-to-ids`
+#### [`GET /api/vecs/index-to-ids`](https://bitcoinresearchkit.org/api/vecs/index-to-ids)
 
 A list of all possible vec indexes and their supported vec ids
 
+#### `GET /api/{INDEX}-to-{ID}`
+
+This endpoint retrieves data based on the specified vector index and id.
+
+**Parameters:**
+
+| Parameter | Type | Required | Description |
+| --- | --- | --- | --- |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
+| `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
+
+**Examples:**
+
+```
+GET /api/date-to-close
+GET /date-to-close?from=-100
+GET /date-to-close?count=100&format=csv
+```
+
 #### `GET /api/query`
 
 This endpoint retrieves data based on the specified vector index and values.
@@ -67,8 +104,9 @@ This endpoint retrieves data based on the specified vector index and values.
 | --- | --- | --- | --- |
 | `index` | `VecIndex` | Yes | The vector index to query. |
 | `values` | `VecId[]` | Yes | A comma or space-separated list of vector IDs to retrieve. |
-| `from` | `unsigned int` | No | The starting index for pagination (default is 0). |
-| `to` | `unsigned int` | No | The ending index for pagination (default is the total number of results). |
+| `from` | `signed int` | No | Inclusive starting index for pagination (default is 0). |
+| `to` | `signed int` | No | Exclusive ending index for pagination (default is the total number of results). Overrides `count` |
+| `count` | `unsigned int` | No | The number of values requested |
 | `format` | `string` | No | The format of the response. Options include `json`, `csv`, `tsv`, or `md` (default is `json`). |
 
 **Examples:**
@@ -80,7 +118,7 @@ GET /api/query?index=week&values=ohlc,block-interval-average&from=0&to=20&format
 
 ### Meta
 
-#### `GET /version`
+#### [`GET /version`](https://bitcoinresearchkit.org/version)
 
 The version of the server and thus BRK.
 

crates/brk_server/examples/main.rs

@@ -31,7 +31,7 @@ pub fn main() -> color_eyre::Result<()> {
 
     let format = Format::Compressed;
 
-    let mut indexer = Indexer::new(outputs_dir, format, true)?;
+    let mut indexer = Indexer::new(outputs_dir, true)?;
     indexer.import_stores()?;
     indexer.import_vecs()?;
 
@@ -51,7 +51,7 @@ pub fn main() -> color_eyre::Result<()> {
             let server = Server::new(served_indexer, served_computer, Website::Default)?;
 
             let server = tokio::spawn(async move {
-                server.serve().await.unwrap();
+                server.serve(true).await.unwrap();
             });
 
             if process {

crates/brk_server/src/api/explorer/mod.rs

@@ -0,0 +1 @@
+

crates/brk_server/src/api/mod.rs

@@ -1,18 +1,20 @@
 use std::collections::BTreeMap;
 
 use axum::{
-    Router,
-    extract::State,
+    Json, Router,
+    extract::{Path, Query, State},
+    http::HeaderMap,
     response::{IntoResponse, Redirect, Response},
     routing::get,
 };
+use brk_query::{Params, ParamsOpt};
 
 use super::AppState;
 
 mod explorer;
 mod query;
 
-pub use query::DTS;
+pub use query::Bridge;
 
 pub trait ApiRoutes {
     fn add_api_routes(self) -> Self;
@@ -20,24 +22,29 @@ pub trait ApiRoutes {
 
 impl ApiRoutes for Router<AppState> {
     fn add_api_routes(self) -> Self {
-        self.route(
-            "/api",
-            get(|| async {
-                Redirect::permanent(
-                    "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
-                )
-            }),
-        )
-        .route("/api/query", get(query::handler))
-        .route("/api/vecs/ids", get(vecids_handler))
-        .route("/api/vecs/indexes", get(vecindexes_handler))
-        .route("/api/vecs/id-to-indexes", get(vecid_to_vecindexes_handler))
-        .route("/api/vecs/index-to-ids", get(vecindex_to_vecids_handler))
+        self.route("/api/query", get(query::handler))
+            .route("/api/vecs/id-count", get(id_count_handler))
+            .route("/api/vecs/index-count", get(index_count_handler))
+            .route("/api/vecs/variant-count", get(variant_count_handler))
+            .route("/api/vecs/ids", get(ids_handler))
+            .route("/api/vecs/indexes", get(indexes_handler))
+            .route("/api/vecs/variants", get(variants_handler))
+            .route("/api/vecs/id-to-indexes", get(id_to_indexes_handler))
+            .route("/api/vecs/index-to-ids", get(index_to_ids_handler))
+            .route("/api/{variant}", get(variant_handler))
+            .route(
+                "/api",
+                get(|| async {
+                    Redirect::temporary(
+                        "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
+                    )
+                }),
+            )
     }
 }
 
-pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -48,8 +55,29 @@ pub async fn vecids_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(
+pub async fn variant_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .values()
+            .map(|tree| tree.len())
+            .sum::<usize>(),
+    )
+    .into_response()
+}
+
+pub async fn id_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.id_to_index_to_vec.keys().count()).into_response()
+}
+
+pub async fn index_count_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.index_to_id_to_vec.keys().count()).into_response()
+}
+
+pub async fn indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(
         app_state
             .query
             .vec_trees
@@ -61,10 +89,49 @@ pub async fn vecindexes_handler(State(app_state): State<AppState>) -> Response {
     .into_response()
 }
 
-pub async fn vecid_to_vecindexes_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+pub async fn variants_handler(State(app_state): State<AppState>) -> Response {
+    Json(
+        app_state
+            .query
+            .vec_trees
+            .index_to_id_to_vec
+            .iter()
+            .flat_map(|(index, id_to_vec)| {
+                let index_ser = index.serialize_long();
+                id_to_vec
+                    .keys()
+                    .map(|id| format!("{}-to-{}", index_ser, id))
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>(),
+    )
+    .into_response()
 }
 
-pub async fn vecindex_to_vecids_handler(State(app_state): State<AppState>) -> Response {
-    axum::Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+pub async fn id_to_indexes_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_id_to_index_to_vec()).into_response()
+}
+
+pub async fn index_to_ids_handler(State(app_state): State<AppState>) -> Response {
+    Json(app_state.query.vec_trees.serialize_index_to_id_to_vec()).into_response()
+}
+
+const TO_SEPARATOR: &str = "-to-";
+
+pub async fn variant_handler(
+    headers: HeaderMap,
+    Path(variant): Path<String>,
+    Query(params_opt): Query<ParamsOpt>,
+    state: State<AppState>,
+) -> Response {
+    let variant = variant.replace("_", "-");
+    let mut split = variant.split(TO_SEPARATOR);
+    let params = Params::from((
+        (
+            split.next().unwrap().to_string(),
+            split.collect::<Vec<_>>().join(TO_SEPARATOR),
+        ),
+        params_opt,
+    ));
+    query::handler(headers, Query(params), state).await
 }

crates/brk_server/src/api/query/bridge.rs

@@ -2,17 +2,17 @@ use std::{fs, io, path::Path};
 
 use brk_query::{Index, Query};
 
-use crate::Website;
+use crate::{VERSION, Website};
 
 const SCRIPTS: &str = "scripts";
 
 #[allow(clippy::upper_case_acronyms)]
-pub trait DTS {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
+pub trait Bridge {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()>;
 }
 
-impl DTS for Query<'static> {
-    fn generate_dts_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
+impl Bridge for Query<'static> {
+    fn generate_bridge_file(&self, website: Website, websites_path: &Path) -> io::Result<()> {
         if website.is_none() {
             return Ok(());
         }
@@ -31,10 +31,16 @@ impl DTS for Query<'static> {
 
         let indexes = Index::all();
 
-        let mut contents = "//
+        let mut contents = format!(
+            "//
 // File auto-generated, any modifications will be overwritten
-//\n\n"
-            .to_string();
+//
+
+export const VERSION = \"v{}\";
+
+",
+            VERSION
+        );
 
         contents += &indexes
             .iter()
@@ -55,9 +61,12 @@ impl DTS for Query<'static> {
                 .join(" | ")
         );
 
-        contents += "\n\nexport function createVecIdToIndexes() {\n";
+        contents += "\n\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
+        contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
 
-        contents += "\n\n  return /** @type {const} */ ({\n";
+        contents += "\nexport function createVecIdToIndexes() {\n";
+
+        contents += "  return {\n";
 
         self.vec_trees
             .id_to_index_to_vec
@@ -73,11 +82,7 @@ impl DTS for Query<'static> {
                 contents += &format!("    \"{id}\": [{indexes}],\n");
             });
 
-        contents += "  });\n";
-        contents.push('}');
-
-        contents += "\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";
-        contents += "\n/** @typedef {keyof VecIdToIndexes} VecId */\n";
+        contents += "  };\n}\n";
 
         fs::write(path, contents)
     }

crates/brk_server/src/api/query/mod.rs

@@ -5,14 +5,18 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use brk_query::{Format, Index, Output, Params};
+use brk_vec::{CollectableVec, StoredVec};
+use color_eyre::eyre::eyre;
 
 use crate::traits::{HeaderMapExtended, ModifiedState, ResponseExtended};
 
 use super::AppState;
 
-mod dts;
+mod bridge;
 
-pub use dts::*;
+pub use bridge::*;
+
+const MAX_WEIGHT: usize = 320_000;
 
 pub async fn handler(
     headers: HeaderMap,
@@ -33,11 +37,9 @@ pub async fn handler(
 fn req_to_response_res(
     headers: HeaderMap,
     AxumQuery(Params {
-        format,
-        from,
         index,
-        to,
         values,
+        rest,
     }): AxumQuery<Params>,
     AppState { query, .. }: AppState,
 ) -> color_eyre::Result<Response> {
@@ -48,6 +50,27 @@ fn req_to_response_res(
         &values.iter().map(|v| v.as_str()).collect::<Vec<_>>(),
     );
 
+    if vecs.is_empty() {
+        return Ok(Json(vec![] as Vec<usize>).into_response());
+    }
+
+    let from = rest.from();
+    let to = rest.to();
+    let format = rest.format();
+
+    let weight = vecs
+        .iter()
+        .map(|(_, v)| {
+            let len = v.len();
+            let count = StoredVec::<usize, usize>::range_count(from, to, len);
+            count * v.value_type_to_size_of()
+        })
+        .sum::<usize>();
+
+    if weight > MAX_WEIGHT {
+        return Err(eyre!("Request is too heavy, max weight is {MAX_WEIGHT}"));
+    }
+
     let mut date_modified_opt = None;
 
     if to.is_none() {
@@ -75,8 +98,8 @@ fn req_to_response_res(
         Output::TSV(s) => s.into_response(),
         Output::Json(v) => match v {
             brk_query::Value::Single(v) => Json(v).into_response(),
-            brk_query::Value::List(l) => Json(l).into_response(),
-            brk_query::Value::Matrix(m) => Json(m).into_response(),
+            brk_query::Value::List(v) => Json(v).into_response(),
+            brk_query::Value::Matrix(v) => Json(v).into_response(),
         },
         Output::MD(s) => s.into_response(),
     };

crates/brk_server/src/files/file.rs

@@ -13,8 +13,6 @@ use crate::{
     traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
 };
 
-use super::minify::minify_js;
-
 pub async fn file_handler(
     headers: HeaderMap,
     State(app_state): State<AppState>,
@@ -32,16 +30,12 @@ fn any_handler(
     app_state: AppState,
     path: Option<extract::Path<String>>,
 ) -> Response {
-    let website_path = app_state
-        .websites_path
-        .as_ref()
-        .expect("Should never reach here is websites_path is None")
-        .join(app_state.website.to_folder_name());
+    let dist_path = app_state.dist_path();
 
     if let Some(path) = path.as_ref() {
         let path = path.0.replace("..", "").replace("\\", "");
 
-        let mut path = website_path.join(&path);
+        let mut path = dist_path.join(&path);
 
         if !path.exists() || path.is_dir() {
             if path.extension().is_some() {
@@ -55,13 +49,13 @@ fn any_handler(
 
                 return response;
             } else {
-                path = website_path.join("index.html");
+                path = dist_path.join("index.html");
             }
         }
 
         path_to_response(&headers, &path)
     } else {
-        path_to_response(&headers, &website_path.join("index.html"))
+        path_to_response(&headers, &dist_path.join("index.html"))
     }
 }
 
@@ -85,49 +79,35 @@ fn path_to_response_(headers: &HeaderMap, path: &Path) -> color_eyre::Result<Res
         return Ok(Response::new_not_modified());
     }
 
-    let mut response;
+    let content = fs::read(path).unwrap_or_else(|error| {
+        error!("{error}");
+        let path = path.to_str().unwrap();
+        info!("Can't read file {path}");
+        panic!("")
+    });
 
-    let is_localhost = headers.check_if_host_is_localhost();
-
-    if !is_localhost
-        && path.extension().unwrap_or_else(|| {
-            dbg!(path);
-            panic!();
-        }) == "js"
-    {
-        let content = minify_js(path);
-
-        response = Response::new(content.into());
-    } else {
-        let content = fs::read(path).unwrap_or_else(|error| {
-            error!("{error}");
-            let path = path.to_str().unwrap();
-            info!("Can't read file {path}");
-            panic!("")
-        });
-
-        response = Response::new(content.into());
-    }
+    let mut response = Response::new(content.into());
 
     let headers = response.headers_mut();
     headers.insert_cors();
     headers.insert_content_type(path);
 
-    if !is_localhost {
-        let serialized_path = path.to_str().unwrap();
+    let serialized_path = path.to_str().unwrap();
 
-        if serialized_path.contains("fonts/")
-            || serialized_path.contains("assets/")
-            || serialized_path.contains("packages/")
-            || path.extension().is_some_and(|extension| {
-                extension == "pdf"
-                    || extension == "jpg"
-                    || extension == "png"
-                    || extension == "woff2"
-            })
-        {
-            headers.insert_cache_control_immutable();
-        }
+    if path
+        .extension()
+        .is_some_and(|extension| extension == "html")
+        || serialized_path.ends_with("service-worker.js")
+    {
+        headers.insert_cache_control_must_revalidate();
+    } else if path.extension().is_some_and(|extension| {
+        extension == "jpg"
+            || extension == "png"
+            || extension == "woff2"
+            || extension == "js"
+            || extension == "map"
+    }) {
+        headers.insert_cache_control_immutable();
     }
 
     headers.insert_last_modified(date);

crates/brk_server/src/files/minify.rs

@@ -1,41 +0,0 @@
-// Source: https://github.com/oxc-project/oxc/blob/main/crates/oxc_minifier/examples/minifier.rs
-
-use std::{fs, path::Path};
-
-use oxc::{
-    allocator::Allocator,
-    codegen::{Codegen, CodegenOptions, LegalComment},
-    minifier::{CompressOptions, MangleOptions, Minifier, MinifierOptions},
-    parser::Parser,
-    span::SourceType,
-};
-
-pub fn minify_js(path: &Path) -> String {
-    let source_text = fs::read_to_string(path).unwrap();
-    let source_type = SourceType::from_path(path).unwrap();
-
-    let allocator = Allocator::default();
-
-    let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
-
-    let mut program = parser_return.program;
-
-    let minifier_return = Minifier::new(MinifierOptions {
-        mangle: Some(MangleOptions::default()),
-        compress: Some(CompressOptions::default()),
-    })
-    .build(&allocator, &mut program);
-
-    Codegen::new()
-        .with_options(CodegenOptions {
-            minify: true,
-            single_quote: false,
-            comments: false,
-            annotation_comments: false,
-            source_map_path: None,
-            legal_comments: LegalComment::None,
-        })
-        .with_scoping(minifier_return.scoping)
-        .build(&program)
-        .code
-}

crates/brk_server/src/files/mod.rs

@@ -3,7 +3,6 @@ use axum::{Router, routing::get};
 use super::AppState;
 
 mod file;
-mod minify;
 mod website;
 
 use file::{file_handler, index_handler};

crates/brk_server/src/lib.rs

@@ -10,7 +10,7 @@ use std::{
     time::Duration,
 };
 
-use api::{ApiRoutes, DTS};
+use api::{ApiRoutes, Bridge};
 use axum::{
     Json, Router,
     body::Body,
@@ -19,6 +19,7 @@ use axum::{
     routing::get,
     serve,
 };
+use brk_bundler::bundle;
 use brk_computer::Computer;
 use brk_core::dot_brk_path;
 use brk_indexer::Indexer;
@@ -45,10 +46,20 @@ pub struct AppState {
     websites_path: Option<PathBuf>,
 }
 
+impl AppState {
+    pub fn dist_path(&self) -> PathBuf {
+        self.websites_path
+            .as_ref()
+            .expect("Should never reach here is websites_path is None")
+            .join("dist")
+    }
+}
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+
 const DEV_PATH: &str = "../..";
 const DOWNLOADS: &str = "downloads";
 const WEBSITES: &str = "websites";
-const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 pub struct Server(AppState);
 
@@ -88,7 +99,7 @@ impl Server {
                 downloaded_websites_path
             };
 
-            query.generate_dts_file(website, websites_path.as_path())?;
+            query.generate_bridge_file(website, websites_path.as_path())?;
 
             Some(websites_path)
         } else {
@@ -102,9 +113,13 @@ impl Server {
         }))
     }
 
-    pub async fn serve(self) -> color_eyre::Result<()> {
+    pub async fn serve(self, watch: bool) -> color_eyre::Result<()> {
         let state = self.0;
 
+        if let Some(websites_path) = state.websites_path.clone() {
+            bundle(&websites_path, state.website.to_folder_name(), watch).await?;
+        }
+
         let compression_layer = CompressionLayer::new()
             .br(true)
             .deflate(true)

crates/brk_server/src/traits/header_map.rs

@@ -5,12 +5,11 @@ use std::{
 
 use axum::http::{
     HeaderMap,
-    header::{self, HOST, IF_MODIFIED_SINCE},
+    header::{self, IF_MODIFIED_SINCE},
 };
 use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone};
 use log::info;
 
-const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish
 const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";
 
 #[derive(PartialEq, Eq)]
@@ -20,12 +19,6 @@ pub enum ModifiedState {
 }
 
 pub trait HeaderMapExtended {
-    fn _get_scheme(&self) -> &str;
-    fn get_host(&self) -> &str;
-    fn check_if_host_is_local(&self) -> bool;
-    fn check_if_host_is_0000(&self) -> bool;
-    fn check_if_host_is_localhost(&self) -> bool;
-
     fn insert_cors(&mut self);
 
     fn get_if_modified_since(&self) -> Option<DateTime>;
@@ -36,8 +29,8 @@ pub trait HeaderMapExtended {
         duration: Duration,
     ) -> color_eyre::Result<(ModifiedState, DateTime)>;
 
+    fn insert_cache_control_must_revalidate(&mut self);
     fn insert_cache_control_immutable(&mut self);
-    fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64);
     fn insert_last_modified(&mut self, date: DateTime);
 
     fn insert_content_disposition_attachment(&mut self);
@@ -59,41 +52,22 @@ pub trait HeaderMapExtended {
 }
 
 impl HeaderMapExtended for HeaderMap {
-    fn _get_scheme(&self) -> &str {
-        if self.check_if_host_is_local() {
-            "http"
-        } else {
-            "https"
-        }
-    }
-
-    fn get_host(&self) -> &str {
-        self[HOST].to_str().unwrap()
-    }
-
-    fn check_if_host_is_local(&self) -> bool {
-        self.check_if_host_is_localhost() || self.check_if_host_is_0000()
-    }
-
-    fn check_if_host_is_0000(&self) -> bool {
-        self.get_host().contains("0.0.0.0")
-    }
-
-    fn check_if_host_is_localhost(&self) -> bool {
-        self.get_host().contains("localhost")
-    }
-
     fn insert_cors(&mut self) {
         self.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
         self.insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap());
     }
 
+    fn insert_cache_control_must_revalidate(&mut self) {
+        self.insert(
+            header::CACHE_CONTROL,
+            "public, max-age=0, must-revalidate".parse().unwrap(),
+        );
+    }
+
     fn insert_cache_control_immutable(&mut self) {
         self.insert(
             header::CACHE_CONTROL,
-            format!("public, max-age=604800, immutable, stale-if-error={STALE_IF_ERROR}")
-                .parse()
-                .unwrap(),
+            "public, max-age=31536000, immutable".parse().unwrap(),
         );
     }
 
@@ -101,16 +75,6 @@ impl HeaderMapExtended for HeaderMap {
         self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap());
     }
 
-    fn _insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64) {
-        self.insert(
-        header::CACHE_CONTROL,
-        format!(
-            "public, max-age={max_age}, stale-while-revalidate={stale_while_revalidate}, stale-if-error={STALE_IF_ERROR}")
-        .parse()
-        .unwrap(),
-    );
-    }
-
     fn insert_last_modified(&mut self, date: DateTime) {
         let formatted = date
             .to_zoned(TimeZone::system())
@@ -167,7 +131,7 @@ impl HeaderMapExtended for HeaderMap {
     fn insert_content_type(&mut self, path: &Path) {
         match path.extension().unwrap().to_str().unwrap() {
             "js" => self.insert_content_type_application_javascript(),
-            "json" => self.insert_content_type_application_json(),
+            "json" | "map" => self.insert_content_type_application_json(),
             "html" => self.insert_content_type_text_html(),
             "css" => self.insert_content_type_text_css(),
             "toml" | "txt" => self.insert_content_type_text_plain(),

crates/brk_state/src/outputs/mod.rs

@@ -70,7 +70,7 @@ impl<T> Outputs<T> {
             .collect::<Vec<_>>()
     }
 
-    pub fn as_mut_overlaping_vecs(&mut self) -> Vec<&mut T> {
+    pub fn as_mut_overlapping_vecs(&mut self) -> Vec<&mut T> {
         [&mut self.all]
             .into_iter()
             .chain(self.by_term.as_mut_vec())

crates/brk_vec/src/structs/format.rs

@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
     Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum,
 )]
 pub enum Format {
-    #[default]
     Compressed,
+    #[default]
     Raw,
 }
 

crates/brk_vec/src/traits/any.rs

@@ -13,6 +13,7 @@ pub trait AnyVec: Send + Sync {
     }
     fn modified_time(&self) -> Result<Duration>;
     fn index_type_to_string(&self) -> String;
+    fn value_type_to_size_of(&self) -> usize;
 }
 
 pub trait AnyIterableVec<I, T>: AnyVec {

crates/brk_vec/src/traits/collectable.rs

@@ -27,13 +27,19 @@ where
     #[inline]
     fn i64_to_usize(i: i64, len: usize) -> usize {
         if i >= 0 {
-            i as usize
+            (i as usize).min(len)
         } else {
             let v = len as i64 + i;
             if v < 0 { 0 } else { v as usize }
         }
     }
 
+    fn range_count(from: Option<i64>, to: Option<i64>, len: usize) -> usize {
+        let from = from.map(|i| Self::i64_to_usize(i, len));
+        let to = to.map(|i| Self::i64_to_usize(i, len));
+        (from.unwrap_or_default()..to.unwrap_or(len)).count()
+    }
+
     #[doc(hidden)]
     fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
         let len = self.len();

crates/brk_vec/src/traits/generic.rs

@@ -31,7 +31,7 @@ where
     }
     #[inline]
     fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<T>>> {
-        let stored_len = mmap.len() / Self::SIZE_OF_T;
+        let stored_len = self.stored_len_(mmap);
 
         if index >= stored_len {
             let pushed = self.pushed();
@@ -53,6 +53,7 @@ where
     fn mmap(&self) -> &ArcSwap<Mmap>;
 
     fn stored_len(&self) -> usize;
+    fn stored_len_(&self, mmap: &Mmap) -> usize;
 
     fn pushed(&self) -> &[T];
     #[inline]

crates/brk_vec/src/variants/compressed.rs

@@ -21,7 +21,9 @@ use crate::{
 const ONE_KIB: usize = 1024;
 const ONE_MIB: usize = ONE_KIB * ONE_KIB;
 pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
-pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
+pub const MAX_PAGE_SIZE: usize = 64 * ONE_KIB;
+
+const VERSION: Version = Version::ONE;
 
 #[derive(Debug)]
 pub struct CompressedVec<I, T> {
@@ -39,7 +41,9 @@ where
     pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
 
     /// Same as import but will reset the folder under certain errors, so be careful !
-    pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
+    pub fn forced_import(path: &Path, mut version: Version) -> Result<Self> {
+        version = version + VERSION;
+
         let res = Self::import(path, version);
         match res {
             Err(Error::WrongEndian)
@@ -129,7 +133,7 @@ where
         page_index * Self::PER_PAGE
     }
 
-    fn stored_len_(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
+    fn stored_len__(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
         if let Some(last) = pages_meta.last() {
             (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
         } else {
@@ -178,7 +182,11 @@ where
 
     #[inline]
     fn stored_len(&self) -> usize {
-        Self::stored_len_(&self.pages_meta.load())
+        Self::stored_len__(&self.pages_meta.load())
+    }
+    #[inline]
+    fn stored_len_(&self, _: &Mmap) -> usize {
+        self.stored_len()
     }
 
     #[inline]
@@ -361,6 +369,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for CompressedVec<I, T> {
@@ -472,7 +485,7 @@ where
 
     fn into_iter(self) -> Self::IntoIter {
         let pages_meta = self.pages_meta.load();
-        let stored_len = CompressedVec::<I, T>::stored_len_(&pages_meta);
+        let stored_len = CompressedVec::<I, T>::stored_len__(&pages_meta);
         CompressedVecIterator {
             vec: self,
             guard: self.mmap().load(),

crates/brk_vec/src/variants/computed.rs

@@ -255,6 +255,11 @@ where
             ComputedVec::LazyFrom3(v) => v.modified_time(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub enum ComputedVecIterator<'a, I, T, S1I, S1T, S2I, S2T, S3I, S3T> {

crates/brk_vec/src/variants/eager.rs

@@ -1015,6 +1015,32 @@ where
 
         self.safe_flush(exit)
     }
+
+    pub fn compute_zscore(
+        &mut self,
+        max_from: I,
+        ratio: &impl AnyIterableVec<I, StoredF32>,
+        sma: &impl AnyIterableVec<I, StoredF32>,
+        sd: &impl AnyIterableVec<I, StoredF32>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: From<StoredF32>,
+    {
+        let mut sma_iter = sma.iter();
+        let mut sd_iter = sd.iter();
+
+        self.compute_transform(
+            max_from,
+            ratio,
+            |(i, ratio, ..)| {
+                let sma = sma_iter.unwrap_get_inner(i);
+                let sd = sd_iter.unwrap_get_inner(i);
+                (i, T::from((ratio - sma) / sd))
+            },
+            exit,
+        )
+    }
 }
 
 impl EagerVec<DateIndex, Sats> {
@@ -1116,12 +1142,12 @@ impl EagerVec<DateIndex, Dollars> {
         exit: &Exit,
     ) -> Result<()> {
         self.validate_computed_version_or_reset_file(
-            Version::ZERO + self.inner.version() + stacks.version(),
+            Version::ONE + self.inner.version() + stacks.version(),
         )?;
 
         let index = max_from.min(DateIndex::from(self.len()));
 
-        let first_price_date = DateIndex::try_from(Date::new(2010, 8, 16)).unwrap();
+        let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
 
         stacks.iter_at(index).try_for_each(|(i, stack)| {
             let stack = stack.into_inner();
@@ -1298,6 +1324,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> AnyIterableVec<I, T> for EagerVec<I, T>

crates/brk_vec/src/variants/indexed.rs

@@ -127,6 +127,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 pub trait AnyIndexedVec: AnyVec {

crates/brk_vec/src/variants/lazy1.rs

@@ -146,6 +146,11 @@ where
     fn modified_time(&self) -> Result<std::time::Duration> {
         self.source.modified_time()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T> AnyIterableVec<I, T> for LazyVecFrom1<I, T, S1I, S1T>

crates/brk_vec/src/variants/lazy2.rs

@@ -194,6 +194,11 @@ where
             .modified_time()?
             .min(self.source2.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T> AnyIterableVec<I, T> for LazyVecFrom2<I, T, S1I, S1T, S2I, S2T>

crates/brk_vec/src/variants/lazy3.rs

@@ -229,6 +229,11 @@ where
             .min(self.source2.modified_time()?)
             .min(self.source3.modified_time()?))
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T, S1I, S1T, S2I, S2T, S3I, S3T> AnyIterableVec<I, T>

crates/brk_vec/src/variants/raw.rs

@@ -48,8 +48,10 @@ where
         fs::create_dir_all(path)?;
 
         let version_path = Self::path_version_(path);
-        version.validate(version_path.as_ref())?;
-        version.write(version_path.as_ref())?;
+
+        if !version.validate(version_path.as_ref())? {
+            version.write(version_path.as_ref())?;
+        }
 
         let file = Self::open_file_(Self::path_vec_(path).as_path())?;
         let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
@@ -102,7 +104,11 @@ where
 
     #[inline]
     fn stored_len(&self) -> usize {
-        self.mmap.load().len() / Self::SIZE_OF_T
+        self.stored_len_(&self.mmap.load())
+    }
+    #[inline]
+    fn stored_len_(&self, mmap: &Mmap) -> usize {
+        mmap.len() / Self::SIZE_OF_T
     }
 
     #[inline]
@@ -195,6 +201,11 @@ where
     fn index_type_to_string(&self) -> String {
         I::to_string()
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 impl<I, T> Clone for RawVec<I, T> {

crates/brk_vec/src/variants/stored.rs

@@ -73,6 +73,13 @@ where
             StoredVec::Compressed(v) => v.stored_len(),
         }
     }
+    #[inline]
+    fn stored_len_(&self, mmap: &Mmap) -> usize {
+        match self {
+            StoredVec::Raw(v) => v.stored_len_(mmap),
+            StoredVec::Compressed(v) => v.stored_len_(mmap),
+        }
+    }
 
     #[inline]
     fn pushed(&self) -> &[T] {
@@ -149,6 +156,11 @@ where
             StoredVec::Compressed(v) => v.name(),
         }
     }
+
+    #[inline]
+    fn value_type_to_size_of(&self) -> usize {
+        size_of::<T>()
+    }
 }
 
 #[derive(Debug)]