release: v0.0.42

.gitignore

@@ -8,30 +8,11 @@ target
 *\ copy*
 
 # Ignored
-/_*
+_*
 
 # Editors
 .vscode
 .zed
 
-# Flamegraph
-flamegraph/
-flamegraph.svg
-
-# Benchmarks
-benches
-
-# Snapshots
-snapshots*/
-
-# Docker
-docker/kibo
-
-# Types
-paths.d.ts
-
-# Outputs
-_outputs
-
 # Logs
 .log

CHANGELOG.md

@@ -3,38 +3,9 @@
 ![Image of the kibo Web App version 0.X.Y](https://github.com/kibo-money/kibo/blob/main/_assets/v0.X.Y.jpg)
 -->
 
-# v0.6.0 | WIP | A new beginning
+# v0.X.0 | WIP | A new beginning
 
-## Global
-
-- Completely redesign the back-end
-
-- Merged parser and server crates into a single project (and thus executable), so now both will run at the same time with a single `cargo run -r` [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
-- Added `--no-server` and `--no-parser` to disable each if needed
-- Improved executable parameters
-- Started using `log` and `env_logger` crates instead of custom code [#7392982](https://github.com/kibo-money/kibo/commit/7392982824c2db94bcd57251fd41986117c29a23)
-- Improved logs
-- Fixed input being unfocused right after being focused in Brave browser [#9a9ae61](https://github.com/kibo-money/kibo/commit/9a9ae614d07b54c08b7e9c0e2aefe3b52fdb93c5)
-
-- Reworked server's API code [#6ab0f46]( https://github.com/kibo-money/kibo/commit/6ab0f463119a902a1b7ca9691b54f61543bb8f2f)
-  - New route format: `/api/date-to-realized-price` is now `/api/realized-price?kind=date`
-  - Added status and timing to logs
-- Updated website packages
-- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
-  - `/api/realized-price?kind=t`
-  - `/api/realized-price?kind=timestamp&chunk=860000`
-- Created separate crate for indexing called `bindex`
-- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
-- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibo has its own indexes now
-
-## Git
-
-Added git tags for each version though Markdown won't display formatted on Github so left the default text
-
-## Deprecated
-
-Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
-Since it's not used anymore it will moved out of the repository relatively soon.
+Full rewrite
 
 # [kibo-v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
 

Cargo.toml

@@ -4,7 +4,8 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.34"
+package.version = "0.0.42"
+package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 
 [profile.release]
@@ -16,8 +17,10 @@ panic = "abort"
 inherits = "release"
 
 [workspace.dependencies]
+arc-swap = "1.7.1"
 axum = "0.8.4"
-bitcoin = { version = "0.32.5", features = ["serde"] }
+bincode = { version = "2.0.1", features = ["serde"] }
+bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
 brk_cli = { version = "0", path = "crates/brk_cli" }
 brk_computer = { version = "0", path = "crates/brk_computer" }
@@ -29,21 +32,25 @@ brk_logger = { version = "0", path = "crates/brk_logger" }
 brk_parser = { version = "0", path = "crates/brk_parser" }
 brk_query = { version = "0", path = "crates/brk_query" }
 brk_server = { version = "0", path = "crates/brk_server" }
+brk_state = { version = "0", path = "crates/brk_state" }
+brk_store = { version = "0", path = "crates/brk_store" }
 brk_vec = { version = "0", path = "crates/brk_vec" }
-byteview = "0.6.1"
-clap = { version = "4.5.37", features = ["string"] }
+byteview = "=0.6.1"
+clap = { version = "4.5.39", features = ["string"] }
 clap_derive = "4.5.32"
-color-eyre = "0.6.4"
+color-eyre = "0.6.5"
 derive_deref = "1.1.1"
-fjall = "2.10.0"
-jiff = "0.2.13"
+fjall = "2.11.0"
+jiff = "0.2.14"
 log = { version = "0.4.27" }
 minreq = { version = "2.13.4", features = ["https", "serde_json"] }
 rayon = "1.10.0"
 serde = { version = "1.0.219" }
+serde_bytes = "0.11.17"
 serde_derive = "1.0.219"
 serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
-tabled = "0.19.0"
+tabled = "0.20.0"
+tokio = { version = "1.45.1", features = ["rt-multi-thread"] }
 zerocopy = { version = "0.8.25" }
 zerocopy-derive = "0.8.25"
 

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 bitcoinresearchkit, kibo.money
+Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>
@@ -77,6 +74,8 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
 - [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
 - [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets.
 - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
+- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer
+- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
 - [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
 
 ## Acknowledgments
@@ -92,7 +91,7 @@ Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagm
 If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
 
 - 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
-- 99.9% SLA
+- 99.99% SLA
 - Configurated for speed (`raw + eager`)
 - Updates delivered at your convenience
 - Direct communication for feature requests and support

crates/brk/Cargo.toml

@@ -3,6 +3,7 @@ name = "brk"
 description.workspace = true
 license.workspace = true
 readme.workspace = true
+homepage.workspace = true
 repository.workspace = true
 edition.workspace = true
 version.workspace = true
@@ -18,6 +19,8 @@ full = [
     "parser",
     "query",
     "server",
+    "state",
+    "store",
     "vec",
 ]
 core = ["brk_core"]
@@ -29,6 +32,8 @@ logger = ["brk_logger"]
 parser = ["brk_parser"]
 query = ["brk_query"]
 server = ["brk_server"]
+state = ["brk_state"]
+store = ["brk_store"]
 vec = ["brk_vec"]
 
 [dependencies]
@@ -42,6 +47,8 @@ brk_logger = { workspace = true, optional = true }
 brk_parser = { workspace = true, optional = true }
 brk_query = { workspace = true, optional = true }
 brk_server = { workspace = true, optional = true }
+brk_state = { workspace = true, optional = true }
+brk_store = { workspace = true, optional = true }
 brk_vec = { workspace = true, optional = true }
 
 [package.metadata.docs.rs]

crates/brk/src/lib.rs

@@ -36,6 +36,14 @@ pub use brk_query as query;
 #[doc(inline)]
 pub use brk_server as server;
 
+#[cfg(feature = "state")]
+#[doc(inline)]
+pub use brk_state as state;
+
+#[cfg(feature = "store")]
+#[doc(inline)]
+pub use brk_store as store;
+
 #[cfg(feature = "vec")]
 #[doc(inline)]
 pub use brk_vec as vec;

crates/brk_cli/Cargo.toml

@@ -4,9 +4,11 @@ description = "A command line interface to interact with the full Bitcoin Resear
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
+bitcoincore-rpc = { workspace = true }
 brk_computer = { workspace = true }
 brk_core = { workspace = true }
 brk_exit = { workspace = true }
@@ -23,7 +25,8 @@ color-eyre = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true }
 tabled = { workspace = true }
-toml = "0.8.22"
+tokio = { workspace = true }
+toml = "0.8.23"
 
 [[bin]]
 name = "brk"

crates/brk_cli/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>

crates/brk_cli/src/query.rs

@@ -8,12 +8,12 @@ use crate::run::RunConfig;
 pub fn query(params: QueryParams) -> color_eyre::Result<()> {
     let config = RunConfig::import(None)?;
 
-    let compressed = config.compressed();
+    let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
     indexer.import_vecs()?;
 
-    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
+    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
     computer.import_vecs(&indexer, config.computation())?;
 
     let query = Query::build(&indexer, &computer);

crates/brk_cli/src/run.rs

@@ -1,18 +1,18 @@
 use std::{
     fs,
     path::{Path, PathBuf},
-    thread::sleep,
+    thread::{self, sleep},
     time::Duration,
 };
 
+use bitcoincore_rpc::{self, Auth, Client, RpcApi};
 use brk_computer::Computer;
 use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_parser::rpc::{self, Auth, Client, RpcApi};
-use brk_server::{Server, Website, tokio};
-use brk_vec::Computation;
+use brk_server::{Server, Website};
+use brk_vec::{Computation, Format};
 use clap_derive::{Parser, ValueEnum};
 use color_eyre::eyre::eyre;
 use log::info;
@@ -27,83 +27,91 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
 
     let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
 
-    let compressed = config.compressed();
+    let format = config.format();
 
-    let mut indexer = Indexer::new(&config.outputsdir(), compressed, config.check_collisions())?;
+    let mut indexer = Indexer::new(&config.outputsdir(), format, config.check_collisions())?;
     indexer.import_stores()?;
     indexer.import_vecs()?;
 
-    let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), compressed);
-    computer.import_stores(&indexer)?;
-    computer.import_vecs(&indexer, config.computation())?;
+    let wait_for_synced_node = || -> color_eyre::Result<()> {
+        let is_synced = || -> color_eyre::Result<bool> {
+            let info = rpc.get_blockchain_info()?;
+            Ok(info.headers == info.blocks)
+        };
 
-    tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()?
-        .block_on(async {
-            let server = if config.serve() {
-                let served_indexer = indexer.clone();
-                let served_computer = computer.clone();
+        if !is_synced()? {
+            info!("Waiting for node to be synced...");
+            while !is_synced()? {
+                sleep(Duration::from_secs(1))
+            }
+        }
 
-                let server = Server::new(served_indexer, served_computer, config.website())?;
+        Ok(())
+    };
 
-                let opt = Some(tokio::spawn(async move {
-                    server.serve().await.unwrap();
-                }));
+    let f = move || -> color_eyre::Result<()> {
+        let mut computer = Computer::new(&config.outputsdir(), config.fetcher(), format);
+        computer.import_stores(&indexer)?;
+        computer.import_vecs(&indexer, config.computation())?;
 
-                sleep(Duration::from_secs(1));
+        tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()?
+            .block_on(async {
+                let server = if config.serve() {
+                    let served_indexer = indexer.clone();
+                    let served_computer = computer.clone();
 
-                opt
-            } else {
-                None
-            };
+                    let server = Server::new(served_indexer, served_computer, config.website())?;
 
-            if config.process() {
-                let wait_for_synced_node = || -> color_eyre::Result<()> {
-                    let is_synced = || -> color_eyre::Result<bool> {
-                        let info = rpc.get_blockchain_info()?;
-                        Ok(info.headers == info.blocks)
-                    };
+                    let opt = Some(tokio::spawn(async move {
+                        server.serve().await.unwrap();
+                    }));
 
-                    if !is_synced()? {
-                        info!("Waiting for node to be synced...");
-                        while !is_synced()? {
+                    sleep(Duration::from_secs(1));
+
+                    opt
+                } else {
+                    None
+                };
+
+                if config.process() {
+                    loop {
+                        wait_for_synced_node()?;
+
+                        let block_count = rpc.get_block_count()?;
+
+                        info!("{} blocks found.", block_count + 1);
+
+                        let starting_indexes = indexer.index(&parser, rpc, &exit)?;
+
+                        computer.compute(&mut indexer, starting_indexes, &exit)?;
+
+                        if let Some(delay) = config.delay() {
+                            sleep(Duration::from_secs(delay))
+                        }
+
+                        info!("Waiting for new blocks...");
+
+                        while block_count == rpc.get_block_count()? {
                             sleep(Duration::from_secs(1))
                         }
                     }
-
-                    Ok(())
-                };
-
-                loop {
-                    wait_for_synced_node()?;
-
-                    let block_count = rpc.get_block_count()?;
-
-                    info!("{} blocks found.", block_count + 1);
-
-                    let starting_indexes = indexer.index(&parser, rpc, &exit)?;
-
-                    computer.compute(&mut indexer, starting_indexes, &exit)?;
-
-                    if let Some(delay) = config.delay() {
-                        sleep(Duration::from_secs(delay))
-                    }
-
-                    info!("Waiting for new blocks...");
-
-                    while block_count == rpc.get_block_count()? {
-                        sleep(Duration::from_secs(1))
-                    }
                 }
-            }
 
-            if let Some(handle) = server {
-                handle.await.unwrap();
-            }
+                if let Some(handle) = server {
+                    handle.await.unwrap();
+                }
 
-            Ok(())
-        })
+                Ok(())
+            })
+    };
+
+    thread::Builder::new()
+        .stack_size(128 * 1024 * 1024)
+        .spawn(f)?
+        .join()
+        .unwrap()
 }
 
 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
@@ -125,18 +133,18 @@ pub struct RunConfig {
     mode: Option<Mode>,
 
     /// Computation mode for compatible datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: Lazy, saved
-    #[arg(short = 'C', long)]
+    #[arg(short, long)]
     computation: Option<Computation>,
 
     /// Activate compression of datasets, set to true to save disk space or false if prioritize speed, default: true, saved
-    #[arg(short, long, value_name = "BOOL")]
-    compressed: Option<bool>,
+    #[arg(short, long, value_name = "FORMAT")]
+    format: Option<Format>,
 
     /// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
-    #[arg(short, long, value_name = "BOOL")]
+    #[arg(short = 'F', long, value_name = "BOOL")]
     fetch: Option<bool>,
 
-    /// Website served by the server (if active), default: kibo.money, saved
+    /// Website served by the server (if active), default: default, saved
     #[arg(short, long)]
     website: Option<Website>,
 
@@ -196,12 +204,16 @@ impl RunConfig {
                 config_saved.mode = Some(mode);
             }
 
+            if let Some(computation) = config_args.computation.take() {
+                config_saved.computation = Some(computation);
+            }
+
             if let Some(fetch) = config_args.fetch.take() {
                 config_saved.fetch = Some(fetch);
             }
 
-            if let Some(compressed) = config_args.compressed.take() {
-                config_saved.compressed = Some(compressed);
+            if let Some(format) = config_args.format.take() {
+                config_saved.format = Some(format);
             }
 
             if let Some(website) = config_args.website.take() {
@@ -306,7 +318,7 @@ impl RunConfig {
     }
 
     pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
-        Ok(Box::leak(Box::new(rpc::Client::new(
+        Ok(Box::leak(Box::new(Client::new(
             &format!(
                 "http://{}:{}",
                 self.rpcconnect().unwrap_or(&"localhost".to_string()),
@@ -406,7 +418,7 @@ impl RunConfig {
     }
 
     pub fn website(&self) -> Website {
-        self.website.unwrap_or(Website::KiboMoney)
+        self.website.unwrap_or(Website::Default)
     }
 
     pub fn fetch(&self) -> bool {
@@ -422,8 +434,8 @@ impl RunConfig {
         self.computation.unwrap_or_default()
     }
 
-    pub fn compressed(&self) -> bool {
-        self.compressed.is_none_or(|b| b)
+    pub fn format(&self) -> Format {
+        self.format.unwrap_or_default()
     }
 
     pub fn check_collisions(&self) -> bool {

crates/brk_computer/Cargo.toml

@@ -4,20 +4,22 @@ description = "A Bitcoin dataset computer, built on top of brk_indexer"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
+bitcoin = { workspace = true }
+bitcoincore-rpc = { workspace = true }
 brk_core = { workspace = true }
 brk_exit = { workspace = true }
 brk_fetcher = { workspace = true }
 brk_indexer = { workspace = true }
 brk_logger = { workspace = true }
 brk_parser = { workspace = true }
+brk_state = { workspace = true }
 brk_vec = { workspace = true }
-clap = { workspace = true }
-clap_derive = { workspace = true }
 color-eyre = { workspace = true }
 fjall = { workspace = true }
+jiff = { workspace = true }
 log = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
+rayon = { workspace = true }

crates/brk_computer/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>

crates/brk_computer/examples/main.rs

@@ -1,12 +1,12 @@
-use std::path::Path;
+use std::{path::Path, thread};
 
 use brk_computer::Computer;
-use brk_core::default_bitcoin_path;
+use brk_core::{default_bitcoin_path, default_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_parser::{Parser, rpc};
-use brk_vec::Computation;
+use brk_parser::Parser;
+use brk_vec::{Computation, Format};
 
 pub fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -15,31 +15,40 @@ pub fn main() -> color_eyre::Result<()> {
 
     let bitcoin_dir = default_bitcoin_path();
 
-    let rpc = Box::leak(Box::new(rpc::Client::new(
+    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
         "http://localhost:8332",
-        rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
+        bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
     )?));
     let exit = Exit::new();
 
-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    // Can't increase main thread's stack programatically, thus we need to use another thread
+    thread::Builder::new()
+        .stack_size(32 * 1024 * 1024)
+        .spawn(move || -> color_eyre::Result<()> {
+            let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
 
-    let outputs_dir = Path::new("../../_outputs");
+            let _outputs_dir = default_brk_path().join("outputs");
+            let outputs_dir = _outputs_dir.as_path();
+            // let outputs_dir = Path::new("../../_outputs");
 
-    let compressed = false;
+            let format = Format::Raw;
 
-    let mut indexer = Indexer::new(outputs_dir, compressed, true)?;
-    indexer.import_stores()?;
-    indexer.import_vecs()?;
+            let mut indexer = Indexer::new(outputs_dir, format, true)?;
+            indexer.import_stores()?;
+            indexer.import_vecs()?;
 
-    let fetcher = Fetcher::import(None)?;
+            let fetcher = Fetcher::import(None)?;
 
-    let mut computer = Computer::new(outputs_dir, Some(fetcher), compressed);
-    computer.import_stores(&indexer)?;
-    computer.import_vecs(&indexer, Computation::Lazy)?;
+            let mut computer = Computer::new(outputs_dir, Some(fetcher), format);
+            computer.import_stores(&indexer)?;
+            computer.import_vecs(&indexer, Computation::Lazy)?;
 
-    let starting_indexes = indexer.index(&parser, rpc, &exit)?;
+            let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
-    computer.compute(&mut indexer, starting_indexes, &exit)?;
+            computer.compute(&mut indexer, starting_indexes, &exit)?;
 
-    Ok(())
+            Ok(())
+        })?
+        .join()
+        .unwrap()
 }

crates/brk_computer/src/lib.rs

@@ -5,16 +5,19 @@
 
 use std::path::{Path, PathBuf};
 
+use brk_core::Version;
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
-use brk_indexer::{Indexer, Indexes};
-pub use brk_parser::rpc;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, Computation, Format};
 
-mod storage;
+mod stores;
+mod utils;
+mod vecs;
 
-use brk_vec::{AnyCollectableVec, Compressed, Computation};
 use log::info;
-use storage::{Stores, Vecs};
+use stores::Stores;
+use vecs::Vecs;
 
 #[derive(Clone)]
 pub struct Computer {
@@ -22,17 +25,19 @@ pub struct Computer {
     fetcher: Option<Fetcher>,
     vecs: Option<Vecs>,
     stores: Option<Stores>,
-    compressed: Compressed,
+    format: Format,
 }
 
+const VERSION: Version = Version::ONE;
+
 impl Computer {
-    pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, compressed: bool) -> Self {
+    pub fn new(outputs_dir: &Path, fetcher: Option<Fetcher>, format: Format) -> Self {
         Self {
             path: outputs_dir.to_owned(),
             fetcher,
             vecs: None,
             stores: None,
-            compressed: Compressed::from(compressed),
+            format,
         }
     }
 
@@ -42,11 +47,13 @@ impl Computer {
         computation: Computation,
     ) -> color_eyre::Result<()> {
         self.vecs = Some(Vecs::import(
+            // TODO: Give self.path, join inside import
             &self.path.join("vecs/computed"),
+            VERSION + Version::ZERO,
             indexer,
             self.fetcher.is_some(),
             computation,
-            self.compressed,
+            self.format,
         )?);
         Ok(())
     }
@@ -55,7 +62,9 @@ impl Computer {
     /// Clone struct instead
     pub fn import_stores(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
         self.stores = Some(Stores::import(
+            // TODO: Give self.path, join inside import
             &self.path.join("stores"),
+            VERSION + Version::ZERO,
             indexer.keyspace(),
         )?);
         Ok(())
@@ -66,19 +75,14 @@ impl Computer {
     pub fn compute(
         &mut self,
         indexer: &mut Indexer,
-        starting_indexes: Indexes,
+        starting_indexes: brk_indexer::Indexes,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
         info!("Computing...");
-
-        self.vecs.as_mut().unwrap().compute(
-            indexer,
-            starting_indexes,
-            self.fetcher.as_mut(),
-            exit,
-        )?;
-
-        Ok(())
+        self.vecs
+            .as_mut()
+            .unwrap()
+            .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
     }
 
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {

crates/brk_computer/src/storage/mod.rs

@@ -1,5 +0,0 @@
-mod stores;
-mod vecs;
-
-pub use stores::*;
-pub use vecs::*;

crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs

@@ -1,155 +0,0 @@
-use std::path::Path;
-
-use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
-use brk_exit::Exit;
-use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
-
-use crate::storage::{Indexes, indexes};
-
-use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
-
-#[derive(Clone)]
-pub struct ComputedVecsFromDateindex<T>
-where
-    T: ComputedType + PartialOrd,
-{
-    pub dateindex: EagerVec<DateIndex, T>,
-    pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
-    pub weekindex: ComputedVecBuilder<WeekIndex, T>,
-    pub monthindex: ComputedVecBuilder<MonthIndex, T>,
-    pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
-    pub yearindex: ComputedVecBuilder<YearIndex, T>,
-    pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
-}
-
-const VERSION: Version = Version::ZERO;
-
-impl<T> ComputedVecsFromDateindex<T>
-where
-    T: ComputedType,
-{
-    pub fn forced_import(
-        path: &Path,
-        name: &str,
-        version: Version,
-        compressed: Compressed,
-        options: StorableVecGeneatorOptions,
-    ) -> color_eyre::Result<Self> {
-        let version = VERSION + version;
-
-        let dateindex_extra = ComputedVecBuilder::forced_import(
-            path,
-            name,
-            version,
-            compressed,
-            options.copy_self_extra(),
-        )?;
-
-        let options = options.remove_percentiles();
-
-        Ok(Self {
-            dateindex: EagerVec::forced_import(
-                &path.join(format!("dateindex_to_{name}")),
-                version,
-                compressed,
-            )?,
-            dateindex_extra,
-            weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            monthindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-            quarterindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-            yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            decadeindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-        })
-    }
-
-    pub fn compute<F>(
-        &mut self,
-        indexer: &Indexer,
-        indexes: &indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-        mut compute: F,
-    ) -> color_eyre::Result<()>
-    where
-        F: FnMut(
-            &mut EagerVec<DateIndex, T>,
-            &Indexer,
-            &indexes::Vecs,
-            &Indexes,
-            &Exit,
-        ) -> Result<()>,
-    {
-        compute(
-            &mut self.dateindex,
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-        )?;
-
-        self.dateindex_extra
-            .extend(starting_indexes.dateindex, &self.dateindex, exit)?;
-
-        self.weekindex.compute(
-            starting_indexes.weekindex,
-            &self.dateindex,
-            &indexes.weekindex_to_first_dateindex,
-            &indexes.weekindex_to_dateindex_count,
-            exit,
-        )?;
-
-        self.monthindex.compute(
-            starting_indexes.monthindex,
-            &self.dateindex,
-            &indexes.monthindex_to_first_dateindex,
-            &indexes.monthindex_to_dateindex_count,
-            exit,
-        )?;
-
-        self.quarterindex.from_aligned(
-            starting_indexes.quarterindex,
-            &self.monthindex,
-            &indexes.quarterindex_to_first_monthindex,
-            &indexes.quarterindex_to_monthindex_count,
-            exit,
-        )?;
-
-        self.yearindex.from_aligned(
-            starting_indexes.yearindex,
-            &self.monthindex,
-            &indexes.yearindex_to_first_monthindex,
-            &indexes.yearindex_to_monthindex_count,
-            exit,
-        )?;
-
-        self.decadeindex.from_aligned(
-            starting_indexes.decadeindex,
-            &self.yearindex,
-            &indexes.decadeindex_to_first_yearindex,
-            &indexes.decadeindex_to_yearindex_count,
-            exit,
-        )?;
-
-        Ok(())
-    }
-
-    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
-        [
-            vec![&self.dateindex as &dyn AnyCollectableVec],
-            self.dateindex_extra.vecs(),
-            self.weekindex.vecs(),
-            self.monthindex.vecs(),
-            self.quarterindex.vecs(),
-            self.yearindex.vecs(),
-            self.decadeindex.vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs

@@ -1,222 +0,0 @@
-use std::path::Path;
-
-use brk_core::{
-    DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, TxIndex, WeekIndex,
-    YearIndex,
-};
-use brk_exit::Exit;
-use brk_indexer::Indexer;
-use brk_vec::{
-    AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
-};
-
-use crate::storage::{Indexes, indexes};
-
-use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
-
-#[derive(Clone)]
-pub struct ComputedVecsFromTxindex<T>
-where
-    T: ComputedType + PartialOrd,
-{
-    pub txindex: Option<EagerVec<TxIndex, T>>,
-    pub height: ComputedVecBuilder<Height, T>,
-    pub dateindex: ComputedVecBuilder<DateIndex, T>,
-    pub weekindex: ComputedVecBuilder<WeekIndex, T>,
-    pub difficultyepoch: ComputedVecBuilder<DifficultyEpoch, T>,
-    pub monthindex: ComputedVecBuilder<MonthIndex, T>,
-    pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
-    pub yearindex: ComputedVecBuilder<YearIndex, T>,
-    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
-    pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
-}
-
-const VERSION: Version = Version::ZERO;
-
-impl<T> ComputedVecsFromTxindex<T>
-where
-    T: ComputedType + Ord + From<f64>,
-    f64: From<T>,
-{
-    pub fn forced_import(
-        path: &Path,
-        name: &str,
-        compute_source: bool,
-        version: Version,
-        compressed: Compressed,
-        options: StorableVecGeneatorOptions,
-    ) -> color_eyre::Result<Self> {
-        let version = VERSION + version;
-
-        let txindex = compute_source.then(|| {
-            EagerVec::forced_import(
-                &path.join(format!("txindex_to_{name}")),
-                version,
-                compressed,
-            )
-            .unwrap()
-        });
-
-        let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
-
-        let options = options.remove_percentiles();
-
-        Ok(Self {
-            txindex,
-            height,
-            dateindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            difficultyepoch: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-            monthindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-            quarterindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-            yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
-            decadeindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
-            )?,
-        })
-    }
-
-    #[allow(unused)]
-    pub fn compute_all<F>(
-        &mut self,
-        indexer: &Indexer,
-        indexes: &indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-        mut compute: F,
-    ) -> color_eyre::Result<()>
-    where
-        F: FnMut(
-            &mut EagerVec<TxIndex, T>,
-            &Indexer,
-            &indexes::Vecs,
-            &Indexes,
-            &Exit,
-        ) -> Result<()>,
-    {
-        compute(
-            self.txindex.as_mut().unwrap(),
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-        )?;
-
-        let txindex: Option<&StoredVec<TxIndex, T>> = None;
-        self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
-
-        Ok(())
-    }
-
-    pub fn compute_rest(
-        &mut self,
-        indexer: &Indexer,
-        indexes: &indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-        txindex: Option<&impl CollectableVec<TxIndex, T>>,
-    ) -> color_eyre::Result<()> {
-        if let Some(txindex) = txindex {
-            self.height.compute(
-                starting_indexes.height,
-                txindex,
-                &indexer.vecs().height_to_first_txindex,
-                &indexes.height_to_txindex_count,
-                exit,
-            )?;
-        } else {
-            let txindex = self.txindex.as_ref().unwrap();
-
-            self.height.compute(
-                starting_indexes.height,
-                txindex,
-                &indexer.vecs().height_to_first_txindex,
-                &indexes.height_to_txindex_count,
-                exit,
-            )?;
-        }
-
-        self.dateindex.from_aligned(
-            starting_indexes.dateindex,
-            &self.height,
-            &indexes.dateindex_to_first_height,
-            &indexes.dateindex_to_height_count,
-            exit,
-        )?;
-
-        self.weekindex.from_aligned(
-            starting_indexes.weekindex,
-            &self.dateindex,
-            &indexes.weekindex_to_first_dateindex,
-            &indexes.weekindex_to_dateindex_count,
-            exit,
-        )?;
-
-        self.monthindex.from_aligned(
-            starting_indexes.monthindex,
-            &self.dateindex,
-            &indexes.monthindex_to_first_dateindex,
-            &indexes.monthindex_to_dateindex_count,
-            exit,
-        )?;
-
-        self.quarterindex.from_aligned(
-            starting_indexes.quarterindex,
-            &self.monthindex,
-            &indexes.quarterindex_to_first_monthindex,
-            &indexes.quarterindex_to_monthindex_count,
-            exit,
-        )?;
-
-        self.yearindex.from_aligned(
-            starting_indexes.yearindex,
-            &self.monthindex,
-            &indexes.yearindex_to_first_monthindex,
-            &indexes.yearindex_to_monthindex_count,
-            exit,
-        )?;
-
-        self.decadeindex.from_aligned(
-            starting_indexes.decadeindex,
-            &self.yearindex,
-            &indexes.decadeindex_to_first_yearindex,
-            &indexes.decadeindex_to_yearindex_count,
-            exit,
-        )?;
-
-        self.difficultyepoch.from_aligned(
-            starting_indexes.difficultyepoch,
-            &self.height,
-            &indexes.difficultyepoch_to_first_height,
-            &indexes.difficultyepoch_to_height_count,
-            exit,
-        )?;
-
-        Ok(())
-    }
-
-    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
-        [
-            self.txindex
-                .as_ref()
-                .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
-            self.height.vecs(),
-            self.dateindex.vecs(),
-            self.weekindex.vecs(),
-            self.difficultyepoch.vecs(),
-            self.monthindex.vecs(),
-            self.quarterindex.vecs(),
-            self.yearindex.vecs(),
-            // self.halvingepoch.vecs(),
-            self.decadeindex.vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/market.rs

@@ -1,253 +0,0 @@
-use std::{fs, path::Path};
-
-use brk_core::{Dollars, StoredF64, StoredUsize};
-use brk_exit::Exit;
-use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, Compressed, Computation, StoredIndex, VecIterator, Version};
-
-use super::{
-    Indexes, fetched,
-    grouped::{ComputedVecsFromDateindex, StorableVecGeneatorOptions},
-    indexes, transactions,
-};
-
-#[derive(Clone)]
-pub struct Vecs {
-    pub indexes_to_marketcap: ComputedVecsFromDateindex<Dollars>,
-    pub indexes_to_ath: ComputedVecsFromDateindex<Dollars>,
-    pub indexes_to_drawdown: ComputedVecsFromDateindex<StoredF64>,
-    pub indexes_to_days_since_ath: ComputedVecsFromDateindex<StoredUsize>,
-    pub indexes_to_max_days_between_ath: ComputedVecsFromDateindex<StoredUsize>,
-    pub indexes_to_max_years_between_ath: ComputedVecsFromDateindex<StoredF64>,
-}
-
-impl Vecs {
-    pub fn forced_import(
-        path: &Path,
-        _computation: Computation,
-        compressed: Compressed,
-    ) -> color_eyre::Result<Self> {
-        fs::create_dir_all(path)?;
-
-        Ok(Self {
-            indexes_to_marketcap: ComputedVecsFromDateindex::forced_import(
-                path,
-                "marketcap",
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-            indexes_to_ath: ComputedVecsFromDateindex::forced_import(
-                path,
-                "ath",
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-            indexes_to_drawdown: ComputedVecsFromDateindex::forced_import(
-                path,
-                "drawdown",
-                Version::ONE,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-            indexes_to_days_since_ath: ComputedVecsFromDateindex::forced_import(
-                path,
-                "days_since_ath",
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-            indexes_to_max_days_between_ath: ComputedVecsFromDateindex::forced_import(
-                path,
-                "max_days_between_ath",
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-            indexes_to_max_years_between_ath: ComputedVecsFromDateindex::forced_import(
-                path,
-                "max_years_between_ath",
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_last(),
-            )?,
-        })
-    }
-
-    pub fn compute(
-        &mut self,
-        indexer: &Indexer,
-        indexes: &indexes::Vecs,
-        fetched: &fetched::Vecs,
-        transactions: &mut transactions::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-    ) -> color_eyre::Result<()> {
-        self.indexes_to_marketcap.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                let mut total_subsidy_in_btc = transactions
-                    .indexes_to_subsidy
-                    .bitcoin
-                    .dateindex
-                    .unwrap_total()
-                    .into_iter();
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &fetched.timeindexes_to_close.dateindex,
-                    |(i, close, ..)| {
-                        let supply = total_subsidy_in_btc.unwrap_get_inner(i);
-                        (i, *close * supply)
-                    },
-                    exit,
-                )
-            },
-        )?;
-
-        self.indexes_to_ath.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                let mut prev = None;
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &fetched.timeindexes_to_high.dateindex,
-                    |(i, high, slf)| {
-                        if prev.is_none() {
-                            let i = i.unwrap_to_usize();
-                            prev.replace(if i > 0 {
-                                slf.into_iter().unwrap_get_inner_(i - 1)
-                            } else {
-                                Dollars::ZERO
-                            });
-                        }
-                        let ath = prev.unwrap().max(*high);
-                        prev.replace(ath);
-                        (i, ath)
-                    },
-                    exit,
-                )
-            },
-        )?;
-
-        self.indexes_to_drawdown.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                let mut close_iter = fetched.timeindexes_to_close.dateindex.into_iter();
-
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &self.indexes_to_ath.dateindex,
-                    |(i, ath, ..)| {
-                        if ath == Dollars::ZERO {
-                            return (i, StoredF64::default());
-                        }
-                        let close = *close_iter.unwrap_get_inner(i);
-                        let drawdown = StoredF64::from((*ath - *close) / *ath * -100.0);
-                        (i, drawdown)
-                    },
-                    exit,
-                )
-            },
-        )?;
-
-        self.indexes_to_days_since_ath.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                let mut high_iter = fetched.timeindexes_to_high.dateindex.into_iter();
-                let mut prev = None;
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &self.indexes_to_ath.dateindex,
-                    |(i, ath, slf)| {
-                        if prev.is_none() {
-                            let i = i.unwrap_to_usize();
-                            prev.replace(if i > 0 {
-                                slf.into_iter().unwrap_get_inner_(i - 1)
-                            } else {
-                                StoredUsize::default()
-                            });
-                        }
-                        let days = if *high_iter.unwrap_get_inner(i) == ath {
-                            StoredUsize::default()
-                        } else {
-                            prev.unwrap() + StoredUsize::from(1)
-                        };
-                        prev.replace(days);
-                        (i, days)
-                    },
-                    exit,
-                )
-            },
-        )?;
-
-        self.indexes_to_max_days_between_ath.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                let mut prev = None;
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &self.indexes_to_days_since_ath.dateindex,
-                    |(i, days, slf)| {
-                        if prev.is_none() {
-                            let i = i.unwrap_to_usize();
-                            prev.replace(if i > 0 {
-                                slf.into_iter().unwrap_get_inner_(i - 1)
-                            } else {
-                                StoredUsize::ZERO
-                            });
-                        }
-                        let max = prev.unwrap().max(days);
-                        prev.replace(max);
-                        (i, max)
-                    },
-                    exit,
-                )
-            },
-        )?;
-
-        self.indexes_to_max_years_between_ath.compute(
-            indexer,
-            indexes,
-            starting_indexes,
-            exit,
-            |v, _, _, starting_indexes, exit| {
-                v.compute_transform(
-                    starting_indexes.dateindex,
-                    &self.indexes_to_max_days_between_ath.dateindex,
-                    |(i, max, ..)| (i, StoredF64::from(*max as f64 / 365.0)),
-                    exit,
-                )
-            },
-        )?;
-
-        Ok(())
-    }
-
-    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
-        [
-            self.indexes_to_marketcap.vecs(),
-            self.indexes_to_ath.vecs(),
-            self.indexes_to_drawdown.vecs(),
-            self.indexes_to_days_since_ath.vecs(),
-            self.indexes_to_max_days_between_ath.vecs(),
-            self.indexes_to_max_years_between_ath.vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/mod.rs

@@ -1,118 +0,0 @@
-use std::{fs, path::Path};
-
-use brk_exit::Exit;
-use brk_fetcher::Fetcher;
-use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, Compressed, Computation};
-
-pub mod blocks;
-pub mod fetched;
-pub mod grouped;
-pub mod indexes;
-pub mod market;
-pub mod mining;
-pub mod transactions;
-
-pub use indexes::Indexes;
-
-#[derive(Clone)]
-pub struct Vecs {
-    pub indexes: indexes::Vecs,
-    pub blocks: blocks::Vecs,
-    pub mining: mining::Vecs,
-    pub market: market::Vecs,
-    pub transactions: transactions::Vecs,
-    pub fetched: Option<fetched::Vecs>,
-}
-
-impl Vecs {
-    pub fn import(
-        path: &Path,
-        indexer: &Indexer,
-        fetch: bool,
-        computation: Computation,
-        compressed: Compressed,
-    ) -> color_eyre::Result<Self> {
-        fs::create_dir_all(path)?;
-
-        let indexes = indexes::Vecs::forced_import(path, indexer, computation, compressed)?;
-
-        let fetched =
-            fetch.then(|| fetched::Vecs::forced_import(path, computation, compressed).unwrap());
-
-        Ok(Self {
-            blocks: blocks::Vecs::forced_import(path, computation, compressed)?,
-            mining: mining::Vecs::forced_import(path, computation, compressed)?,
-            market: market::Vecs::forced_import(path, computation, compressed)?,
-            transactions: transactions::Vecs::forced_import(
-                path,
-                indexer,
-                &indexes,
-                computation,
-                compressed,
-                fetched.as_ref(),
-            )?,
-            indexes,
-            fetched,
-        })
-    }
-
-    pub fn compute(
-        &mut self,
-        indexer: &Indexer,
-        starting_indexes: brk_indexer::Indexes,
-        fetcher: Option<&mut Fetcher>,
-        exit: &Exit,
-    ) -> color_eyre::Result<()> {
-        let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
-
-        self.blocks
-            .compute(indexer, &self.indexes, &starting_indexes, exit)?;
-
-        self.mining
-            .compute(indexer, &self.indexes, &starting_indexes, exit)?;
-
-        if let Some(fetched) = self.fetched.as_mut() {
-            fetched.compute(
-                indexer,
-                &self.indexes,
-                &starting_indexes,
-                fetcher.unwrap(),
-                exit,
-            )?;
-        }
-
-        self.transactions.compute(
-            indexer,
-            &self.indexes,
-            &starting_indexes,
-            self.fetched.as_ref(),
-            exit,
-        )?;
-
-        if let Some(fetched) = self.fetched.as_ref() {
-            self.market.compute(
-                indexer,
-                &self.indexes,
-                fetched,
-                &mut self.transactions,
-                &starting_indexes,
-                exit,
-            )?;
-        }
-
-        Ok(())
-    }
-
-    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
-        [
-            self.indexes.vecs(),
-            self.blocks.vecs(),
-            self.mining.vecs(),
-            self.market.vecs(),
-            self.transactions.vecs(),
-            self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/stores.rs

@@ -1,7 +1,10 @@
 use std::path::Path;
 
+use brk_core::Version;
 use fjall::TransactionalKeyspace;
 
+const _VERSION: Version = Version::ZERO;
+
 #[derive(Clone)]
 pub struct Stores {
     // pub address_to_utxos_received: Store<AddressIndexOutputIndex, Unit>,
@@ -9,18 +12,18 @@ pub struct Stores {
 }
 
 impl Stores {
-    pub fn import(_: &Path, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
+    pub fn import(_: &Path, _: Version, _: &TransactionalKeyspace) -> color_eyre::Result<Self> {
         // let address_to_utxos_received = Store::import(
         //     keyspace.clone(),
         //     path,
         //     "address_to_utxos_received",
-        //     Version::ZERO,
+        //     version + VERSION + Version::ZERO,
         // )?;
         // let address_to_utxos_spent = Store::import(
         //     keyspace.clone(),
         //     path,
         //     "address_to_utxos_spent",
-        //     Version::ZERO,
+        //     version + VERSION + Version::ZERO,
         // )?;
 
         Ok(Self {

crates/brk_computer/src/utils.rs

@@ -0,0 +1,27 @@
+use std::ops::{Add, Div};
+
+pub fn get_percentile<T>(sorted: &[T], percentile: f64) -> T
+where
+    T: Clone + Div<usize, Output = T> + Add<T, Output = T>,
+{
+    let len = sorted.len();
+
+    if len == 0 {
+        panic!();
+    } else if len == 1 {
+        sorted[0].clone()
+    } else {
+        let index = (len - 1) as f64 * percentile;
+
+        let fract = index.fract();
+
+        if fract != 0.0 {
+            let left = sorted.get(index as usize).unwrap().clone();
+            let right = sorted.get(index.ceil() as usize).unwrap().clone();
+            left / 2 + right / 2
+        } else {
+            // dbg!(sorted.len(), index);
+            sorted.get(index as usize).unwrap().clone()
+        }
+    }
+}

crates/brk_computer/src/vecs/blocks.rs

@@ -2,26 +2,27 @@ use std::{fs, path::Path};
 
 use brk_core::{
     CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
-    Timestamp, Weight,
+    Timestamp, Version, Weight,
 };
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_parser::bitcoin;
-use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
+use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
 
 use super::{
     Indexes,
-    grouped::{ComputedVecsFromDateindex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
+    grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
     indexes,
 };
 
+const VERSION: Version = Version::ZERO;
+
 #[derive(Clone)]
 pub struct Vecs {
     pub height_to_interval: EagerVec<Height, Timestamp>,
     pub height_to_vbytes: EagerVec<Height, StoredU64>,
     pub difficultyepoch_to_timestamp: EagerVec<DifficultyEpoch, Timestamp>,
     pub halvingepoch_to_timestamp: EagerVec<HalvingEpoch, Timestamp>,
-    pub timeindexes_to_timestamp: ComputedVecsFromDateindex<Timestamp>,
+    pub timeindexes_to_timestamp: ComputedVecsFromDateIndex<Timestamp>,
     pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
     pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
     pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
@@ -32,30 +33,33 @@ pub struct Vecs {
 impl Vecs {
     pub fn forced_import(
         path: &Path,
+        version: Version,
         _computation: Computation,
-        compressed: Compressed,
+        format: Format,
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
         Ok(Self {
             height_to_interval: EagerVec::forced_import(
-                &path.join("height_to_interval"),
-                Version::ZERO,
-                compressed,
+                path,
+                "interval",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
-            timeindexes_to_timestamp: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "timestamp",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
             indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
                 path,
                 "block_interval",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_minmax()
@@ -65,48 +69,59 @@ impl Vecs {
                 path,
                 "block_count",
                 true,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
                 path,
                 "block_weight",
                 false,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             indexes_to_block_size: ComputedVecsFromHeight::forced_import(
                 path,
                 "block_size",
                 false,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             height_to_vbytes: EagerVec::forced_import(
-                &path.join("height_to_vbytes"),
-                Version::ZERO,
-                compressed,
+                path,
+                "vbytes",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
                 path,
                 "block_vbytes",
                 false,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             difficultyepoch_to_timestamp: EagerVec::forced_import(
-                &path.join("difficultyepoch_to_timestamp"),
-                Version::ZERO,
-                compressed,
+                path,
+                "timestamp",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             halvingepoch_to_timestamp: EagerVec::forced_import(
-                &path.join("halvingepoch_to_timestamp"),
-                Version::ZERO,
-                compressed,
+                path,
+                "timestamp",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
         })
     }
@@ -118,7 +133,7 @@ impl Vecs {
         starting_indexes: &Indexes,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
-        self.timeindexes_to_timestamp.compute(
+        self.timeindexes_to_timestamp.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -242,6 +257,8 @@ impl Vecs {
             self.indexes_to_block_vbytes.vecs(),
             self.indexes_to_block_weight.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/constants.rs

@@ -0,0 +1,154 @@
+use std::{fs, path::Path};
+
+use brk_core::{StoredU8, Version};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
+
+use super::{
+    Indexes,
+    grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
+    indexes,
+};
+
+const VERSION: Version = Version::ZERO;
+
+#[derive(Clone)]
+pub struct Vecs {
+    pub _0: ComputedVecsFromHeight<StoredU8>,
+    pub _1: ComputedVecsFromHeight<StoredU8>,
+    pub _50: ComputedVecsFromHeight<StoredU8>,
+    pub _100: ComputedVecsFromHeight<StoredU8>,
+}
+
+impl Vecs {
+    pub fn forced_import(
+        path: &Path,
+        version: Version,
+        _computation: Computation,
+        format: Format,
+    ) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        Ok(Self {
+            _0: ComputedVecsFromHeight::forced_import(
+                path,
+                "0",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
+            _1: ComputedVecsFromHeight::forced_import(
+                path,
+                "1",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
+            _50: ComputedVecsFromHeight::forced_import(
+                path,
+                "50",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
+            _100: ComputedVecsFromHeight::forced_import(
+                path,
+                "100",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
+            )?,
+        })
+    }
+
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+    ) -> color_eyre::Result<()> {
+        self._0.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, indexes, starting_indexes, exit| {
+                vec.compute_to(
+                    starting_indexes.height,
+                    indexes.height_to_date.len(),
+                    indexes.height_to_date.version(),
+                    |i| (i, StoredU8::new(0)),
+                    exit,
+                )
+            },
+        )?;
+
+        self._1.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, indexes, starting_indexes, exit| {
+                vec.compute_to(
+                    starting_indexes.height,
+                    indexes.height_to_date.len(),
+                    indexes.height_to_date.version(),
+                    |i| (i, StoredU8::new(1)),
+                    exit,
+                )
+            },
+        )?;
+
+        self._50.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, indexes, starting_indexes, exit| {
+                vec.compute_to(
+                    starting_indexes.height,
+                    indexes.height_to_date.len(),
+                    indexes.height_to_date.version(),
+                    |i| (i, StoredU8::new(50)),
+                    exit,
+                )
+            },
+        )?;
+
+        self._100.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, indexes, starting_indexes, exit| {
+                vec.compute_to(
+                    starting_indexes.height,
+                    indexes.height_to_date.len(),
+                    indexes.height_to_date.version(),
+                    |i| (i, StoredU8::new(100)),
+                    exit,
+                )
+            },
+        )?;
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            self._0.vecs(),
+            self._1.vecs(),
+            self._50.vecs(),
+            self._100.vecs(),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}

crates/brk_computer/src/vecs/fetched.rs

@@ -2,17 +2,17 @@ use std::{fs, path::Path};
 
 use brk_core::{
     Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
-    OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, WeekIndex, YearIndex,
+    OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, Version, WeekIndex, YearIndex,
 };
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
+use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
 
 use super::{
     Indexes,
     grouped::{
-        ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions,
+        ComputedVecsFromDateIndex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions,
     },
     indexes,
 };
@@ -33,14 +33,14 @@ pub struct Vecs {
     pub height_to_ohlc_in_sats: EagerVec<Height, OHLCSats>,
     pub height_to_ohlc_in_cents: EagerVec<Height, OHLCCents>,
     pub height_to_open_in_cents: EagerVec<Height, Open<Cents>>,
-    pub timeindexes_to_close: ComputedVecsFromDateindex<Close<Dollars>>,
-    pub timeindexes_to_high: ComputedVecsFromDateindex<High<Dollars>>,
-    pub timeindexes_to_low: ComputedVecsFromDateindex<Low<Dollars>>,
-    pub timeindexes_to_open: ComputedVecsFromDateindex<Open<Dollars>>,
-    pub timeindexes_to_open_in_sats: ComputedVecsFromDateindex<Open<Sats>>,
-    pub timeindexes_to_high_in_sats: ComputedVecsFromDateindex<High<Sats>>,
-    pub timeindexes_to_low_in_sats: ComputedVecsFromDateindex<Low<Sats>>,
-    pub timeindexes_to_close_in_sats: ComputedVecsFromDateindex<Close<Sats>>,
+    pub timeindexes_to_close: ComputedVecsFromDateIndex<Close<Dollars>>,
+    pub timeindexes_to_high: ComputedVecsFromDateIndex<High<Dollars>>,
+    pub timeindexes_to_low: ComputedVecsFromDateIndex<Low<Dollars>>,
+    pub timeindexes_to_open: ComputedVecsFromDateIndex<Open<Dollars>>,
+    pub timeindexes_to_open_in_sats: ComputedVecsFromDateIndex<Open<Sats>>,
+    pub timeindexes_to_high_in_sats: ComputedVecsFromDateIndex<High<Sats>>,
+    pub timeindexes_to_low_in_sats: ComputedVecsFromDateIndex<Low<Sats>>,
+    pub timeindexes_to_close_in_sats: ComputedVecsFromDateIndex<Close<Sats>>,
     pub chainindexes_to_close: ComputedVecsFromHeightStrict<Close<Dollars>>,
     pub chainindexes_to_high: ComputedVecsFromHeightStrict<High<Dollars>>,
     pub chainindexes_to_low: ComputedVecsFromHeightStrict<Low<Dollars>>,
@@ -66,13 +66,14 @@ pub struct Vecs {
 }
 
 const VERSION: Version = Version::ZERO;
-const VERSION_IN_SATS: Version = Version::ONE;
+const VERSION_IN_SATS: Version = Version::ZERO;
 
 impl Vecs {
     pub fn forced_import(
         path: &Path,
+        version: Version,
         _computation: Computation,
-        compressed: Compressed,
+        format: Format,
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
@@ -82,247 +83,282 @@ impl Vecs {
 
         Ok(Self {
             dateindex_to_ohlc_in_cents: EagerVec::forced_import(
-                &fetched_path.join("dateindex_to_ohlc_in_cents"),
-                Version::ZERO,
-                compressed,
+                &fetched_path,
+                "ohlc_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             dateindex_to_ohlc: EagerVec::forced_import(
-                &path.join("dateindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             dateindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("dateindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             dateindex_to_close_in_cents: EagerVec::forced_import(
-                &path.join("dateindex_to_close_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "close_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             dateindex_to_high_in_cents: EagerVec::forced_import(
-                &path.join("dateindex_to_high_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "high_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             dateindex_to_low_in_cents: EagerVec::forced_import(
-                &path.join("dateindex_to_low_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "low_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             dateindex_to_open_in_cents: EagerVec::forced_import(
-                &path.join("dateindex_to_open_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "open_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_ohlc_in_cents: EagerVec::forced_import(
-                &fetched_path.join("height_to_ohlc_in_cents"),
-                Version::ZERO,
-                compressed,
+                &fetched_path,
+                "ohlc_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_ohlc: EagerVec::forced_import(
-                &path.join("height_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("height_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             height_to_close_in_cents: EagerVec::forced_import(
-                &path.join("height_to_close_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "close_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_high_in_cents: EagerVec::forced_import(
-                &path.join("height_to_high_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "high_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_low_in_cents: EagerVec::forced_import(
-                &path.join("height_to_low_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "low_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_open_in_cents: EagerVec::forced_import(
-                &path.join("height_to_open_in_cents"),
-                Version::ZERO,
-                compressed,
+                path,
+                "open_in_cents",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
-            timeindexes_to_open: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "open",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
-            timeindexes_to_high: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "high",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
-            timeindexes_to_low: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "low",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
-            timeindexes_to_close: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "close",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            timeindexes_to_open_in_sats: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "open_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
-            timeindexes_to_high_in_sats: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "high_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
-            timeindexes_to_low_in_sats: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "low_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
-            timeindexes_to_close_in_sats: ComputedVecsFromDateindex::forced_import(
+            timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "close_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
             chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "open",
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
             chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "high",
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
             chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "low",
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
             chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "close",
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
             chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "open_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
             chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "high_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
             chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "low_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
             chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
                 path,
                 "close_in_sats",
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
             weekindex_to_ohlc: EagerVec::forced_import(
-                &path.join("weekindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             weekindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("weekindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             difficultyepoch_to_ohlc: EagerVec::forced_import(
-                &path.join("difficultyepoch_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("difficultyepoch_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             monthindex_to_ohlc: EagerVec::forced_import(
-                &path.join("monthindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             monthindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("monthindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             quarterindex_to_ohlc: EagerVec::forced_import(
-                &path.join("quarterindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("quarterindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
             yearindex_to_ohlc: EagerVec::forced_import(
-                &path.join("yearindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             yearindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("yearindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
-            // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?,
+            // halvingepoch_to_ohlc: StorableVec::forced_import(path,
+            // "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
             decadeindex_to_ohlc: EagerVec::forced_import(
-                &path.join("decadeindex_to_ohlc"),
-                Version::ZERO,
-                compressed,
+                path,
+                "ohlc",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
-                &path.join("decadeindex_to_ohlc_in_sats"),
-                VERSION + VERSION_IN_SATS + Version::ZERO,
-                compressed,
+                path,
+                "ohlc_in_sats",
+                version + VERSION + VERSION_IN_SATS + Version::ZERO,
+                format,
             )?,
         })
     }
@@ -435,7 +471,7 @@ impl Vecs {
             exit,
         )?;
 
-        self.timeindexes_to_close.compute(
+        self.timeindexes_to_close.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -450,7 +486,7 @@ impl Vecs {
             },
         )?;
 
-        self.timeindexes_to_high.compute(
+        self.timeindexes_to_high.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -465,7 +501,7 @@ impl Vecs {
             },
         )?;
 
-        self.timeindexes_to_low.compute(
+        self.timeindexes_to_low.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -480,7 +516,7 @@ impl Vecs {
             },
         )?;
 
-        self.timeindexes_to_open.compute(
+        self.timeindexes_to_open.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -748,7 +784,7 @@ impl Vecs {
             },
         )?;
 
-        self.timeindexes_to_open_in_sats.compute(
+        self.timeindexes_to_open_in_sats.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -756,14 +792,14 @@ impl Vecs {
             |v, _, _, starting_indexes, exit| {
                 v.compute_transform(
                     starting_indexes.dateindex,
-                    &self.timeindexes_to_open.dateindex,
+                    self.timeindexes_to_open.dateindex.as_ref().unwrap(),
                     |(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
                     exit,
                 )
             },
         )?;
 
-        self.timeindexes_to_high_in_sats.compute(
+        self.timeindexes_to_high_in_sats.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -771,14 +807,14 @@ impl Vecs {
             |v, _, _, starting_indexes, exit| {
                 v.compute_transform(
                     starting_indexes.dateindex,
-                    &self.timeindexes_to_low.dateindex,
+                    self.timeindexes_to_low.dateindex.as_ref().unwrap(),
                     |(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
                     exit,
                 )
             },
         )?;
 
-        self.timeindexes_to_low_in_sats.compute(
+        self.timeindexes_to_low_in_sats.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -786,14 +822,14 @@ impl Vecs {
             |v, _, _, starting_indexes, exit| {
                 v.compute_transform(
                     starting_indexes.dateindex,
-                    &self.timeindexes_to_high.dateindex,
+                    self.timeindexes_to_high.dateindex.as_ref().unwrap(),
                     |(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
                     exit,
                 )
             },
         )?;
 
-        self.timeindexes_to_close_in_sats.compute(
+        self.timeindexes_to_close_in_sats.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -801,7 +837,7 @@ impl Vecs {
             |v, _, _, starting_indexes, exit| {
                 v.compute_transform(
                     starting_indexes.dateindex,
-                    &self.timeindexes_to_close.dateindex,
+                    self.timeindexes_to_close.dateindex.as_ref().unwrap(),
                     |(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
                     exit,
                 )
@@ -828,12 +864,30 @@ impl Vecs {
             exit,
         )?;
 
-        let mut dateindex_first_iter = self.timeindexes_to_open_in_sats.dateindex.iter();
-        let mut dateindex_max_iter = self.timeindexes_to_high_in_sats.dateindex.iter();
-        let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
+        let mut dateindex_first_iter = self
+            .timeindexes_to_open_in_sats
+            .dateindex
+            .as_ref()
+            .unwrap()
+            .iter();
+        let mut dateindex_max_iter = self
+            .timeindexes_to_high_in_sats
+            .dateindex
+            .as_ref()
+            .unwrap()
+            .iter();
+        let mut dateindex_min_iter = self
+            .timeindexes_to_low_in_sats
+            .dateindex
+            .as_ref()
+            .unwrap()
+            .iter();
         self.dateindex_to_ohlc_in_sats.compute_transform(
             starting_indexes.dateindex,
-            &self.timeindexes_to_close_in_sats.dateindex,
+            self.timeindexes_to_close_in_sats
+                .dateindex
+                .as_ref()
+                .unwrap(),
             |(i, close, ..)| {
                 (
                     i,
@@ -1097,6 +1151,8 @@ impl Vecs {
             self.chainindexes_to_low_in_sats.vecs(),
             self.chainindexes_to_open_in_sats.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/grouped/builder.rs

@@ -1,13 +1,12 @@
 use std::path::Path;
 
-use brk_core::{CheckedSub, StoredUsize};
+use brk_core::{CheckedSub, Result, StoredUsize, Version};
 use brk_exit::Exit;
-use brk_vec::{
-    AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, StoredIndex, StoredType,
-    Version,
-};
+use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
 use color_eyre::eyre::ContextCompat;
 
+use crate::utils::get_percentile;
+
 use super::ComputedType;
 
 #[derive(Clone, Debug)]
@@ -16,18 +15,18 @@ where
     I: StoredIndex,
     T: ComputedType,
 {
-    first: Option<EagerVec<I, T>>,
-    average: Option<EagerVec<I, T>>,
-    sum: Option<EagerVec<I, T>>,
-    max: Option<EagerVec<I, T>>,
-    _90p: Option<EagerVec<I, T>>,
-    _75p: Option<EagerVec<I, T>>,
-    median: Option<EagerVec<I, T>>,
-    _25p: Option<EagerVec<I, T>>,
-    _10p: Option<EagerVec<I, T>>,
-    min: Option<EagerVec<I, T>>,
-    last: Option<EagerVec<I, T>>,
-    total: Option<EagerVec<I, T>>,
+    pub first: Option<Box<EagerVec<I, T>>>,
+    pub average: Option<Box<EagerVec<I, T>>>,
+    pub sum: Option<Box<EagerVec<I, T>>>,
+    pub max: Option<Box<EagerVec<I, T>>>,
+    pub _90p: Option<Box<EagerVec<I, T>>>,
+    pub _75p: Option<Box<EagerVec<I, T>>>,
+    pub median: Option<Box<EagerVec<I, T>>>,
+    pub _25p: Option<Box<EagerVec<I, T>>>,
+    pub _10p: Option<Box<EagerVec<I, T>>>,
+    pub min: Option<Box<EagerVec<I, T>>>,
+    pub last: Option<Box<EagerVec<I, T>>>,
+    pub cumulative: Option<Box<EagerVec<I, T>>>,
 }
 
 const VERSION: Version = Version::ZERO;
@@ -41,97 +40,157 @@ where
         path: &Path,
         name: &str,
         version: Version,
-        compressed: Compressed,
+        format: Format,
         options: StorableVecGeneatorOptions,
     ) -> color_eyre::Result<Self> {
-        let key = I::to_string().split("::").last().unwrap().to_lowercase();
-
         let only_one_active = options.is_only_one_active();
 
-        let default = || path.join(format!("{key}_to_{name}"));
-
-        let prefix = |s: &str| path.join(format!("{key}_to_{s}_{name}"));
+        let prefix = |s: &str| format!("{s}_{name}");
 
         let maybe_prefix = |s: &str| {
             if only_one_active {
-                default()
+                name.to_string()
             } else {
                 prefix(s)
             }
         };
 
-        let suffix = |s: &str| path.join(format!("{key}_to_{name}_{s}"));
+        let suffix = |s: &str| format!("{name}_{s}");
 
         let maybe_suffix = |s: &str| {
             if only_one_active {
-                default()
+                name.to_string()
             } else {
                 suffix(s)
             }
         };
 
-        let version = VERSION + version;
-
         let s = Self {
             first: options.first.then(|| {
-                EagerVec::forced_import(&maybe_prefix("first"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_prefix("first"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             last: options.last.then(|| {
-                EagerVec::forced_import(
-                    &path.join(format!("{key}_to_{name}")),
-                    version + Version::ZERO,
-                    compressed,
+                Box::new(
+                    EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
                 )
-                .unwrap()
             }),
             min: options.min.then(|| {
-                EagerVec::forced_import(&maybe_suffix("min"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("min"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             max: options.max.then(|| {
-                EagerVec::forced_import(&maybe_suffix("max"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("max"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             median: options.median.then(|| {
-                EagerVec::forced_import(
-                    &maybe_suffix("median"),
-                    version + Version::ZERO,
-                    compressed,
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("median"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
                 )
-                .unwrap()
             }),
             average: options.average.then(|| {
-                EagerVec::forced_import(
-                    &maybe_suffix("average"),
-                    version + Version::ZERO,
-                    compressed,
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("average"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
                 )
-                .unwrap()
             }),
             sum: options.sum.then(|| {
-                EagerVec::forced_import(&maybe_suffix("sum"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("sum"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
-            total: options.total.then(|| {
-                EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed)
-                    .unwrap()
+            cumulative: options.cumulative.then(|| {
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &prefix("cumulative"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             _90p: options._90p.then(|| {
-                EagerVec::forced_import(&maybe_suffix("90p"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("90p"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             _75p: options._75p.then(|| {
-                EagerVec::forced_import(&maybe_suffix("75p"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("75p"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             _25p: options._25p.then(|| {
-                EagerVec::forced_import(&maybe_suffix("25p"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("25p"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
             _10p: options._10p.then(|| {
-                EagerVec::forced_import(&maybe_suffix("10p"), version + Version::ZERO, compressed)
-                    .unwrap()
+                Box::new(
+                    EagerVec::forced_import(
+                        path,
+                        &maybe_suffix("10p"),
+                        version + VERSION + Version::ZERO,
+                        format,
+                    )
+                    .unwrap(),
+                )
             }),
         };
 
@@ -144,20 +203,22 @@ where
         source: &impl AnyIterableVec<I, T>,
         exit: &Exit,
     ) -> Result<()> {
-        if self.total.is_none() {
+        if self.cumulative.is_none() {
             return Ok(());
         };
 
+        self.validate_computed_version_or_reset_file(source.version())?;
+
         let index = self.starting_index(max_from);
 
-        let total_vec = self.total.as_mut().unwrap();
+        let cumulative_vec = self.cumulative.as_mut().unwrap();
 
-        let mut total = index.decremented().map_or(T::from(0_usize), |index| {
-            total_vec.iter().unwrap_get_inner(index)
+        let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
+            cumulative_vec.iter().unwrap_get_inner(index)
         });
         source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
-            total = total.clone() + v.into_inner();
-            total_vec.forced_push_at(i, total.clone(), exit)
+            cumulative = cumulative.clone() + v.into_inner();
+            cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
         })?;
 
         self.safe_flush(exit)?;
@@ -176,20 +237,20 @@ where
     where
         I2: StoredIndex + StoredType + CheckedSub<I2>,
     {
-        let index = self.starting_index(max_from);
-
         self.validate_computed_version_or_reset_file(
             source.version() + first_indexes.version() + count_indexes.version(),
         )?;
 
+        let index = self.starting_index(max_from);
+
         let mut count_indexes_iter = count_indexes.iter();
         let mut source_iter = source.iter();
 
-        let total_vec = self.total.as_mut();
+        let cumulative_vec = self.cumulative.as_mut();
 
-        let mut total = total_vec.map(|total_vec| {
+        let mut cumulative = cumulative_vec.map(|cumulative_vec| {
             index.decremented().map_or(T::from(0_usize), |index| {
-                total_vec.iter().unwrap_get_inner(index)
+                cumulative_vec.iter().unwrap_get_inner(index)
             })
         });
 
@@ -223,8 +284,9 @@ where
                     last.forced_push_at(index, v, exit)?;
                 }
 
-                let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
-                let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
+                let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
+                let needs_average_sum_or_cumulative =
+                    needs_sum_or_cumulative || self.average.is_some();
                 let needs_sorted = self.max.is_some()
                     || self._90p.is_some()
                     || self._75p.is_some()
@@ -232,7 +294,7 @@ where
                     || self._25p.is_some()
                     || self._10p.is_some()
                     || self.min.is_some();
-                let needs_values = needs_sorted || needs_average_sum_or_total;
+                let needs_values = needs_sorted || needs_average_sum_or_cumulative;
 
                 if needs_values {
                     source_iter.set(first_index);
@@ -269,23 +331,23 @@ where
                         }
 
                         if let Some(_90p) = self._90p.as_mut() {
-                            _90p.forced_push_at(i, Self::get_percentile(&values, 0.90), exit)?;
+                            _90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?;
                         }
 
                         if let Some(_75p) = self._75p.as_mut() {
-                            _75p.forced_push_at(i, Self::get_percentile(&values, 0.75), exit)?;
+                            _75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?;
                         }
 
                         if let Some(median) = self.median.as_mut() {
-                            median.forced_push_at(i, Self::get_percentile(&values, 0.50), exit)?;
+                            median.forced_push_at(i, get_percentile(&values, 0.50), exit)?;
                         }
 
                         if let Some(_25p) = self._25p.as_mut() {
-                            _25p.forced_push_at(i, Self::get_percentile(&values, 0.25), exit)?;
+                            _25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?;
                         }
 
                         if let Some(_10p) = self._10p.as_mut() {
-                            _10p.forced_push_at(i, Self::get_percentile(&values, 0.10), exit)?;
+                            _10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?;
                         }
 
                         if let Some(min) = self.min.as_mut() {
@@ -293,7 +355,7 @@ where
                         }
                     }
 
-                    if needs_average_sum_or_total {
+                    if needs_average_sum_or_cumulative {
                         let len = values.len();
                         let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
 
@@ -302,15 +364,15 @@ where
                             average.forced_push_at(i, avg, exit)?;
                         }
 
-                        if needs_sum_or_total {
+                        if needs_sum_or_cumulative {
                             if let Some(sum_vec) = self.sum.as_mut() {
                                 sum_vec.forced_push_at(i, sum.clone(), exit)?;
                             }
 
-                            if let Some(total_vec) = self.total.as_mut() {
-                                let t = total.as_ref().unwrap().clone() + sum;
-                                total.replace(t.clone());
-                                total_vec.forced_push_at(i, t, exit)?;
+                            if let Some(cumulative_vec) = self.cumulative.as_mut() {
+                                let t = cumulative.as_ref().unwrap().clone() + sum;
+                                cumulative.replace(t.clone());
+                                cumulative_vec.forced_push_at(i, t, exit)?;
                             }
                         }
                     }
@@ -360,9 +422,9 @@ where
         let mut source_average_iter = source.average.as_ref().map(|f| f.iter());
         let mut source_sum_iter = source.sum.as_ref().map(|f| f.iter());
 
-        let mut total = self.total.as_mut().map(|total_vec| {
+        let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
             index.decremented().map_or(T::from(0_usize), |index| {
-                total_vec.iter().unwrap_get_inner(index)
+                cumulative_vec.iter().unwrap_get_inner(index)
             })
         });
 
@@ -394,10 +456,11 @@ where
                     last.forced_push_at(index, v, exit)?;
                 }
 
-                let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
-                let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
+                let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
+                let needs_average_sum_or_cumulative =
+                    needs_sum_or_cumulative || self.average.is_some();
                 let needs_sorted = self.max.is_some() || self.min.is_some();
-                let needs_values = needs_sorted || needs_average_sum_or_total;
+                let needs_values = needs_sorted || needs_average_sum_or_cumulative;
 
                 if needs_values {
                     if needs_sorted {
@@ -424,7 +487,7 @@ where
                         }
                     }
 
-                    if needs_average_sum_or_total {
+                    if needs_average_sum_or_cumulative {
                         if let Some(average) = self.average.as_mut() {
                             let source_average_iter = source_average_iter.as_mut().unwrap();
                             source_average_iter.set(first_index);
@@ -434,14 +497,14 @@ where
                                 .collect::<Vec<_>>();
 
                             let len = values.len();
-                            let total = values.into_iter().fold(T::from(0), |a, b| a + b);
-                            // TODO: Multiply by count then divide by total
+                            let cumulative = values.into_iter().fold(T::from(0), |a, b| a + b);
+                            // TODO: Multiply by count then divide by cumulative
                             // Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
-                            let avg = total / len;
+                            let avg = cumulative / len;
                             average.forced_push_at(i, avg, exit)?;
                         }
 
-                        if needs_sum_or_total {
+                        if needs_sum_or_cumulative {
                             let source_sum_iter = source_sum_iter.as_mut().unwrap();
                             source_sum_iter.set(first_index);
                             let values = source_sum_iter
@@ -455,10 +518,10 @@ where
                                 sum_vec.forced_push_at(i, sum.clone(), exit)?;
                             }
 
-                            if let Some(total_vec) = self.total.as_mut() {
-                                let t = total.as_ref().unwrap().clone() + sum;
-                                total.replace(t.clone());
-                                total_vec.forced_push_at(i, t, exit)?;
+                            if let Some(cumulative_vec) = self.cumulative.as_mut() {
+                                let t = cumulative.as_ref().unwrap().clone() + sum;
+                                cumulative.replace(t.clone());
+                                cumulative_vec.forced_push_at(i, t, exit)?;
                             }
                         }
                     }
@@ -472,116 +535,94 @@ where
         Ok(())
     }
 
-    fn get_percentile(sorted: &[T], percentile: f64) -> T {
-        let len = sorted.len();
-
-        if len == 0 {
-            panic!();
-        } else if len == 1 {
-            sorted[0].clone()
-        } else {
-            let index = (len - 1) as f64 * percentile;
-
-            let fract = index.fract();
-
-            if fract != 0.0 {
-                let left = sorted.get(index as usize).unwrap().clone();
-                let right = sorted.get(index.ceil() as usize).unwrap().clone();
-                left / 2 + right / 2
-            } else {
-                sorted.get(index as usize).unwrap().clone()
-            }
-        }
-    }
-
-    fn starting_index(&self, max_from: I) -> I {
+    pub fn starting_index(&self, max_from: I) -> I {
         max_from.min(I::from(
             self.vecs().into_iter().map(|v| v.len()).min().unwrap(),
         ))
     }
 
-    pub fn unwrap_first(&mut self) -> &mut EagerVec<I, T> {
-        self.first.as_mut().unwrap()
+    pub fn unwrap_first(&self) -> &EagerVec<I, T> {
+        self.first.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_average(&mut self) -> &mut EagerVec<I, T> {
-        self.average.as_mut().unwrap()
+    pub fn unwrap_average(&self) -> &EagerVec<I, T> {
+        self.average.as_ref().unwrap()
     }
-    pub fn unwrap_sum(&mut self) -> &mut EagerVec<I, T> {
-        self.sum.as_mut().unwrap()
+    pub fn unwrap_sum(&self) -> &EagerVec<I, T> {
+        self.sum.as_ref().unwrap()
     }
-    pub fn unwrap_max(&mut self) -> &mut EagerVec<I, T> {
-        self.max.as_mut().unwrap()
+    pub fn unwrap_max(&self) -> &EagerVec<I, T> {
+        self.max.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_90p(&mut self) -> &mut EagerVec<I, T> {
-        self._90p.as_mut().unwrap()
+    pub fn unwrap_90p(&self) -> &EagerVec<I, T> {
+        self._90p.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_75p(&mut self) -> &mut EagerVec<I, T> {
-        self._75p.as_mut().unwrap()
+    pub fn unwrap_75p(&self) -> &EagerVec<I, T> {
+        self._75p.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_median(&mut self) -> &mut EagerVec<I, T> {
-        self.median.as_mut().unwrap()
+    pub fn unwrap_median(&self) -> &EagerVec<I, T> {
+        self.median.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_25p(&mut self) -> &mut EagerVec<I, T> {
-        self._25p.as_mut().unwrap()
+    pub fn unwrap_25p(&self) -> &EagerVec<I, T> {
+        self._25p.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_10p(&mut self) -> &mut EagerVec<I, T> {
-        self._10p.as_mut().unwrap()
+    pub fn unwrap_10p(&self) -> &EagerVec<I, T> {
+        self._10p.as_ref().unwrap()
     }
-    pub fn unwrap_min(&mut self) -> &mut EagerVec<I, T> {
-        self.min.as_mut().unwrap()
+    pub fn unwrap_min(&self) -> &EagerVec<I, T> {
+        self.min.as_ref().unwrap()
     }
-    pub fn unwrap_last(&mut self) -> &mut EagerVec<I, T> {
-        self.last.as_mut().unwrap()
+    pub fn unwrap_last(&self) -> &EagerVec<I, T> {
+        self.last.as_ref().unwrap()
     }
     #[allow(unused)]
-    pub fn unwrap_total(&mut self) -> &mut EagerVec<I, T> {
-        self.total.as_mut().unwrap()
+    pub fn unwrap_cumulative(&self) -> &EagerVec<I, T> {
+        self.cumulative.as_ref().unwrap()
     }
 
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
         let mut v: Vec<&dyn AnyCollectableVec> = vec![];
 
         if let Some(first) = self.first.as_ref() {
-            v.push(first);
+            v.push(first.as_ref());
         }
         if let Some(last) = self.last.as_ref() {
-            v.push(last);
+            v.push(last.as_ref());
         }
         if let Some(min) = self.min.as_ref() {
-            v.push(min);
+            v.push(min.as_ref());
         }
         if let Some(max) = self.max.as_ref() {
-            v.push(max);
+            v.push(max.as_ref());
         }
         if let Some(median) = self.median.as_ref() {
-            v.push(median);
+            v.push(median.as_ref());
         }
         if let Some(average) = self.average.as_ref() {
-            v.push(average);
+            v.push(average.as_ref());
         }
         if let Some(sum) = self.sum.as_ref() {
-            v.push(sum);
+            v.push(sum.as_ref());
         }
-        if let Some(total) = self.total.as_ref() {
-            v.push(total);
+        if let Some(cumulative) = self.cumulative.as_ref() {
+            v.push(cumulative.as_ref());
         }
         if let Some(_90p) = self._90p.as_ref() {
-            v.push(_90p);
+            v.push(_90p.as_ref());
         }
         if let Some(_75p) = self._75p.as_ref() {
-            v.push(_75p);
+            v.push(_75p.as_ref());
         }
         if let Some(_25p) = self._25p.as_ref() {
-            v.push(_25p);
+            v.push(_25p.as_ref());
         }
         if let Some(_10p) = self._10p.as_ref() {
-            v.push(_10p);
+            v.push(_10p.as_ref());
         }
 
         v
@@ -609,8 +650,8 @@ where
         if let Some(sum) = self.sum.as_mut() {
             sum.safe_flush(exit)?;
         }
-        if let Some(total) = self.total.as_mut() {
-            total.safe_flush(exit)?;
+        if let Some(cumulative) = self.cumulative.as_mut() {
+            cumulative.safe_flush(exit)?;
         }
         if let Some(_90p) = self._90p.as_mut() {
             _90p.safe_flush(exit)?;
@@ -628,7 +669,7 @@ where
         Ok(())
     }
 
-    fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
+    pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
         if let Some(first) = self.first.as_mut() {
             first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
         }
@@ -650,8 +691,8 @@ where
         if let Some(sum) = self.sum.as_mut() {
             sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
         }
-        if let Some(total) = self.total.as_mut() {
-            total.validate_computed_version_or_reset_file(Version::ZERO + version)?;
+        if let Some(cumulative) = self.cumulative.as_mut() {
+            cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
         }
         if let Some(_90p) = self._90p.as_mut() {
             _90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
@@ -683,7 +724,7 @@ pub struct StorableVecGeneatorOptions {
     min: bool,
     first: bool,
     last: bool,
-    total: bool,
+    cumulative: bool,
 }
 
 impl StorableVecGeneatorOptions {
@@ -747,8 +788,8 @@ impl StorableVecGeneatorOptions {
         self
     }
 
-    pub fn add_total(mut self) -> Self {
-        self.total = true;
+    pub fn add_cumulative(mut self) -> Self {
+        self.cumulative = true;
         self
     }
 
@@ -807,8 +848,8 @@ impl StorableVecGeneatorOptions {
     }
 
     #[allow(unused)]
-    pub fn rm_total(mut self) -> Self {
-        self.total = false;
+    pub fn rm_cumulative(mut self) -> Self {
+        self.cumulative = false;
         self
     }
 
@@ -849,7 +890,7 @@ impl StorableVecGeneatorOptions {
             self.min,
             self.first,
             self.last,
-            self.total,
+            self.cumulative,
         ]
         .iter()
         .filter(|b| **b)
@@ -859,7 +900,7 @@ impl StorableVecGeneatorOptions {
 
     pub fn copy_self_extra(&self) -> Self {
         Self {
-            total: self.total,
+            cumulative: self.cumulative,
             ..Self::default()
         }
     }

crates/brk_computer/src/vecs/grouped/from_dateindex.rs

@@ -0,0 +1,218 @@
+use std::path::Path;
+
+use brk_core::{
+    DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex,
+};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
+
+use crate::vecs::{Indexes, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromDateIndex<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub dateindex: Option<EagerVec<DateIndex, T>>,
+    pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
+    pub weekindex: ComputedVecBuilder<WeekIndex, T>,
+    pub monthindex: ComputedVecBuilder<MonthIndex, T>,
+    pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
+    pub yearindex: ComputedVecBuilder<YearIndex, T>,
+    pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
+}
+
+const VERSION: Version = Version::ZERO;
+
+impl<T> ComputedVecsFromDateIndex<T>
+where
+    T: ComputedType,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        format: Format,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let dateindex = compute_source.then(|| {
+            EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
+        });
+
+        let dateindex_extra = ComputedVecBuilder::forced_import(
+            path,
+            name,
+            version + VERSION + Version::ZERO,
+            format,
+            options.copy_self_extra(),
+        )?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            dateindex,
+            dateindex_extra,
+            weekindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            monthindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            quarterindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            yearindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            decadeindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+        })
+    }
+
+    pub fn compute_all<F>(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut EagerVec<DateIndex, T>,
+            &Indexer,
+            &indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            self.dateindex.as_mut().unwrap(),
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        let dateindex: Option<&EagerVec<DateIndex, T>> = None;
+        self.compute_rest(indexes, starting_indexes, exit, dateindex)
+    }
+
+    pub fn compute_rest(
+        &mut self,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
+    ) -> color_eyre::Result<()> {
+        if let Some(dateindex) = dateindex {
+            self.dateindex_extra
+                .extend(starting_indexes.dateindex, dateindex, exit)?;
+
+            self.weekindex.compute(
+                starting_indexes.weekindex,
+                dateindex,
+                &indexes.weekindex_to_first_dateindex,
+                &indexes.weekindex_to_dateindex_count,
+                exit,
+            )?;
+
+            self.monthindex.compute(
+                starting_indexes.monthindex,
+                dateindex,
+                &indexes.monthindex_to_first_dateindex,
+                &indexes.monthindex_to_dateindex_count,
+                exit,
+            )?;
+        } else {
+            let dateindex = self.dateindex.as_ref().unwrap();
+
+            self.dateindex_extra
+                .extend(starting_indexes.dateindex, dateindex, exit)?;
+
+            self.weekindex.compute(
+                starting_indexes.weekindex,
+                dateindex,
+                &indexes.weekindex_to_first_dateindex,
+                &indexes.weekindex_to_dateindex_count,
+                exit,
+            )?;
+
+            self.monthindex.compute(
+                starting_indexes.monthindex,
+                dateindex,
+                &indexes.monthindex_to_first_dateindex,
+                &indexes.monthindex_to_dateindex_count,
+                exit,
+            )?;
+        }
+
+        self.quarterindex.from_aligned(
+            starting_indexes.quarterindex,
+            &self.monthindex,
+            &indexes.quarterindex_to_first_monthindex,
+            &indexes.quarterindex_to_monthindex_count,
+            exit,
+        )?;
+
+        self.yearindex.from_aligned(
+            starting_indexes.yearindex,
+            &self.monthindex,
+            &indexes.yearindex_to_first_monthindex,
+            &indexes.yearindex_to_monthindex_count,
+            exit,
+        )?;
+
+        self.decadeindex.from_aligned(
+            starting_indexes.decadeindex,
+            &self.yearindex,
+            &indexes.decadeindex_to_first_yearindex,
+            &indexes.decadeindex_to_yearindex_count,
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            self.dateindex
+                .as_ref()
+                .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
+            self.dateindex_extra.vecs(),
+            self.weekindex.vecs(),
+            self.monthindex.vecs(),
+            self.quarterindex.vecs(),
+            self.yearindex.vecs(),
+            self.decadeindex.vecs(),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}

crates/brk_computer/src/vecs/grouped/from_height.rs

@@ -1,13 +1,14 @@
 use std::path::Path;
 
 use brk_core::{
-    DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex,
+    DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version,
+    WeekIndex, YearIndex,
 };
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, Version};
+use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format};
 
-use crate::storage::{Indexes, indexes};
+use crate::vecs::{Indexes, indexes};
 
 use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
 
@@ -40,26 +41,28 @@ where
         name: &str,
         compute_source: bool,
         version: Version,
-        compressed: Compressed,
+        format: Format,
         options: StorableVecGeneatorOptions,
     ) -> color_eyre::Result<Self> {
-        let version = VERSION + version;
-
         let height = compute_source.then(|| {
-            EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)
-                .unwrap()
+            EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
         });
 
         let height_extra = ComputedVecBuilder::forced_import(
             path,
             name,
-            version,
-            compressed,
+            version + VERSION + Version::ZERO,
+            format,
             options.copy_self_extra(),
         )?;
 
-        let dateindex =
-            ComputedVecBuilder::forced_import(path, name, version, compressed, options)?;
+        let dateindex = ComputedVecBuilder::forced_import(
+            path,
+            name,
+            version + VERSION + Version::ZERO,
+            format,
+            options,
+        )?;
 
         let options = options.remove_percentiles();
 
@@ -67,20 +70,48 @@ where
             height,
             height_extra,
             dateindex,
-            weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
+            weekindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
             difficultyepoch: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
             )?,
             monthindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
             )?,
             quarterindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
             )?,
-            yearindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?,
-            // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
+            yearindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
             decadeindex: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
             )?,
         })
     }
@@ -105,9 +136,7 @@ where
         )?;
 
         let height: Option<&EagerVec<Height, T>> = None;
-        self.compute_rest(indexes, starting_indexes, exit, height)?;
-
-        Ok(())
+        self.compute_rest(indexes, starting_indexes, exit, height)
     }
 
     pub fn compute_rest(
@@ -217,6 +246,8 @@ where
             // self.halvingepoch.vecs(),
             self.decadeindex.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/grouped/from_height_strict.rs

@@ -1,11 +1,11 @@
 use std::path::Path;
 
-use brk_core::{DifficultyEpoch, Height};
+use brk_core::{DifficultyEpoch, Height, Result, Version};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
+use brk_vec::{AnyCollectableVec, EagerVec, Format};
 
-use crate::storage::{Indexes, indexes};
+use crate::vecs::{Indexes, indexes};
 
 use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
 
@@ -31,19 +31,17 @@ where
         path: &Path,
         name: &str,
         version: Version,
-        compressed: Compressed,
+        format: Format,
         options: StorableVecGeneatorOptions,
     ) -> color_eyre::Result<Self> {
-        let version = VERSION + version;
-
         let height =
-            EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)?;
+            EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
 
         let height_extra = ComputedVecBuilder::forced_import(
             path,
             name,
-            version,
-            compressed,
+            version + VERSION + Version::ZERO,
+            format,
             options.copy_self_extra(),
         )?;
 
@@ -53,9 +51,13 @@ where
             height,
             height_extra,
             difficultyepoch: ComputedVecBuilder::forced_import(
-                path, name, version, compressed, options,
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
             )?,
-            // halvingepoch: StorableVecGeneator::forced_import(path, name, version, compressed, options)?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
         })
     }
 
@@ -93,6 +95,8 @@ where
             self.difficultyepoch.vecs(),
             // self.halvingepoch.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/grouped/from_txindex.rs

@@ -0,0 +1,629 @@
+use std::path::Path;
+
+use brk_core::{
+    Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
+    Result, Sats, TxIndex, Version, WeekIndex, YearIndex,
+};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{
+    AnyCollectableVec, AnyVec, CollectableVec, EagerVec, Format, StoredIndex, VecIterator,
+};
+
+use crate::vecs::{Indexes, fetched, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromTxindex<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub txindex: Option<Box<EagerVec<TxIndex, T>>>,
+    pub height: ComputedVecBuilder<Height, T>,
+    pub dateindex: ComputedVecBuilder<DateIndex, T>,
+    pub weekindex: ComputedVecBuilder<WeekIndex, T>,
+    pub difficultyepoch: ComputedVecBuilder<DifficultyEpoch, T>,
+    pub monthindex: ComputedVecBuilder<MonthIndex, T>,
+    pub quarterindex: ComputedVecBuilder<QuarterIndex, T>,
+    pub yearindex: ComputedVecBuilder<YearIndex, T>,
+    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
+    pub decadeindex: ComputedVecBuilder<DecadeIndex, T>,
+}
+
+const VERSION: Version = Version::ZERO;
+
+impl<T> ComputedVecsFromTxindex<T>
+where
+    T: ComputedType + Ord + From<f64>,
+    f64: From<T>,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        format: Format,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let txindex = compute_source.then(|| {
+            Box::new(
+                EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
+                    .unwrap(),
+            )
+        });
+
+        let height = ComputedVecBuilder::forced_import(
+            path,
+            name,
+            version + VERSION + Version::ZERO,
+            format,
+            options,
+        )?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            txindex,
+            height,
+            dateindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            weekindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            difficultyepoch: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            monthindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            quarterindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            yearindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
+            decadeindex: ComputedVecBuilder::forced_import(
+                path,
+                name,
+                version + VERSION + Version::ZERO,
+                format,
+                options,
+            )?,
+        })
+    }
+
+    // #[allow(unused)]
+    // pub fn compute_all<F>(
+    //     &mut self,
+    //     indexer: &Indexer,
+    //     indexes: &indexes::Vecs,
+    //     starting_indexes: &Indexes,
+    //     exit: &Exit,
+    //     mut compute: F,
+    // ) -> color_eyre::Result<()>
+    // where
+    //     F: FnMut(
+    //         &mut EagerVec<TxIndex, T>,
+    //         &Indexer,
+    //         &indexes::Vecs,
+    //         &Indexes,
+    //         &Exit,
+    //     ) -> Result<()>,
+    // {
+    //     compute(
+    //         self.txindex.as_mut().unwrap(),
+    //         indexer,
+    //         indexes,
+    //         starting_indexes,
+    //         exit,
+    //     )?;
+
+    //     let txindex: Option<&StoredVec<TxIndex, T>> = None;
+    //     self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
+
+    //     Ok(())
+    // }
+
+    pub fn compute_rest(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        txindex: Option<&impl CollectableVec<TxIndex, T>>,
+    ) -> Result<()> {
+        if let Some(txindex) = txindex {
+            self.height.compute(
+                starting_indexes.height,
+                txindex,
+                &indexer.vecs().height_to_first_txindex,
+                &indexes.height_to_txindex_count,
+                exit,
+            )?;
+        } else {
+            let txindex = self.txindex.as_ref().unwrap().as_ref();
+
+            self.height.compute(
+                starting_indexes.height,
+                txindex,
+                &indexer.vecs().height_to_first_txindex,
+                &indexes.height_to_txindex_count,
+                exit,
+            )?;
+        }
+
+        self.compute_after_height(indexes, starting_indexes, exit)
+    }
+
+    fn compute_after_height(
+        &mut self,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+    ) -> Result<()> {
+        self.dateindex.from_aligned(
+            starting_indexes.dateindex,
+            &self.height,
+            &indexes.dateindex_to_first_height,
+            &indexes.dateindex_to_height_count,
+            exit,
+        )?;
+
+        self.weekindex.from_aligned(
+            starting_indexes.weekindex,
+            &self.dateindex,
+            &indexes.weekindex_to_first_dateindex,
+            &indexes.weekindex_to_dateindex_count,
+            exit,
+        )?;
+
+        self.monthindex.from_aligned(
+            starting_indexes.monthindex,
+            &self.dateindex,
+            &indexes.monthindex_to_first_dateindex,
+            &indexes.monthindex_to_dateindex_count,
+            exit,
+        )?;
+
+        self.quarterindex.from_aligned(
+            starting_indexes.quarterindex,
+            &self.monthindex,
+            &indexes.quarterindex_to_first_monthindex,
+            &indexes.quarterindex_to_monthindex_count,
+            exit,
+        )?;
+
+        self.yearindex.from_aligned(
+            starting_indexes.yearindex,
+            &self.monthindex,
+            &indexes.yearindex_to_first_monthindex,
+            &indexes.yearindex_to_monthindex_count,
+            exit,
+        )?;
+
+        self.decadeindex.from_aligned(
+            starting_indexes.decadeindex,
+            &self.yearindex,
+            &indexes.decadeindex_to_first_yearindex,
+            &indexes.decadeindex_to_yearindex_count,
+            exit,
+        )?;
+
+        self.difficultyepoch.from_aligned(
+            starting_indexes.difficultyepoch,
+            &self.height,
+            &indexes.difficultyepoch_to_first_height,
+            &indexes.difficultyepoch_to_height_count,
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            self.txindex
+                .as_ref()
+                .map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
+            self.height.vecs(),
+            self.dateindex.vecs(),
+            self.weekindex.vecs(),
+            self.difficultyepoch.vecs(),
+            self.monthindex.vecs(),
+            self.quarterindex.vecs(),
+            self.yearindex.vecs(),
+            // self.halvingepoch.vecs(),
+            self.decadeindex.vecs(),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}
+
+impl ComputedVecsFromTxindex<Bitcoin> {
+    pub fn compute_rest_from_sats(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        sats: &ComputedVecsFromTxindex<Sats>,
+        txindex: Option<&impl CollectableVec<TxIndex, Bitcoin>>,
+    ) -> Result<()> {
+        let txindex_version = if let Some(txindex) = txindex {
+            txindex.version()
+        } else {
+            self.txindex.as_ref().unwrap().as_ref().version()
+        };
+
+        self.height
+            .validate_computed_version_or_reset_file(txindex_version)?;
+
+        let starting_index = self.height.starting_index(starting_indexes.height);
+
+        (starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
+            .map(Height::from)
+            .try_for_each(|height| -> Result<()> {
+                if let Some(first) = self.height.first.as_mut() {
+                    first.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_first()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(average) = self.height.average.as_mut() {
+                    average.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_average()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(sum) = self.height.sum.as_mut() {
+                    sum.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_sum()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(max) = self.height.max.as_mut() {
+                    max.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_max()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(_90p) = self.height._90p.as_mut() {
+                    _90p.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_90p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(_75p) = self.height._75p.as_mut() {
+                    _75p.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_75p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(median) = self.height.median.as_mut() {
+                    median.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_median()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(_25p) = self.height._25p.as_mut() {
+                    _25p.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_25p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(_10p) = self.height._10p.as_mut() {
+                    _10p.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_10p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(min) = self.height.min.as_mut() {
+                    min.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_min()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(last) = self.height.last.as_mut() {
+                    last.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_last()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                if let Some(cumulative) = self.height.cumulative.as_mut() {
+                    cumulative.forced_push_at(
+                        height,
+                        Bitcoin::from(
+                            sats.height
+                                .unwrap_cumulative()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        ),
+                        exit,
+                    )?;
+                }
+                Ok(())
+            })?;
+
+        self.height.safe_flush(exit)?;
+
+        self.compute_after_height(indexes, starting_indexes, exit)
+    }
+}
+
+impl ComputedVecsFromTxindex<Dollars> {
+    #[allow(clippy::too_many_arguments)]
+    pub fn compute_rest_from_bitcoin(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        bitcoin: &ComputedVecsFromTxindex<Bitcoin>,
+        txindex: Option<&impl CollectableVec<TxIndex, Dollars>>,
+        fetched: &fetched::Vecs,
+    ) -> Result<()> {
+        let txindex_version = if let Some(txindex) = txindex {
+            txindex.version()
+        } else {
+            self.txindex.as_ref().unwrap().as_ref().version()
+        };
+
+        self.height
+            .validate_computed_version_or_reset_file(txindex_version)?;
+
+        let starting_index = self.height.starting_index(starting_indexes.height);
+
+        let mut close_iter = fetched.chainindexes_to_close.height.into_iter();
+
+        (starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len())
+            .map(Height::from)
+            .try_for_each(|height| -> Result<()> {
+                let price = *close_iter.unwrap_get_inner(height);
+
+                if let Some(first) = self.height.first.as_mut() {
+                    first.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_first()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(average) = self.height.average.as_mut() {
+                    average.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_average()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(sum) = self.height.sum.as_mut() {
+                    sum.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_sum()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(max) = self.height.max.as_mut() {
+                    max.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_max()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(_90p) = self.height._90p.as_mut() {
+                    _90p.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_90p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(_75p) = self.height._75p.as_mut() {
+                    _75p.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_75p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(median) = self.height.median.as_mut() {
+                    median.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_median()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(_25p) = self.height._25p.as_mut() {
+                    _25p.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_25p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(_10p) = self.height._10p.as_mut() {
+                    _10p.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_10p()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(min) = self.height.min.as_mut() {
+                    min.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_min()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(last) = self.height.last.as_mut() {
+                    last.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_last()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                if let Some(cumulative) = self.height.cumulative.as_mut() {
+                    cumulative.forced_push_at(
+                        height,
+                        price
+                            * bitcoin
+                                .height
+                                .unwrap_cumulative()
+                                .into_iter()
+                                .unwrap_get_inner(height),
+                        exit,
+                    )?;
+                }
+                Ok(())
+            })?;
+
+        self.height.safe_flush(exit)?;
+
+        self.compute_after_height(indexes, starting_indexes, exit)
+    }
+}

crates/brk_computer/src/vecs/grouped/mod.rs

@@ -3,15 +3,21 @@ mod from_dateindex;
 mod from_height;
 mod from_height_strict;
 mod from_txindex;
+mod ratio_from_dateindex;
 mod r#type;
+mod value_from_dateindex;
 mod value_from_height;
 mod value_from_txindex;
+mod value_height;
 
 pub use builder::*;
 pub use from_dateindex::*;
 pub use from_height::*;
 pub use from_height_strict::*;
 pub use from_txindex::*;
+pub use ratio_from_dateindex::*;
 use r#type::*;
+pub use value_from_dateindex::*;
 pub use value_from_height::*;
 pub use value_from_txindex::*;
+pub use value_height::*;

crates/brk_computer/src/vecs/grouped/value_from_dateindex.rs

@@ -0,0 +1,176 @@
+use std::path::Path;
+
+use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
+
+use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
+
+use super::StorableVecGeneatorOptions;
+
+#[derive(Clone)]
+pub struct ComputedValueVecsFromDateIndex {
+    pub sats: ComputedVecsFromDateIndex<Sats>,
+    pub bitcoin: ComputedVecsFromDateIndex<Bitcoin>,
+    pub dollars: Option<ComputedVecsFromDateIndex<Dollars>>,
+}
+
+const VERSION: Version = Version::ZERO;
+
+impl ComputedValueVecsFromDateIndex {
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        format: Format,
+        options: StorableVecGeneatorOptions,
+        compute_dollars: bool,
+    ) -> color_eyre::Result<Self> {
+        Ok(Self {
+            sats: ComputedVecsFromDateIndex::forced_import(
+                path,
+                name,
+                compute_source,
+                version + VERSION,
+                format,
+                options,
+            )?,
+            bitcoin: ComputedVecsFromDateIndex::forced_import(
+                path,
+                &format!("{name}_in_btc"),
+                true,
+                version + VERSION,
+                format,
+                options,
+            )?,
+            dollars: compute_dollars.then(|| {
+                ComputedVecsFromDateIndex::forced_import(
+                    path,
+                    &format!("{name}_in_usd"),
+                    true,
+                    version + VERSION,
+                    format,
+                    options,
+                )
+                .unwrap()
+            }),
+        })
+    }
+
+    pub fn compute_all<F>(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        fetched: Option<&fetched::Vecs>,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut EagerVec<DateIndex, Sats>,
+            &Indexer,
+            &indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            self.sats.dateindex.as_mut().unwrap(),
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
+        self.compute_rest(indexer, indexes, fetched, starting_indexes, exit, dateindex)?;
+
+        Ok(())
+    }
+
+    pub fn compute_rest(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        fetched: Option<&fetched::Vecs>,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        dateindex: Option<&impl CollectableVec<DateIndex, Sats>>,
+    ) -> color_eyre::Result<()> {
+        if let Some(dateindex) = dateindex {
+            self.sats
+                .compute_rest(indexes, starting_indexes, exit, Some(dateindex))?;
+
+            self.bitcoin.compute_all(
+                indexer,
+                indexes,
+                starting_indexes,
+                exit,
+                |v, _, _, starting_indexes, exit| {
+                    v.compute_from_sats(starting_indexes.dateindex, dateindex, exit)
+                },
+            )?;
+        } else {
+            let dateindex: Option<&StoredVec<DateIndex, Sats>> = None;
+
+            self.sats
+                .compute_rest(indexes, starting_indexes, exit, dateindex)?;
+
+            self.bitcoin.compute_all(
+                indexer,
+                indexes,
+                starting_indexes,
+                exit,
+                |v, _, _, starting_indexes, exit| {
+                    v.compute_from_sats(
+                        starting_indexes.dateindex,
+                        self.sats.dateindex.as_ref().unwrap(),
+                        exit,
+                    )
+                },
+            )?;
+        }
+
+        let dateindex_to_bitcoin = self.bitcoin.dateindex.as_ref().unwrap();
+        let dateindex_to_close = fetched
+            .as_ref()
+            .unwrap()
+            .timeindexes_to_close
+            .dateindex
+            .as_ref()
+            .unwrap();
+
+        if let Some(dollars) = self.dollars.as_mut() {
+            dollars.compute_all(
+                indexer,
+                indexes,
+                starting_indexes,
+                exit,
+                |v, _, _, starting_indexes, exit| {
+                    v.compute_from_bitcoin(
+                        starting_indexes.dateindex,
+                        dateindex_to_bitcoin,
+                        dateindex_to_close,
+                        exit,
+                    )
+                },
+            )?;
+        }
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            self.sats.vecs(),
+            self.bitcoin.vecs(),
+            self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}

crates/brk_computer/src/vecs/grouped/value_from_height.rs

@@ -1,16 +1,11 @@
 use std::path::Path;
 
-use brk_core::{Bitcoin, Dollars, Height, Sats};
+use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{
-    AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
-};
+use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
 
-use crate::storage::{
-    fetched,
-    vecs::{Indexes, indexes},
-};
+use crate::vecs::{Indexes, fetched, indexes};
 
 use super::{ComputedVecsFromHeight, StorableVecGeneatorOptions};
 
@@ -21,7 +16,7 @@ pub struct ComputedValueVecsFromHeight {
     pub dollars: Option<ComputedVecsFromHeight<Dollars>>,
 }
 
-const VERSION: Version = Version::ONE;
+const VERSION: Version = Version::ZERO;
 
 impl ComputedValueVecsFromHeight {
     pub fn forced_import(
@@ -29,7 +24,7 @@ impl ComputedValueVecsFromHeight {
         name: &str,
         compute_source: bool,
         version: Version,
-        compressed: Compressed,
+        format: Format,
         options: StorableVecGeneatorOptions,
         compute_dollars: bool,
     ) -> color_eyre::Result<Self> {
@@ -38,16 +33,16 @@ impl ComputedValueVecsFromHeight {
                 path,
                 name,
                 compute_source,
-                VERSION + version,
-                compressed,
+                version + VERSION,
+                format,
                 options,
             )?,
             bitcoin: ComputedVecsFromHeight::forced_import(
                 path,
                 &format!("{name}_in_btc"),
                 true,
-                VERSION + version,
-                compressed,
+                version + VERSION,
+                format,
                 options,
             )?,
             dollars: compute_dollars.then(|| {
@@ -55,8 +50,8 @@ impl ComputedValueVecsFromHeight {
                     path,
                     &format!("{name}_in_usd"),
                     true,
-                    VERSION + version,
-                    compressed,
+                    version + VERSION,
+                    format,
                     options,
                 )
                 .unwrap()
@@ -139,8 +134,8 @@ impl ComputedValueVecsFromHeight {
             )?;
         }
 
-        let txindex = self.bitcoin.height.as_ref().unwrap();
-        let price = &fetched.as_ref().unwrap().chainindexes_to_close.height;
+        let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
+        let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
 
         if let Some(dollars) = self.dollars.as_mut() {
             dollars.compute_all(
@@ -149,7 +144,12 @@ impl ComputedValueVecsFromHeight {
                 starting_indexes,
                 exit,
                 |v, _, _, starting_indexes, exit| {
-                    v.compute_from_bitcoin(starting_indexes.height, txindex, price, exit)
+                    v.compute_from_bitcoin(
+                        starting_indexes.height,
+                        height_to_bitcoin,
+                        height_to_close,
+                        exit,
+                    )
                 },
             )?;
         }
@@ -163,6 +163,8 @@ impl ComputedValueVecsFromHeight {
             self.bitcoin.vecs(),
             self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/grouped/value_from_txindex.rs

@@ -1,17 +1,14 @@
 use std::path::Path;
 
-use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex};
+use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{
-    AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Compressed,
-    Computation, ComputedVecFrom3, LazyVecFrom1, StoredIndex, StoredVec, Version,
+    AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Computation,
+    ComputedVecFrom3, Format, LazyVecFrom1, StoredIndex, StoredVec,
 };
 
-use crate::storage::{
-    fetched,
-    vecs::{Indexes, indexes},
-};
+use crate::vecs::{Indexes, fetched, indexes};
 
 use super::{ComputedVecsFromTxindex, StorableVecGeneatorOptions};
 
@@ -36,7 +33,7 @@ pub struct ComputedValueVecsFromTxindex {
     pub dollars: Option<ComputedVecsFromTxindex<Dollars>>,
 }
 
-const VERSION: Version = Version::ONE;
+const VERSION: Version = Version::ZERO;
 
 impl ComputedValueVecsFromTxindex {
     #[allow(clippy::too_many_arguments)]
@@ -47,25 +44,28 @@ impl ComputedValueVecsFromTxindex {
         source: Option<BoxedAnyIterableVec<TxIndex, Sats>>,
         version: Version,
         computation: Computation,
-        compressed: Compressed,
+        format: Format,
         fetched: Option<&fetched::Vecs>,
         options: StorableVecGeneatorOptions,
     ) -> color_eyre::Result<Self> {
         let compute_source = source.is_none();
         let compute_dollars = fetched.is_some();
 
+        let name_in_btc = format!("{name}_in_btc");
+        let name_in_usd = format!("{name}_in_usd");
+
         let sats = ComputedVecsFromTxindex::forced_import(
             path,
             name,
             compute_source,
-            VERSION + version,
-            compressed,
+            version + VERSION,
+            format,
             options,
         )?;
 
         let bitcoin_txindex = LazyVecFrom1::init(
-            "txindex_to_{name}_in_btc",
-            VERSION + version,
+            &name_in_btc,
+            version + VERSION,
             source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s),
             |txindex: TxIndex, iter| {
                 iter.next_at(txindex.unwrap_to_usize()).map(|(_, value)| {
@@ -77,10 +77,10 @@ impl ComputedValueVecsFromTxindex {
 
         let bitcoin = ComputedVecsFromTxindex::forced_import(
             path,
-            &format!("{name}_in_btc"),
+            &name_in_btc,
             false,
-            VERSION + version,
-            compressed,
+            version + VERSION,
+            format,
             options,
         )?;
 
@@ -88,9 +88,9 @@ impl ComputedValueVecsFromTxindex {
             ComputedVecFrom3::forced_import_or_init_from_3(
                 computation,
                 path,
-                "txindex_to_{name}_in_usd",
-                VERSION + version,
-                compressed,
+                &name_in_usd,
+                version + VERSION,
+                format,
                 bitcoin_txindex.boxed_clone(),
                 indexes.txindex_to_height.boxed_clone(),
                 fetched.chainindexes_to_close.height.boxed_clone(),
@@ -123,10 +123,10 @@ impl ComputedValueVecsFromTxindex {
             dollars: compute_dollars.then(|| {
                 ComputedVecsFromTxindex::forced_import(
                     path,
-                    &format!("{name}_in_usd"),
+                    &name_in_usd,
                     false,
-                    VERSION + version,
-                    compressed,
+                    version + VERSION,
+                    format,
                     options,
                 )
                 .unwrap()
@@ -180,6 +180,7 @@ impl ComputedValueVecsFromTxindex {
         starting_indexes: &Indexes,
         exit: &Exit,
         txindex: Option<&impl CollectableVec<TxIndex, Sats>>,
+        fetched: Option<&fetched::Vecs>,
     ) -> color_eyre::Result<()> {
         if let Some(txindex) = txindex {
             self.sats
@@ -190,25 +191,32 @@ impl ComputedValueVecsFromTxindex {
                 .compute_rest(indexer, indexes, starting_indexes, exit, txindex)?;
         }
 
-        self.bitcoin.compute_rest(
+        self.bitcoin.compute_rest_from_sats(
             indexer,
             indexes,
             starting_indexes,
             exit,
+            &self.sats,
             Some(&self.bitcoin_txindex),
         )?;
 
         if let Some(dollars) = self.dollars.as_mut() {
             let dollars_txindex = self.dollars_txindex.as_mut().unwrap();
 
-            dollars_txindex.compute_if_necessary(starting_indexes.txindex, exit)?;
+            dollars_txindex.compute_if_necessary(
+                starting_indexes.txindex,
+                &indexer.vecs().txindex_to_txid,
+                exit,
+            )?;
 
-            dollars.compute_rest(
+            dollars.compute_rest_from_bitcoin(
                 indexer,
                 indexes,
                 starting_indexes,
                 exit,
+                &self.bitcoin,
                 Some(dollars_txindex),
+                fetched.as_ref().unwrap(),
             )?;
         }
 
@@ -221,6 +229,8 @@ impl ComputedValueVecsFromTxindex {
             self.bitcoin.vecs(),
             self.dollars.as_ref().map_or(vec![], |v| v.vecs()),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/grouped/value_height.rs

@@ -0,0 +1,126 @@
+use std::path::Path;
+
+use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
+
+use crate::vecs::{Indexes, fetched, indexes};
+
+#[derive(Clone)]
+pub struct ComputedHeightValueVecs {
+    pub sats: Option<EagerVec<Height, Sats>>,
+    pub bitcoin: EagerVec<Height, Bitcoin>,
+    pub dollars: Option<EagerVec<Height, Dollars>>,
+}
+
+const VERSION: Version = Version::ZERO;
+
+impl ComputedHeightValueVecs {
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        format: Format,
+        compute_dollars: bool,
+    ) -> color_eyre::Result<Self> {
+        Ok(Self {
+            sats: compute_source.then(|| {
+                EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
+                    .unwrap()
+            }),
+            bitcoin: EagerVec::forced_import(
+                path,
+                &format!("{name}_in_btc"),
+                version + VERSION + Version::ZERO,
+                format,
+            )?,
+            dollars: compute_dollars.then(|| {
+                EagerVec::forced_import(
+                    path,
+                    &format!("{name}_in_usd"),
+                    version + VERSION + Version::ZERO,
+                    format,
+                )
+                .unwrap()
+            }),
+        })
+    }
+
+    pub fn compute_all<F>(
+        &mut self,
+        indexer: &Indexer,
+        indexes: &indexes::Vecs,
+        fetched: Option<&fetched::Vecs>,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut EagerVec<Height, Sats>,
+            &Indexer,
+            &indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            self.sats.as_mut().unwrap(),
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        let height: Option<&StoredVec<Height, Sats>> = None;
+        self.compute_rest(fetched, starting_indexes, exit, height)?;
+
+        Ok(())
+    }
+
+    pub fn compute_rest(
+        &mut self,
+        fetched: Option<&fetched::Vecs>,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        height: Option<&impl CollectableVec<Height, Sats>>,
+    ) -> color_eyre::Result<()> {
+        if let Some(height) = height {
+            self.bitcoin
+                .compute_from_sats(starting_indexes.height, height, exit)?;
+        } else {
+            self.bitcoin.compute_from_sats(
+                starting_indexes.height,
+                self.sats.as_ref().unwrap(),
+                exit,
+            )?;
+        }
+
+        let height_to_bitcoin = &self.bitcoin;
+        let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
+
+        if let Some(dollars) = self.dollars.as_mut() {
+            dollars.compute_from_bitcoin(
+                starting_indexes.height,
+                height_to_bitcoin,
+                height_to_close,
+                exit,
+            )?;
+        }
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            vec![&self.bitcoin as &dyn AnyCollectableVec],
+            self.sats.as_ref().map_or(vec![], |v| vec![v]),
+            self.dollars.as_ref().map_or(vec![], |v| vec![v]),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}

crates/brk_computer/src/vecs/mining.rs

@@ -1,28 +1,31 @@
 use std::{fs, path::Path};
 
-use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
+use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyCollectableVec, Compressed, Computation, VecIterator, Version};
+use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
 
 use super::{
     Indexes,
-    grouped::{ComputedVecsFromDateindex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
+    grouped::{ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions},
     indexes,
 };
 
+const VERSION: Version = Version::ZERO;
+
 #[derive(Clone)]
 pub struct Vecs {
     pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
-    pub indexes_to_difficultyepoch: ComputedVecsFromDateindex<DifficultyEpoch>,
-    pub indexes_to_halvingepoch: ComputedVecsFromDateindex<HalvingEpoch>,
+    pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
+    pub indexes_to_halvingepoch: ComputedVecsFromDateIndex<HalvingEpoch>,
 }
 
 impl Vecs {
     pub fn forced_import(
         path: &Path,
+        version: Version,
         _computation: Computation,
-        compressed: Compressed,
+        format: Format,
     ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
@@ -31,22 +34,24 @@ impl Vecs {
                 path,
                 "difficulty",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            indexes_to_difficultyepoch: ComputedVecsFromDateindex::forced_import(
+            indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "difficultyepoch",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            indexes_to_halvingepoch: ComputedVecsFromDateindex::forced_import(
+            indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
                 path,
                 "halvingepoch",
-                Version::ZERO,
-                compressed,
+                true,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
         })
@@ -60,7 +65,7 @@ impl Vecs {
         exit: &Exit,
     ) -> color_eyre::Result<()> {
         let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
-        self.indexes_to_difficultyepoch.compute(
+        self.indexes_to_difficultyepoch.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -84,7 +89,7 @@ impl Vecs {
         )?;
 
         let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
-        self.indexes_to_halvingepoch.compute(
+        self.indexes_to_halvingepoch.compute_all(
             indexer,
             indexes,
             starting_indexes,
@@ -123,6 +128,8 @@ impl Vecs {
             self.indexes_to_difficultyepoch.vecs(),
             self.indexes_to_halvingepoch.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_computer/src/vecs/mod.rs

@@ -0,0 +1,193 @@
+use std::{fs, path::Path};
+
+use brk_core::Version;
+use brk_exit::Exit;
+use brk_fetcher::Fetcher;
+use brk_indexer::Indexer;
+use brk_vec::{AnyCollectableVec, Computation, Format};
+
+pub mod blocks;
+pub mod constants;
+pub mod fetched;
+pub mod grouped;
+pub mod indexes;
+pub mod market;
+pub mod mining;
+pub mod stateful;
+pub mod transactions;
+
+pub use indexes::Indexes;
+use log::info;
+
+const VERSION: Version = Version::ONE;
+
+#[derive(Clone)]
+pub struct Vecs {
+    pub indexes: indexes::Vecs,
+    pub constants: constants::Vecs,
+    pub blocks: blocks::Vecs,
+    pub mining: mining::Vecs,
+    pub market: market::Vecs,
+    pub transactions: transactions::Vecs,
+    pub stateful: stateful::Vecs,
+    pub fetched: Option<fetched::Vecs>,
+}
+
+impl Vecs {
+    pub fn import(
+        path: &Path,
+        version: Version,
+        indexer: &Indexer,
+        fetch: bool,
+        computation: Computation,
+        format: Format,
+    ) -> color_eyre::Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let indexes = indexes::Vecs::forced_import(
+            path,
+            version + VERSION + Version::ZERO,
+            indexer,
+            computation,
+            format,
+        )?;
+
+        let fetched = fetch.then(|| {
+            fetched::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+            )
+            .unwrap()
+        });
+
+        Ok(Self {
+            blocks: blocks::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+            )?,
+            mining: mining::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+            )?,
+            constants: constants::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+            )?,
+            market: market::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+            )?,
+            stateful: stateful::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                computation,
+                format,
+                fetched.as_ref(),
+            )?,
+            transactions: transactions::Vecs::forced_import(
+                path,
+                version + VERSION + Version::ZERO,
+                indexer,
+                &indexes,
+                computation,
+                format,
+                fetched.as_ref(),
+            )?,
+            indexes,
+            fetched,
+        })
+    }
+
+    pub fn compute(
+        &mut self,
+        indexer: &Indexer,
+        starting_indexes: brk_indexer::Indexes,
+        fetcher: Option<&mut Fetcher>,
+        exit: &Exit,
+    ) -> color_eyre::Result<()> {
+        let starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
+
+        info!("Computing constants...");
+        self.constants
+            .compute(indexer, &self.indexes, &starting_indexes, exit)?;
+
+        info!("Computing blocks...");
+        self.blocks
+            .compute(indexer, &self.indexes, &starting_indexes, exit)?;
+
+        info!("Computing mining...");
+        self.mining
+            .compute(indexer, &self.indexes, &starting_indexes, exit)?;
+
+        if let Some(fetched) = self.fetched.as_mut() {
+            info!("Computing fetched...");
+            fetched.compute(
+                indexer,
+                &self.indexes,
+                &starting_indexes,
+                fetcher.unwrap(),
+                exit,
+            )?;
+        }
+
+        info!("Computing transactions...");
+        self.transactions.compute(
+            indexer,
+            &self.indexes,
+            &starting_indexes,
+            self.fetched.as_ref(),
+            exit,
+        )?;
+
+        if let Some(fetched) = self.fetched.as_ref() {
+            info!("Computing market...");
+            self.market.compute(
+                indexer,
+                &self.indexes,
+                fetched,
+                &mut self.transactions,
+                &starting_indexes,
+                exit,
+            )?;
+        }
+
+        info!("Computing stateful...");
+        self.stateful.compute(
+            indexer,
+            &self.indexes,
+            &self.transactions,
+            self.fetched.as_ref(),
+            &self.market,
+            starting_indexes,
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
+        [
+            self.constants.vecs(),
+            self.indexes.vecs(),
+            self.blocks.vecs(),
+            self.mining.vecs(),
+            self.market.vecs(),
+            self.transactions.vecs(),
+            self.stateful.vecs(),
+            self.fetched.as_ref().map_or(vec![], |v| v.vecs()),
+        ]
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+    }
+}

crates/brk_computer/src/vecs/stateful/outputs.rs

@@ -0,0 +1,258 @@
+use std::{collections::BTreeMap, ops::ControlFlow};
+
+use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp};
+use brk_exit::Exit;
+use brk_state::{BlockState, OutputFilter, Outputs, Transacted};
+use brk_vec::StoredIndex;
+use rayon::prelude::*;
+
+use crate::vecs::Indexes;
+
+use super::cohort;
+
+pub trait OutputCohorts {
+    fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp);
+    fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]);
+    fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>);
+    fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()>;
+}
+
+impl OutputCohorts for Outputs<(OutputFilter, cohort::Vecs)> {
+    fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) {
+        if chain_state.is_empty() {
+            return;
+        }
+
+        let prev_timestamp = chain_state.last().unwrap().timestamp;
+
+        self.by_date_range
+            .as_mut_vec()
+            .into_par_iter()
+            .for_each(|(filter, v)| {
+                let state = &mut v.state;
+
+                let _ = chain_state
+                    .iter()
+                    .try_for_each(|block_state| -> ControlFlow<()> {
+                        let prev_days_old = block_state
+                            .timestamp
+                            .difference_in_days_between(prev_timestamp);
+                        let days_old = block_state.timestamp.difference_in_days_between(timestamp);
+
+                        if prev_days_old == days_old {
+                            return ControlFlow::Continue(());
+                        }
+
+                        let is = filter.contains(days_old);
+                        let was = filter.contains(prev_days_old);
+
+                        if is && !was {
+                            state.increment(&block_state.supply, block_state.price);
+                        } else if was && !is {
+                            state.decrement(&block_state.supply, block_state.price);
+                        }
+
+                        ControlFlow::Continue(())
+                    });
+            });
+    }
+
+    fn send(&mut self, height_to_sent: BTreeMap<Height, Transacted>, chain_state: &[BlockState]) {
+        let mut time_based_vecs = self
+            .by_date_range
+            .as_mut_vec()
+            .into_iter()
+            .chain(self.by_epoch.as_mut_vec())
+            .collect::<Vec<_>>();
+
+        let last_timestamp = chain_state.last().unwrap().timestamp;
+        let current_price = chain_state.last().unwrap().price;
+
+        // dbg!(&height_to_sent);
+
+        height_to_sent.into_iter().for_each(|(height, sent)| {
+            let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
+            let prev_price = block_state.price;
+
+            let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize();
+
+            let days_old = block_state
+                .timestamp
+                .difference_in_days_between(last_timestamp);
+
+            let days_old_foat = block_state
+                .timestamp
+                .difference_in_days_between_float(last_timestamp);
+
+            let older_than_hour =
+                jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap())
+                    .as_second()
+                    >= 60 * 60;
+
+            time_based_vecs
+                .iter_mut()
+                .filter(|(filter, _)| match filter {
+                    OutputFilter::From(from) => *from <= days_old,
+                    OutputFilter::To(to) => *to > days_old,
+                    OutputFilter::Range(range) => range.contains(&days_old),
+                    OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height),
+                    _ => unreachable!(),
+                })
+                .for_each(|(_, vecs)| {
+                    vecs.state.send(
+                        &sent.spendable_supply,
+                        current_price,
+                        prev_price,
+                        blocks_old,
+                        days_old_foat,
+                        older_than_hour,
+                    );
+                });
+
+            sent.by_type.spendable.as_typed_vec().into_iter().for_each(
+                |(output_type, supply_state)| {
+                    self.by_type.get_mut(output_type).1.state.send(
+                        supply_state,
+                        current_price,
+                        prev_price,
+                        blocks_old,
+                        days_old_foat,
+                        older_than_hour,
+                    )
+                },
+            );
+
+            sent.by_size_group
+                .into_iter()
+                .for_each(|(group, supply_state)| {
+                    self.by_size_range.get_mut(group).1.state.send(
+                        &supply_state,
+                        current_price,
+                        prev_price,
+                        blocks_old,
+                        days_old_foat,
+                        older_than_hour,
+                    );
+                });
+        });
+    }
+
+    fn receive(&mut self, received: Transacted, height: Height, price: Option<Dollars>) {
+        let supply_state = received.spendable_supply;
+
+        [
+            &mut self.by_date_range.start_to_1d.1,
+            &mut self.by_epoch.mut_vec_from_height(height).1,
+        ]
+        .into_iter()
+        .for_each(|v| {
+            v.state.receive(&supply_state, price);
+        });
+
+        self.by_type
+            .as_mut_vec()
+            .into_iter()
+            .for_each(|(filter, vecs)| {
+                let output_type = match filter {
+                    OutputFilter::Type(output_type) => *output_type,
+                    _ => unreachable!(),
+                };
+                vecs.state.receive(received.by_type.get(output_type), price)
+            });
+
+        received
+            .by_size_group
+            .into_iter()
+            .for_each(|(group, supply_state)| {
+                self.by_size_range
+                    .get_mut(group)
+                    .1
+                    .state
+                    .receive(&supply_state, price);
+            });
+    }
+
+    fn compute_overlaping_vecs(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
+        let by_date_range = self.by_date_range.as_vec();
+        let by_size_range = self.by_size_range.as_vec();
+
+        [
+            vec![(&mut self.all.1, self.by_epoch.vecs().to_vec())],
+            self.by_from_date
+                .as_mut_vec()
+                .into_iter()
+                .map(|(filter, vecs)| {
+                    (
+                        vecs,
+                        by_date_range
+                            .into_iter()
+                            .filter(|(other, _)| filter.includes(other))
+                            .map(|(_, v)| v)
+                            .collect::<Vec<_>>(),
+                    )
+                })
+                .collect::<Vec<_>>(),
+            self.by_up_to_date
+                .as_mut_vec()
+                .into_iter()
+                .map(|(filter, vecs)| {
+                    (
+                        vecs,
+                        by_date_range
+                            .into_iter()
+                            .filter(|(other, _)| filter.includes(other))
+                            .map(|(_, v)| v)
+                            .collect::<Vec<_>>(),
+                    )
+                })
+                .collect::<Vec<_>>(),
+            self.by_term
+                .as_mut_vec()
+                .into_iter()
+                .map(|(filter, vecs)| {
+                    (
+                        vecs,
+                        by_date_range
+                            .into_iter()
+                            .filter(|(other, _)| filter.includes(other))
+                            .map(|(_, v)| v)
+                            .collect::<Vec<_>>(),
+                    )
+                })
+                .collect::<Vec<_>>(),
+            self.by_from_size
+                .as_mut_vec()
+                .into_iter()
+                .map(|(filter, vecs)| {
+                    (
+                        vecs,
+                        by_size_range
+                            .into_iter()
+                            .filter(|(other, _)| filter.includes(other))
+                            .map(|(_, v)| v)
+                            .collect::<Vec<_>>(),
+                    )
+                })
+                .collect::<Vec<_>>(),
+            self.by_up_to_size
+                .as_mut_vec()
+                .into_iter()
+                .map(|(filter, vecs)| {
+                    (
+                        vecs,
+                        by_size_range
+                            .into_iter()
+                            .filter(|(other, _)| filter.includes(other))
+                            .map(|(_, v)| v)
+                            .collect::<Vec<_>>(),
+                    )
+                })
+                .collect::<Vec<_>>(),
+        ]
+        .into_par_iter()
+        .flatten()
+        .try_for_each(|(vecs, stateful)| {
+            vecs.compute_from_stateful(starting_indexes, &stateful, exit)
+        })
+    }
+}

crates/brk_computer/src/vecs/transactions.rs

@@ -1,15 +1,14 @@
 use std::{fs, path::Path};
 
 use brk_core::{
-    CheckedSub, Feerate, Height, InputIndex, OutputIndex, Sats, StoredU32, StoredUsize, TxIndex,
-    TxVersion, Weight,
+    CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
+    StoredUsize, TxIndex, TxVersion, Version, Weight,
 };
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_parser::bitcoin;
 use brk_vec::{
-    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Compressed, Computation,
-    ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, Version,
+    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
+    ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
 };
 
 use super::{
@@ -21,6 +20,8 @@ use super::{
     indexes,
 };
 
+const VERSION: Version = Version::ZERO;
+
 #[derive(Clone)]
 pub struct Vecs {
     // pub txindex_to_is_v1: LazyVec<Txindex, bool>,
@@ -64,6 +65,7 @@ pub struct Vecs {
     pub indexes_to_p2wpkh_count: ComputedVecsFromHeight<StoredUsize>,
     pub indexes_to_p2wsh_count: ComputedVecsFromHeight<StoredUsize>,
     pub indexes_to_subsidy: ComputedValueVecsFromHeight,
+    pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight,
     pub indexes_to_tx_count: ComputedVecsFromHeight<StoredUsize>,
     pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredUsize>,
     pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredUsize>,
@@ -73,27 +75,25 @@ pub struct Vecs {
     pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredUsize>,
     pub inputindex_to_value:
         ComputedVecFrom2<InputIndex, Sats, InputIndex, OutputIndex, OutputIndex, Sats>,
-    pub txindex_to_input_count:
-        ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, InputIndex, InputIndex, OutputIndex>,
     pub indexes_to_input_count: ComputedVecsFromTxindex<StoredUsize>,
     pub txindex_to_is_coinbase: ComputedVecFrom2<TxIndex, bool, TxIndex, Height, Height, TxIndex>,
-    pub txindex_to_output_count:
-        ComputedVecFrom2<TxIndex, StoredUsize, TxIndex, OutputIndex, OutputIndex, Sats>,
     pub indexes_to_output_count: ComputedVecsFromTxindex<StoredUsize>,
     pub txindex_to_vsize: ComputedVecFrom1<TxIndex, StoredUsize, TxIndex, Weight>,
     pub txindex_to_weight:
         ComputedVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
     pub txindex_to_fee: ComputedVecFrom2<TxIndex, Sats, TxIndex, Sats, TxIndex, Sats>,
     pub txindex_to_feerate: ComputedVecFrom2<TxIndex, Feerate, TxIndex, Sats, TxIndex, StoredUsize>,
+    pub indexes_to_exact_utxo_count: ComputedVecsFromHeight<StoredUsize>,
 }
 
 impl Vecs {
     pub fn forced_import(
         path: &Path,
+        version: Version,
         indexer: &Indexer,
         indexes: &indexes::Vecs,
         computation: Computation,
-        compressed: Compressed,
+        format: Format,
         fetched: Option<&fetched::Vecs>,
     ) -> color_eyre::Result<Self> {
         let compute_dollars = fetched.is_some();
@@ -103,9 +103,9 @@ impl Vecs {
         let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
             computation,
             path,
-            "inputindex_to_value",
-            Version::ZERO,
-            compressed,
+            "value",
+            version + VERSION + Version::ZERO,
+            format,
             indexer.vecs().inputindex_to_outputindex.boxed_clone(),
             indexer.vecs().outputindex_to_value.boxed_clone(),
             |index: InputIndex, inputindex_to_outputindex_iter, outputindex_to_value_iter| {
@@ -130,9 +130,9 @@ impl Vecs {
         let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
             computation,
             path,
-            "txindex_to_weight",
-            Version::ZERO,
-            compressed,
+            "weight",
+            version + VERSION + Version::ZERO,
+            format,
             indexer.vecs().txindex_to_base_size.boxed_clone(),
             indexer.vecs().txindex_to_total_size.boxed_clone(),
             |index: TxIndex, txindex_to_base_size_iter, txindex_to_total_size_iter| {
@@ -158,9 +158,9 @@ impl Vecs {
         let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
             computation,
             path,
-            "txindex_to_vsize",
-            Version::ZERO,
-            compressed,
+            "vsize",
+            version + VERSION + Version::ZERO,
+            format,
             txindex_to_weight.boxed_clone(),
             |index: TxIndex, iter| {
                 let index = index.unwrap_to_usize();
@@ -175,9 +175,9 @@ impl Vecs {
         let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
             computation,
             path,
-            "txindex_to_is_coinbase",
-            Version::ZERO,
-            compressed,
+            "is_coinbase",
+            version + VERSION + Version::ZERO,
+            format,
             indexes.txindex_to_height.boxed_clone(),
             indexer.vecs().height_to_first_txindex.boxed_clone(),
             |index: TxIndex, txindex_to_height_iter, height_to_first_txindex_iter| {
@@ -196,37 +196,14 @@ impl Vecs {
             },
         )?;
 
-        let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
-            computation,
-            path,
-            "txindex_to_input_count",
-            Version::ZERO,
-            compressed,
-            indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
-            indexer.vecs().inputindex_to_outputindex.boxed_clone(),
-            |index: TxIndex, txindex_to_first_inputindex_iter, inputindex_to_outputindex_iter| {
-                let txindex = index.unwrap_to_usize();
-                txindex_to_first_inputindex_iter
-                    .next_at(txindex)
-                    .map(|(_, start)| {
-                        let start = usize::from(start.into_inner());
-                        let end = txindex_to_first_inputindex_iter
-                            .next_at(txindex + 1)
-                            .map(|(_, v)| usize::from(v.into_inner()))
-                            .unwrap_or_else(|| inputindex_to_outputindex_iter.len());
-                        StoredUsize::from((start..end).count())
-                    })
-            },
-        )?;
-
         let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
             computation,
             path,
-            "txindex_to_input_value",
-            Version::ZERO,
-            compressed,
+            "input_value",
+            version + VERSION + Version::ZERO,
+            format,
             indexer.vecs().txindex_to_first_inputindex.boxed_clone(),
-            txindex_to_input_count.boxed_clone(),
+            indexes.txindex_to_input_count.boxed_clone(),
             inputindex_to_value.boxed_clone(),
             |index: TxIndex,
              txindex_to_first_inputindex_iter,
@@ -260,45 +237,22 @@ impl Vecs {
         //         path,
         //         "input_value",
         //         true,
-        //         Version::ZERO,
-        //         compressed,
+        //         version + VERSION + Version::ZERO,
+        //         format,
         //         StorableVecGeneatorOptions::default()
         //             .add_average()
         //             .add_sum()
-        //             .add_total(),
+        //             .add_cumulative(),
         //     )?;
 
-        let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
-            computation,
-            path,
-            "txindex_to_output_count",
-            Version::ZERO,
-            compressed,
-            indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
-            indexer.vecs().outputindex_to_value.boxed_clone(),
-            |index: TxIndex, txindex_to_first_outputindex_iter, outputindex_to_value_iter| {
-                let txindex = index.unwrap_to_usize();
-                txindex_to_first_outputindex_iter
-                    .next_at(txindex)
-                    .map(|(_, start)| {
-                        let start = usize::from(start.into_inner());
-                        let end = txindex_to_first_outputindex_iter
-                            .next_at(txindex + 1)
-                            .map(|(_, v)| usize::from(v.into_inner()))
-                            .unwrap_or_else(|| outputindex_to_value_iter.len());
-                        StoredUsize::from((start..end).count())
-                    })
-            },
-        )?;
-
         let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
             computation,
             path,
-            "txindex_to_output_value",
-            Version::ZERO,
-            compressed,
+            "output_value",
+            version + VERSION + Version::ZERO,
+            format,
             indexer.vecs().txindex_to_first_outputindex.boxed_clone(),
-            txindex_to_output_count.boxed_clone(),
+            indexes.txindex_to_output_count.boxed_clone(),
             indexer.vecs().outputindex_to_value.boxed_clone(),
             |index: TxIndex,
              txindex_to_first_outputindex_iter,
@@ -332,20 +286,20 @@ impl Vecs {
         //         path,
         //         "output_value",
         //         true,
-        //         Version::ZERO,
-        //         compressed,
+        //         version + VERSION + Version::ZERO,
+        //         format,
         //         StorableVecGeneatorOptions::default()
         //             .add_average()
         //             .add_sum()
-        //             .add_total(),
+        //             .add_cumulative(),
         //     )?;
 
         let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
             computation,
             path,
-            "txindex_to_fee",
-            Version::ZERO,
-            compressed,
+            "fee",
+            version + VERSION + Version::ZERO,
+            format,
             txindex_to_input_value.boxed_clone(),
             txindex_to_output_value.boxed_clone(),
             |txindex: TxIndex, input_iter, output_iter| {
@@ -366,9 +320,9 @@ impl Vecs {
         let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
             computation,
             path,
-            "txindex_to_feerate",
-            Version::ZERO,
-            compressed,
+            "feerate",
+            version + VERSION + Version::ZERO,
+            format,
             txindex_to_fee.boxed_clone(),
             txindex_to_vsize.boxed_clone(),
             |txindex: TxIndex, fee_iter, vsize_iter| {
@@ -388,77 +342,83 @@ impl Vecs {
                 path,
                 "tx_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
                 path,
                 "input_count",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
                 path,
                 "output_count",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
                 path,
                 "tx_v1",
                 true,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
                 path,
                 "tx_v2",
                 true,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
                 path,
                 "tx_v3",
                 true,
-                Version::ZERO,
-                compressed,
-                StorableVecGeneatorOptions::default().add_sum().add_total(),
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
             )?,
             indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
                 path,
                 "fee",
                 indexes,
                 Some(txindex_to_fee.boxed_clone()),
-                Version::ZERO,
+                version + VERSION + Version::ZERO,
                 computation,
-                compressed,
+                format,
                 fetched,
                 StorableVecGeneatorOptions::default()
                     .add_sum()
-                    .add_total()
+                    .add_cumulative()
                     .add_percentiles()
                     .add_minmax()
                     .add_average(),
@@ -467,8 +427,8 @@ impl Vecs {
                 path,
                 "feerate",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_minmax()
@@ -478,8 +438,8 @@ impl Vecs {
                 path,
                 "tx_vsize",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_minmax()
@@ -489,8 +449,8 @@ impl Vecs {
                 path,
                 "tx_weight",
                 false,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_minmax()
@@ -500,12 +460,12 @@ impl Vecs {
                 path,
                 "subsidy",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_sum()
-                    .add_total()
+                    .add_cumulative()
                     .add_minmax()
                     .add_average(),
                 compute_dollars,
@@ -514,171 +474,190 @@ impl Vecs {
                 path,
                 "coinbase",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_sum()
-                    .add_total()
+                    .add_cumulative()
                     .add_percentiles()
                     .add_minmax()
                     .add_average(),
                 compute_dollars,
             )?,
+            indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
+                path,
+                "unclaimed_rewards",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_cumulative(),
+                compute_dollars,
+            )?,
             indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2a_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2ms_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2pk33_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2pk65_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2pkh_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2sh_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2tr_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2wpkh_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "p2wsh_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "opreturn_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "unknownoutput_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
             )?,
             indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
                 path,
                 "emptyoutput_count",
                 true,
-                Version::ZERO,
-                compressed,
+                version + VERSION + Version::ZERO,
+                format,
                 StorableVecGeneatorOptions::default()
                     .add_average()
                     .add_minmax()
                     .add_percentiles()
                     .add_sum()
-                    .add_total(),
+                    .add_cumulative(),
+            )?,
+            indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
+                path,
+                "exact_utxo_count",
+                true,
+                version + VERSION + Version::ZERO,
+                format,
+                StorableVecGeneatorOptions::default().add_last(),
             )?,
             txindex_to_is_coinbase,
             inputindex_to_value,
@@ -690,8 +669,6 @@ impl Vecs {
             txindex_to_feerate,
             txindex_to_vsize,
             txindex_to_weight,
-            txindex_to_input_count,
-            txindex_to_output_count,
         })
     }
 
@@ -723,7 +700,7 @@ impl Vecs {
             indexes,
             starting_indexes,
             exit,
-            Some(&self.txindex_to_input_count),
+            Some(&indexes.txindex_to_input_count),
         )?;
 
         self.indexes_to_output_count.compute_rest(
@@ -731,7 +708,7 @@ impl Vecs {
             indexes,
             starting_indexes,
             exit,
-            Some(&self.txindex_to_output_count),
+            Some(&indexes.txindex_to_output_count),
         )?;
 
         let compute_indexes_to_tx_vany =
@@ -760,20 +737,35 @@ impl Vecs {
         compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
         compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
 
-        self.txindex_to_is_coinbase
-            .compute_if_necessary(starting_indexes.txindex, exit)?;
+        self.txindex_to_is_coinbase.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
 
-        self.txindex_to_weight
-            .compute_if_necessary(starting_indexes.txindex, exit)?;
+        self.txindex_to_weight.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
 
-        self.txindex_to_vsize
-            .compute_if_necessary(starting_indexes.txindex, exit)?;
+        self.txindex_to_vsize.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
 
-        self.inputindex_to_value
-            .compute_if_necessary(starting_indexes.inputindex, exit)?;
+        self.inputindex_to_value.compute_if_necessary(
+            starting_indexes.inputindex,
+            &indexer.vecs().inputindex_to_outputindex,
+            exit,
+        )?;
 
-        self.txindex_to_output_value
-            .compute_if_necessary(starting_indexes.txindex, exit)?;
+        self.txindex_to_output_value.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
 
         // self.indexes_to_output_value.compute_all(
         //     indexer,
@@ -791,8 +783,11 @@ impl Vecs {
         //     },
         // )?;
 
-        self.txindex_to_input_value
-            .compute_if_necessary(starting_indexes.txindex, exit)?;
+        self.txindex_to_input_value.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
 
         // self.indexes_to_input_value.compute_all(
         //     indexer,
@@ -810,12 +805,25 @@ impl Vecs {
         //     },
         // )?;
 
+        self.txindex_to_fee.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
+
+        self.txindex_to_feerate.compute_if_necessary(
+            starting_indexes.txindex,
+            &indexer.vecs().txindex_to_txid,
+            exit,
+        )?;
+
         self.indexes_to_fee.compute_rest(
             indexer,
             indexes,
             starting_indexes,
             exit,
             Some(&self.txindex_to_fee),
+            fetched,
         )?;
 
         self.indexes_to_feerate.compute_rest(
@@ -851,7 +859,7 @@ impl Vecs {
             |vec, indexer, _, starting_indexes, exit| {
                 let mut txindex_to_first_outputindex_iter =
                     indexer.vecs().txindex_to_first_outputindex.iter();
-                let mut txindex_to_output_count_iter = self.txindex_to_output_count.iter();
+                let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
                 let mut outputindex_to_value_iter = indexer.vecs().outputindex_to_value.iter();
                 vec.compute_transform(
                     starting_indexes.height,
@@ -887,9 +895,30 @@ impl Vecs {
                 vec.compute_transform(
                     starting_indexes.height,
                     self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
-                    |(height, subsidy, ..)| {
+                    |(height, coinbase, ..)| {
                         let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
-                        (height, subsidy.checked_sub(fees).unwrap())
+                        (height, coinbase.checked_sub(fees).unwrap())
+                    },
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_unclaimed_rewards.compute_all(
+            indexer,
+            indexes,
+            fetched,
+            starting_indexes,
+            exit,
+            |vec, _, _, starting_indexes, exit| {
+                vec.compute_transform(
+                    starting_indexes.height,
+                    self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
+                    |(height, subsidy, ..)| {
+                        let halving = HalvingEpoch::from(height);
+                        let expected =
+                            Sats::FIFTY_BTC / 2_usize.pow(halving.unwrap_to_usize() as u32);
+                        (height, expected.checked_sub(subsidy).unwrap())
                     },
                     exit,
                 )
@@ -1076,6 +1105,58 @@ impl Vecs {
             },
         )?;
 
+        self.indexes_to_exact_utxo_count.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                let mut input_count_iter = self
+                    .indexes_to_input_count
+                    .height
+                    .unwrap_cumulative()
+                    .into_iter();
+                let mut opreturn_count_iter = self
+                    .indexes_to_opreturn_count
+                    .height_extra
+                    .unwrap_cumulative()
+                    .into_iter();
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.indexes_to_output_count.height.unwrap_cumulative(),
+                    |(h, output_count, ..)| {
+                        let input_count = input_count_iter.unwrap_get_inner(h);
+                        let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
+                        let block_count = usize::from(h + 1_usize);
+                        // -1 > genesis output is unspendable
+                        let mut utxo_count =
+                            *output_count - (*input_count - block_count) - *opreturn_count - 1;
+
+                        // txid dup: e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468
+                        // Block 91_722 https://mempool.space/block/00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e
+                        // Block 91_880 https://mempool.space/block/00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721
+                        //
+                        // txid dup: d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599
+                        // Block 91_812 https://mempool.space/block/00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f
+                        // Block 91_842 https://mempool.space/block/00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec
+                        //
+                        // Warning: Dups invalidate the previous coinbase according to
+                        // https://chainquery.com/bitcoin-cli/gettxoutsetinfo
+
+                        if h >= Height::new(91_842) {
+                            utxo_count -= 1;
+                        }
+                        if h >= Height::new(91_880) {
+                            utxo_count -= 1;
+                        }
+
+                        (h, StoredUsize::from(utxo_count))
+                    },
+                    exit,
+                )
+            },
+        )?;
+
         Ok(())
     }
 
@@ -1085,10 +1166,8 @@ impl Vecs {
                 &self.inputindex_to_value as &dyn AnyCollectableVec,
                 &self.txindex_to_fee,
                 &self.txindex_to_feerate,
-                &self.txindex_to_input_count,
                 &self.txindex_to_input_value,
                 &self.txindex_to_is_coinbase,
-                &self.txindex_to_output_count,
                 &self.txindex_to_output_value,
                 &self.txindex_to_vsize,
                 &self.txindex_to_weight,
@@ -1117,7 +1196,11 @@ impl Vecs {
             self.indexes_to_tx_vsize.vecs(),
             self.indexes_to_tx_weight.vecs(),
             self.indexes_to_unknownoutput_count.vecs(),
+            self.indexes_to_exact_utxo_count.vecs(),
+            self.indexes_to_unclaimed_rewards.vecs(),
         ]
-        .concat()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
     }
 }

crates/brk_core/Cargo.toml

@@ -4,20 +4,23 @@ description = "The Core (Structs and Errors) of the Bitcoin Research Kit"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
+bincode = { workspace = true }
 bitcoin = { workspace = true }
 bitcoincore-rpc = { workspace = true }
 byteview = { workspace = true }
 derive_deref = { workspace = true }
+fjall = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
 rapidhash = "1.4.0"
 rlimit = "0.10.2"
 serde = { workspace = true }
-serde_derive = { workspace = true }
-serde_bytes = "0.11.17"
+serde_bytes = { workspace = true }
+serde_json = { workspace = true }
 zerocopy = { workspace = true }
 zerocopy-derive = { workspace = true }
 

crates/brk_core/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>

crates/brk_core/src/enums/error.rs

@@ -1,20 +1,26 @@
 use std::{
     fmt::{self, Debug},
-    io,
-    time::SystemTimeError,
+    io, result, time,
 };
 
 use crate::Version;
 
-pub type Result<T, E = Error> = std::result::Result<T, E>;
+pub type Result<T, E = Error> = result::Result<T, E>;
 
 #[derive(Debug)]
 pub enum Error {
+    IO(io::Error),
+    SerdeJson(serde_json::Error),
+    Jiff(jiff::Error),
+    Fjall(fjall::Error),
+    SystemTimeError(time::SystemTimeError),
+    ZeroCopyError,
+    BincodeEncodeError(bincode::error::EncodeError),
+    BincodeDecodeError(bincode::error::DecodeError),
+
     WrongEndian,
     DifferentVersion { found: Version, expected: Version },
     MmapsVecIsTooSmall,
-    IO(io::Error),
-    ZeroCopyError,
     IndexTooHigh,
     EmptyVec,
     IndexTooLow,
@@ -24,13 +30,16 @@ pub enum Error {
     UnsupportedUnflushedState,
     RangeFromAfterTo(usize, usize),
     DifferentCompressionMode,
-    SystemTimeError,
-    ToSerdeJsonValueError(serde_json::Error),
+    WrongLength,
+    WrongAddressType,
+    UnindexableDate,
+
+    String(&'static str),
 }
 
-impl From<SystemTimeError> for Error {
-    fn from(_: SystemTimeError) -> Self {
-        Self::SystemTimeError
+impl From<time::SystemTimeError> for Error {
+    fn from(value: time::SystemTimeError) -> Self {
+        Self::SystemTimeError(value)
     }
 }
 
@@ -40,6 +49,18 @@ impl From<io::Error> for Error {
     }
 }
 
+impl From<jiff::Error> for Error {
+    fn from(value: jiff::Error) -> Self {
+        Self::Jiff(value)
+    }
+}
+
+impl From<fjall::Error> for Error {
+    fn from(value: fjall::Error) -> Self {
+        Self::Fjall(value)
+    }
+}
+
 impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
     fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
         Self::ZeroCopyError
@@ -54,13 +75,34 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
 
 impl From<serde_json::Error> for Error {
     fn from(error: serde_json::Error) -> Self {
-        Self::ToSerdeJsonValueError(error)
+        Self::SerdeJson(error)
+    }
+}
+
+impl From<bincode::error::DecodeError> for Error {
+    fn from(error: bincode::error::DecodeError) -> Self {
+        Self::BincodeDecodeError(error)
+    }
+}
+
+impl From<bincode::error::EncodeError> for Error {
+    fn from(error: bincode::error::EncodeError) -> Self {
+        Self::BincodeEncodeError(error)
     }
 }
 
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
+            Error::IO(error) => Debug::fmt(&error, f),
+            Error::SystemTimeError(error) => Debug::fmt(&error, f),
+            Error::SerdeJson(error) => Debug::fmt(&error, f),
+            Error::Jiff(error) => Debug::fmt(&error, f),
+            Error::Fjall(error) => Debug::fmt(&error, f),
+            Error::BincodeDecodeError(error) => Debug::fmt(&error, f),
+            Error::BincodeEncodeError(error) => Debug::fmt(&error, f),
+            Error::ZeroCopyError => write!(f, "ZeroCopy error"),
+
             Error::WrongEndian => write!(f, "Wrong endian"),
             Error::DifferentVersion { found, expected } => {
                 write!(
@@ -69,7 +111,6 @@ impl fmt::Display for Error {
                 )
             }
             Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
-            Error::IO(error) => Debug::fmt(&error, f),
             Error::IndexTooHigh => write!(f, "Index too high"),
             Error::IndexTooLow => write!(f, "Index too low"),
             Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -81,12 +122,17 @@ impl fmt::Display for Error {
                     "Unsupported unflush state, please flush before using this function"
                 )
             }
-            Error::ZeroCopyError => write!(f, "Zero copy convert error"),
-            Error::SystemTimeError => write!(f, "SystemTimeError"),
             Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
             Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
             Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
-            Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
+            Error::WrongLength => write!(f, "Wrong length"),
+            Error::WrongAddressType => write!(f, "Wrong address type"),
+            Error::UnindexableDate => write!(
+                f,
+                "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
+            ),
+
+            Error::String(s) => write!(f, "{s}"),
         }
     }
 }

crates/brk_core/src/error.rs

@@ -1,49 +0,0 @@
-use std::{
-    fmt::{self, Debug},
-    io,
-};
-
-pub type Result<T, E = Error> = std::result::Result<T, E>;
-
-#[derive(Debug)]
-pub enum Error {
-    IO(io::Error),
-    Jiff(jiff::Error),
-    ZeroCopyError,
-    WrongLength,
-    WrongAddressType,
-    UnindexableDate,
-}
-
-impl From<io::Error> for Error {
-    fn from(value: io::Error) -> Self {
-        Self::IO(value)
-    }
-}
-
-impl From<jiff::Error> for Error {
-    fn from(value: jiff::Error) -> Self {
-        Self::Jiff(value)
-    }
-}
-
-impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
-    fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
-        Self::ZeroCopyError
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Error::IO(error) => Debug::fmt(&error, f),
-            Error::Jiff(error) => Debug::fmt(&error, f),
-            Error::ZeroCopyError => write!(f, "Zero copy convert error"),
-            Error::WrongLength => write!(f, "Wrong length"),
-            Error::WrongAddressType => write!(f, "Wrong address type"),
-            Error::UnindexableDate => write!(f, "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"),
-        }
-    }
-}
-
-impl std::error::Error for Error {}

crates/brk_core/src/lib.rs

@@ -1,9 +1,9 @@
 #![doc = include_str!("../README.md")]
 
-mod error;
+mod enums;
 mod structs;
 mod utils;
 
-pub use error::*;
+pub use enums::*;
 pub use structs::*;
 pub use utils::*;

crates/brk_core/src/structs/_addressindex.rs

@@ -63,9 +63,8 @@ impl From<AddressIndex> for usize {
     }
 }
 
-impl TryFrom<ByteView> for AddressIndex {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
+impl From<ByteView> for AddressIndex {
+    fn from(value: ByteView) -> Self {
         Ok(Self::read_from_bytes(&value)?)
     }
 }

crates/brk_core/src/structs/_addressindexoutputindex.rs

@@ -15,9 +15,8 @@ pub struct AddressIndexOutputIndex {
     outputindex: Outputindex,
 }
 
-impl TryFrom<ByteView> for AddressIndexOutputIndex {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
+impl From<ByteView> for AddressIndexOutputIndex {
+    fn from(value: ByteView) -> Self {
         Ok(Self::read_from_bytes(&value)?)
     }
 }

crates/brk_core/src/structs/addressbyteshash.rs

@@ -5,8 +5,6 @@ use derive_deref::Deref;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::Error;
-
 use super::{AddressBytes, OutputType};
 
 #[derive(
@@ -41,10 +39,9 @@ impl From<[u8; 8]> for AddressBytesHash {
     }
 }
 
-impl TryFrom<ByteView> for AddressBytesHash {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for AddressBytesHash {
+    fn from(value: ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 

crates/brk_core/src/structs/bitcoin.rs

@@ -1,23 +1,16 @@
-use std::ops::{Add, Div, Mul};
+use std::{
+    cmp::Ordering,
+    ops::{Add, Div, Mul},
+};
 
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use super::Sats;
+use crate::CheckedSub;
 
-#[derive(
-    Debug,
-    Default,
-    Clone,
-    Copy,
-    PartialEq,
-    PartialOrd,
-    FromBytes,
-    Immutable,
-    IntoBytes,
-    KnownLayout,
-    Serialize,
-)]
+use super::{Sats, StoredF64};
+
+#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
 pub struct Bitcoin(f64);
 
 impl Add for Bitcoin {
@@ -34,6 +27,21 @@ impl Mul for Bitcoin {
     }
 }
 
+impl Mul<usize> for Bitcoin {
+    type Output = Self;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Self::from(Sats::from(self) * rhs)
+    }
+}
+
+impl Div<Bitcoin> for Bitcoin {
+    type Output = StoredF64;
+    fn div(self, rhs: Bitcoin) -> Self::Output {
+        StoredF64::from(self.0 / rhs.0)
+        // Self::from(Sats::from(self) / Sats::from(rhs))
+    }
+}
+
 impl Div<usize> for Bitcoin {
     type Output = Self;
     fn div(self, rhs: usize) -> Self::Output {
@@ -53,6 +61,12 @@ impl From<f64> for Bitcoin {
     }
 }
 
+impl From<StoredF64> for Bitcoin {
+    fn from(value: StoredF64) -> Self {
+        Self(*value)
+    }
+}
+
 impl From<Bitcoin> for f64 {
     fn from(value: Bitcoin) -> Self {
         value.0
@@ -65,11 +79,40 @@ impl From<usize> for Bitcoin {
     }
 }
 
+impl PartialEq for Bitcoin {
+    fn eq(&self, other: &Self) -> bool {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => true,
+            (true, false) => false,
+            (false, true) => false,
+            (false, false) => self.0 == other.0,
+        }
+    }
+}
+
 impl Eq for Bitcoin {}
 
 #[allow(clippy::derive_ord_xor_partial_ord)]
-impl Ord for Bitcoin {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.0.partial_cmp(&other.0).unwrap()
+impl PartialOrd for Bitcoin {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for Bitcoin {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => Ordering::Equal,
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            (false, false) => self.0.partial_cmp(&other.0).unwrap(),
+        }
+    }
+}
+
+impl CheckedSub<usize> for Bitcoin {
+    fn checked_sub(self, rhs: usize) -> Option<Self> {
+        Some(Self(self.0 - rhs as f64))
     }
 }

crates/brk_core/src/structs/blockhashprefix.rs

@@ -3,7 +3,7 @@ use derive_deref::Deref;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::{Error, copy_first_8bytes};
+use crate::copy_first_8bytes;
 
 use super::BlockHash;
 
@@ -35,10 +35,9 @@ impl From<&BlockHash> for BlockHashPrefix {
     }
 }
 
-impl TryFrom<ByteView> for BlockHashPrefix {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for BlockHashPrefix {
+    fn from(value: ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 

crates/brk_core/src/structs/cents.rs

@@ -1,8 +1,10 @@
-use std::ops::{Add, Div};
+use std::ops::{Add, Div, Mul};
 
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
+use crate::CheckedSub;
+
 use super::Dollars;
 
 #[derive(
@@ -20,11 +22,17 @@ use super::Dollars;
     KnownLayout,
     Serialize,
 )]
-pub struct Cents(u64);
+pub struct Cents(i64);
+
+impl Cents {
+    pub const fn mint(value: i64) -> Self {
+        Self(value)
+    }
+}
 
 impl From<Dollars> for Cents {
     fn from(value: Dollars) -> Self {
-        Self((*value * 100.0).round() as u64)
+        Self((*value * 100.0).round() as i64)
     }
 }
 
@@ -34,15 +42,45 @@ impl From<Cents> for f64 {
     }
 }
 
+impl From<i64> for Cents {
+    fn from(value: i64) -> Self {
+        Self(value)
+    }
+}
+
 impl From<u64> for Cents {
     fn from(value: u64) -> Self {
-        Self(value)
+        Self(value as i64)
+    }
+}
+
+impl From<Cents> for usize {
+    fn from(value: Cents) -> Self {
+        if value.0 < 0 {
+            panic!()
+        }
+        value.0 as usize
+    }
+}
+
+impl From<usize> for Cents {
+    fn from(value: usize) -> Self {
+        Self(value as i64)
+    }
+}
+
+impl From<Cents> for i64 {
+    fn from(value: Cents) -> Self {
+        value.0
     }
 }
 
 impl From<Cents> for u64 {
     fn from(value: Cents) -> Self {
-        value.0
+        if value.0 < 0 {
+            panic!("Shouldn't convert neg cents to u64")
+        }
+        value.0 as u64
     }
 }
 
@@ -53,24 +91,61 @@ impl Add for Cents {
     }
 }
 
+impl Div<Cents> for Cents {
+    type Output = Self;
+    fn div(self, rhs: Self) -> Self::Output {
+        Self(self.0 / rhs.0)
+    }
+}
+
 impl Div<usize> for Cents {
     type Output = Self;
     fn div(self, rhs: usize) -> Self::Output {
-        Self(self.0 / rhs as u64)
+        Self(self.0 / rhs as i64)
     }
 }
 
 impl From<u128> for Cents {
     fn from(value: u128) -> Self {
-        if value > u64::MAX as u128 {
-            panic!("u128 bigger than u64")
+        if value > i64::MAX as u128 {
+            panic!("u128 bigger than i64")
         }
-        Self(value as u64)
+        Self(value as i64)
     }
 }
 
 impl From<Cents> for u128 {
     fn from(value: Cents) -> Self {
+        if value.0 < 0 {
+            panic!("Shouldn't convert neg cents to u128")
+        }
         value.0 as u128
     }
 }
+
+impl Mul<Cents> for Cents {
+    type Output = Cents;
+    fn mul(self, rhs: Cents) -> Self::Output {
+        Self(self.0.checked_mul(rhs.0).unwrap())
+    }
+}
+
+impl Mul<i64> for Cents {
+    type Output = Cents;
+    fn mul(self, rhs: i64) -> Self::Output {
+        Self(self.0 * rhs)
+    }
+}
+
+impl Mul<usize> for Cents {
+    type Output = Cents;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Self(self.0 * rhs as i64)
+    }
+}
+
+impl CheckedSub for Cents {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Cents::from)
+    }
+}

crates/brk_core/src/structs/date.rs

@@ -14,6 +14,7 @@ impl Date {
     pub const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3);
     pub const INDEX_ONE: Self = Self(20090109);
     pub const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9);
+    pub const MIN_RATIO: Self = Self(20120101);
 
     pub fn new(year: u16, month: u8, day: u8) -> Self {
         Self(year as u32 * 1_00_00 + month as u32 * 1_00 + day as u32)

crates/brk_core/src/structs/dollars.rs

@@ -1,29 +1,49 @@
-use std::ops::{Add, Div, Mul};
+use std::{
+    cmp::Ordering,
+    f64,
+    ops::{Add, AddAssign, Div, Mul},
+};
 
+use bincode::{Decode, Encode};
+use byteview::ByteView;
 use derive_deref::Deref;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use super::{Bitcoin, Cents, Sats};
+use crate::{CheckedSub, copy_first_8bytes};
+
+use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64};
 
 #[derive(
     Debug,
     Default,
     Clone,
     Copy,
-    PartialEq,
-    PartialOrd,
     Deref,
     FromBytes,
     Immutable,
     IntoBytes,
     KnownLayout,
     Serialize,
+    Deserialize,
+    Encode,
+    Decode,
 )]
 pub struct Dollars(f64);
 
 impl Dollars {
     pub const ZERO: Self = Self(0.0);
+    pub const NAN: Self = Self(f64::NAN);
+
+    pub const fn mint(dollars: f64) -> Self {
+        Self(dollars)
+    }
+}
+
+impl From<f32> for Dollars {
+    fn from(value: f32) -> Self {
+        Self(value as f64)
+    }
 }
 
 impl From<f64> for Dollars {
@@ -38,12 +58,30 @@ impl From<Cents> for Dollars {
     }
 }
 
+impl From<Dollars> for f32 {
+    fn from(value: Dollars) -> Self {
+        value.0 as f32
+    }
+}
+
 impl From<Dollars> for f64 {
     fn from(value: Dollars) -> Self {
         value.0
     }
 }
 
+impl From<Close<Dollars>> for Dollars {
+    fn from(value: Close<Dollars>) -> Self {
+        Self(value.0)
+    }
+}
+
+impl From<High<Dollars>> for Dollars {
+    fn from(value: High<Dollars>) -> Self {
+        Self(value.0)
+    }
+}
+
 impl From<usize> for Dollars {
     fn from(value: usize) -> Self {
         Self(value as f64)
@@ -57,27 +95,242 @@ impl Add for Dollars {
     }
 }
 
+impl Div<Dollars> for Dollars {
+    type Output = StoredF64;
+    fn div(self, rhs: Dollars) -> Self::Output {
+        if self.is_nan() || rhs == Dollars::ZERO {
+            StoredF64::NAN
+        } else {
+            StoredF64::from(f64::from(self) / f64::from(rhs))
+        }
+    }
+}
+
+impl Div<Close<Dollars>> for Dollars {
+    type Output = StoredF64;
+    fn div(self, rhs: Close<Dollars>) -> Self::Output {
+        if self.is_nan() || *rhs == Dollars::ZERO {
+            StoredF64::NAN
+        } else {
+            StoredF64::from(f64::from(self) / f64::from(*rhs))
+        }
+    }
+}
+
+impl Div<Dollars> for Close<Dollars> {
+    type Output = StoredF64;
+    fn div(self, rhs: Dollars) -> Self::Output {
+        if self.is_nan() || rhs == Dollars::ZERO {
+            StoredF64::NAN
+        } else {
+            StoredF64::from(f64::from(*self) / f64::from(rhs))
+        }
+    }
+}
+
 impl Div<usize> for Dollars {
     type Output = Self;
     fn div(self, rhs: usize) -> Self::Output {
-        Self::from(Cents::from(self) / rhs)
+        if self.is_nan() || rhs == 0 {
+            Dollars::NAN
+        } else {
+            Self::from(Cents::from(self) / rhs)
+        }
+    }
+}
+
+impl Div<Bitcoin> for Dollars {
+    type Output = Self;
+    fn div(self, rhs: Bitcoin) -> Self::Output {
+        if self.is_nan() {
+            self
+        } else {
+            Self(f64::from(self) / f64::from(rhs))
+        }
+    }
+}
+
+impl Mul<Dollars> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: Dollars) -> Self::Output {
+        Self::from(Cents::from(self) * Cents::from(rhs))
+    }
+}
+
+impl Mul<Close<Dollars>> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: Close<Dollars>) -> Self::Output {
+        Self::from(Cents::from(self) * Cents::from(*rhs))
+    }
+}
+
+impl Mul<Dollars> for Close<Dollars> {
+    type Output = Dollars;
+    fn mul(self, rhs: Dollars) -> Self::Output {
+        Dollars::from(Cents::from(*self) * Cents::from(rhs))
+    }
+}
+
+impl Mul<usize> for Close<Dollars> {
+    type Output = Dollars;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Dollars::from(Cents::from(*self) * rhs)
+    }
+}
+
+impl Mul<Bitcoin> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: Bitcoin) -> Self::Output {
+        self * Sats::from(rhs)
+    }
+}
+
+impl Mul<Bitcoin> for Close<Dollars> {
+    type Output = Dollars;
+    fn mul(self, rhs: Bitcoin) -> Self::Output {
+        *self * Sats::from(rhs)
+    }
+}
+
+impl Mul<Sats> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: Sats) -> Self::Output {
+        if self.is_nan() {
+            self
+        } else {
+            Self::from(Cents::from(
+                u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
+            ))
+        }
+    }
+}
+
+impl Mul<StoredF32> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: StoredF32) -> Self::Output {
+        if rhs.fract() != 0.0 {
+            Self::from(self.0 * *rhs as f64)
+        } else {
+            self * *rhs as i64
+        }
+    }
+}
+
+impl Mul<i64> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: i64) -> Self::Output {
+        Self::from(Cents::from(self) * rhs)
+    }
+}
+
+impl Mul<usize> for Dollars {
+    type Output = Self;
+    fn mul(self, rhs: usize) -> Self::Output {
+        if self.is_nan() {
+            self
+        } else {
+            Self::from(Cents::from(self) * rhs)
+        }
+    }
+}
+
+impl From<u128> for Dollars {
+    fn from(value: u128) -> Self {
+        Self::from(Cents::from(value))
+    }
+}
+
+impl From<StoredF64> for Dollars {
+    fn from(value: StoredF64) -> Self {
+        Self(*value)
+    }
+}
+
+impl From<Close<Dollars>> for u128 {
+    fn from(value: Close<Dollars>) -> Self {
+        u128::from(*value)
+    }
+}
+
+impl From<Dollars> for u128 {
+    fn from(value: Dollars) -> Self {
+        u128::from(Cents::from(value))
+    }
+}
+
+impl AddAssign for Dollars {
+    fn add_assign(&mut self, rhs: Self) {
+        *self = Dollars::from(Cents::from(*self) + Cents::from(rhs));
+    }
+}
+
+impl CheckedSub for Dollars {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        if self.is_nan() {
+            Some(self)
+        } else {
+            Cents::from(self)
+                .checked_sub(Cents::from(rhs))
+                .map(Dollars::from)
+        }
+    }
+}
+
+impl CheckedSub<usize> for Dollars {
+    fn checked_sub(self, rhs: usize) -> Option<Self> {
+        Some(Dollars::from(
+            Cents::from(self).checked_sub(Cents::from(rhs)).unwrap(),
+        ))
+    }
+}
+
+impl PartialEq for Dollars {
+    fn eq(&self, other: &Self) -> bool {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => true,
+            (true, false) => false,
+            (false, true) => false,
+            (false, false) => self.0 == other.0,
+        }
     }
 }
 
 impl Eq for Dollars {}
 
 #[allow(clippy::derive_ord_xor_partial_ord)]
-impl Ord for Dollars {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.0.partial_cmp(&other.0).unwrap()
+impl PartialOrd for Dollars {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
     }
 }
 
-impl Mul<Bitcoin> for Dollars {
-    type Output = Dollars;
-    fn mul(self, rhs: Bitcoin) -> Self::Output {
-        Self::from(Cents::from(
-            u128::from(Sats::from(rhs)) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
-        ))
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for Dollars {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => Ordering::Equal,
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            (false, false) => self.0.partial_cmp(&other.0).unwrap(),
+        }
+    }
+}
+
+impl From<ByteView> for Dollars {
+    fn from(value: ByteView) -> Self {
+        let bytes = copy_first_8bytes(&value).unwrap();
+        Self::from(f64::from_be_bytes(bytes))
+    }
+}
+
+impl From<Dollars> for ByteView {
+    fn from(value: Dollars) -> Self {
+        Self::from(&value)
+    }
+}
+
+impl From<&Dollars> for ByteView {
+    fn from(value: &Dollars) -> Self {
+        Self::new(&value.to_be_bytes())
     }
 }

crates/brk_core/src/structs/feerate.rs

@@ -1,22 +1,14 @@
-use std::ops::{Add, Div};
+use std::{
+    cmp::Ordering,
+    ops::{Add, Div},
+};
 
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
 use super::{Sats, StoredUsize};
 
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    Serialize,
-    PartialEq,
-    PartialOrd,
-    FromBytes,
-    Immutable,
-    IntoBytes,
-    KnownLayout,
-)]
+#[derive(Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
 pub struct Feerate(f32);
 
 impl From<(Sats, StoredUsize)> for Feerate {
@@ -56,11 +48,34 @@ impl From<usize> for Feerate {
     }
 }
 
+impl PartialEq for Feerate {
+    fn eq(&self, other: &Self) -> bool {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => true,
+            (true, false) => false,
+            (false, true) => false,
+            (false, false) => self.0 == other.0,
+        }
+    }
+}
+
 impl Eq for Feerate {}
 
 #[allow(clippy::derive_ord_xor_partial_ord)]
-impl Ord for Feerate {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.0.partial_cmp(&other.0).unwrap()
+impl PartialOrd for Feerate {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for Feerate {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => Ordering::Equal,
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            (false, false) => self.0.partial_cmp(&other.0).unwrap(),
+        }
     }
 }

crates/brk_core/src/structs/halvingepoch.rs

@@ -28,6 +28,12 @@ use super::Height;
 )]
 pub struct HalvingEpoch(u8);
 
+impl HalvingEpoch {
+    pub fn new(value: u8) -> Self {
+        Self(value)
+    }
+}
+
 impl From<u8> for HalvingEpoch {
     fn from(value: u8) -> Self {
         Self(value)

crates/brk_core/src/structs/height.rs

@@ -4,12 +4,15 @@ use std::{
 };
 
 use bitcoincore_rpc::{Client, RpcApi};
+use byteview::ByteView;
 use serde::{Deserialize, Serialize};
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
 use crate::CheckedSub;
 
+use super::StoredUsize;
+
 #[derive(
     Debug,
     Clone,
@@ -147,11 +150,17 @@ impl From<u64> for Height {
     }
 }
 
+impl From<StoredUsize> for Height {
+    fn from(value: StoredUsize) -> Self {
+        Self(*value as u32)
+    }
+}
 impl From<usize> for Height {
     fn from(value: usize) -> Self {
         Self(value as u32)
     }
 }
+
 impl From<Height> for usize {
     fn from(value: Height) -> Self {
         value.0 as usize
@@ -189,10 +198,9 @@ impl TryFrom<&std::path::Path> for Height {
     }
 }
 
-impl TryFrom<byteview::ByteView> for Height {
-    type Error = crate::Error;
-    fn try_from(value: byteview::ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for Height {
+    fn from(value: byteview::ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 

crates/brk_core/src/structs/mod.rs

@@ -24,6 +24,7 @@ mod outputtypeindex;
 mod quarterindex;
 mod rawlocktime;
 mod sats;
+mod stored_f32;
 mod stored_f64;
 mod stored_u32;
 mod stored_u64;
@@ -35,6 +36,7 @@ mod txidprefix;
 mod txindex;
 mod txversion;
 mod unit;
+mod version;
 mod vin;
 mod vout;
 mod weekindex;
@@ -67,6 +69,7 @@ pub use outputtypeindex::*;
 pub use quarterindex::*;
 pub use rawlocktime::*;
 pub use sats::*;
+pub use stored_f32::*;
 pub use stored_f64::*;
 pub use stored_u8::*;
 pub use stored_u32::*;
@@ -78,8 +81,11 @@ pub use txidprefix::*;
 pub use txindex::*;
 pub use txversion::*;
 pub use unit::*;
+pub use version::*;
 pub use vin::*;
 pub use vout::*;
 pub use weekindex::*;
 pub use weight::*;
 pub use yearindex::*;
+
+pub use rlimit;

crates/brk_core/src/structs/ohlc.rs

@@ -453,6 +453,15 @@ where
     }
 }
 
+impl<T> From<f32> for Close<T>
+where
+    T: From<f32>,
+{
+    fn from(value: f32) -> Self {
+        Self(T::from(value))
+    }
+}
+
 impl<T> From<f64> for Close<T>
 where
     T: From<f64>,
@@ -462,6 +471,15 @@ where
     }
 }
 
+impl<T> From<Close<T>> for f32
+where
+    f32: From<T>,
+{
+    fn from(value: Close<T>) -> Self {
+        Self::from(value.0)
+    }
+}
+
 impl<T> From<Close<T>> for f64
 where
     f64: From<T>,

crates/brk_core/src/structs/outputtype.rs

@@ -32,6 +32,46 @@ pub enum OutputType {
     Unknown = 255,
 }
 
+impl OutputType {
+    pub fn is_spendable(&self) -> bool {
+        match self {
+            Self::P2PK65 => true,
+            Self::P2PK33 => true,
+            Self::P2PKH => true,
+            Self::P2MS => true,
+            Self::P2SH => true,
+            Self::OpReturn => false,
+            Self::P2WPKH => true,
+            Self::P2WSH => true,
+            Self::P2TR => true,
+            Self::P2A => true,
+            Self::Empty => true,
+            Self::Unknown => true,
+        }
+    }
+
+    pub fn is_unspendable(&self) -> bool {
+        !self.is_spendable()
+    }
+
+    pub fn as_vec() -> Vec<Self> {
+        vec![
+            Self::P2PK65,
+            Self::P2PK33,
+            Self::P2PKH,
+            Self::P2MS,
+            Self::P2SH,
+            Self::OpReturn,
+            Self::P2WPKH,
+            Self::P2WSH,
+            Self::P2TR,
+            Self::P2A,
+            Self::Empty,
+            Self::Unknown,
+        ]
+    }
+}
+
 impl From<&ScriptBuf> for OutputType {
     fn from(script: &ScriptBuf) -> Self {
         if script.is_p2pk() {

crates/brk_core/src/structs/outputtypeindex.rs

@@ -6,7 +6,7 @@ use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::{CheckedSub, Error};
+use crate::CheckedSub;
 
 #[derive(
     Debug,
@@ -82,10 +82,9 @@ impl Add<OutputTypeIndex> for OutputTypeIndex {
         Self(self.0 + rhs.0)
     }
 }
-impl TryFrom<ByteView> for OutputTypeIndex {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for OutputTypeIndex {
+    fn from(value: ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 impl From<OutputTypeIndex> for ByteView {

crates/brk_core/src/structs/sats.rs

@@ -3,11 +3,13 @@ use std::{
     ops::{Add, AddAssign, Div, Mul, SubAssign},
 };
 
+use bincode::{Decode, Encode};
 use bitcoin::Amount;
-use serde::Serialize;
+use byteview::ByteView;
+use serde::{Deserialize, Serialize};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::CheckedSub;
+use crate::{CheckedSub, copy_first_8bytes};
 
 use super::{Bitcoin, Cents, Dollars, Height};
 
@@ -25,17 +27,30 @@ use super::{Bitcoin, Cents, Dollars, Height};
     IntoBytes,
     KnownLayout,
     Serialize,
+    Deserialize,
+    Encode,
+    Decode,
 )]
 pub struct Sats(u64);
 
+#[allow(clippy::inconsistent_digit_grouping)]
 impl Sats {
     pub const ZERO: Self = Self(0);
     pub const MAX: Self = Self(u64::MAX);
-    pub const ONE_BTC: Self = Self(100_000_000);
+    pub const ONE_BTC: Self = Self(1_00_000_000);
+    pub const FIFTY_BTC: Self = Self(50_00_000_000);
+
+    pub fn new(sats: u64) -> Self {
+        Self(sats)
+    }
 
     pub fn is_zero(&self) -> bool {
         *self == Self::ZERO
     }
+
+    pub fn is_not_zero(&self) -> bool {
+        *self != Self::ZERO
+    }
 }
 
 impl Add for Sats {
@@ -57,6 +72,12 @@ impl CheckedSub<Sats> for Sats {
     }
 }
 
+impl CheckedSub<usize> for Sats {
+    fn checked_sub(self, rhs: usize) -> Option<Self> {
+        self.0.checked_sub(rhs as u64).map(Self::from)
+    }
+}
+
 impl SubAssign for Sats {
     fn sub_assign(&mut self, rhs: Self) {
         *self = self.checked_sub(rhs).unwrap();
@@ -66,21 +87,28 @@ impl SubAssign for Sats {
 impl Mul<Sats> for Sats {
     type Output = Self;
     fn mul(self, rhs: Sats) -> Self::Output {
-        Sats::from(self.0 * rhs.0)
+        Sats::from(self.0.checked_mul(rhs.0).unwrap())
+    }
+}
+
+impl Mul<usize> for Sats {
+    type Output = Self;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Sats::from(self.0.checked_mul(rhs as u64).unwrap())
     }
 }
 
 impl Mul<u64> for Sats {
     type Output = Self;
     fn mul(self, rhs: u64) -> Self::Output {
-        Sats::from(self.0 * rhs)
+        Sats::from(self.0.checked_mul(rhs).unwrap())
     }
 }
 
 impl Mul<Height> for Sats {
     type Output = Self;
     fn mul(self, rhs: Height) -> Self::Output {
-        Sats::from(self.0 * u64::from(rhs))
+        Sats::from(self.0.checked_mul(u64::from(rhs)).unwrap())
     }
 }
 
@@ -103,6 +131,17 @@ impl Div<Dollars> for Sats {
     }
 }
 
+impl Div<Sats> for Sats {
+    type Output = Self;
+    fn div(self, rhs: Sats) -> Self::Output {
+        if rhs.0 == 0 {
+            Self(0)
+        } else {
+            Self(self.0 / rhs.0)
+        }
+    }
+}
+
 impl Div<usize> for Sats {
     type Output = Self;
     fn div(self, rhs: usize) -> Self::Output {
@@ -134,6 +173,12 @@ impl From<Sats> for f64 {
     }
 }
 
+impl From<Sats> for usize {
+    fn from(value: Sats) -> Self {
+        value.0 as usize
+    }
+}
+
 impl From<Amount> for Sats {
     fn from(value: Amount) -> Self {
         Self(value.to_sat())
@@ -171,3 +216,29 @@ impl From<Sats> for u128 {
         value.0 as u128
     }
 }
+
+impl From<ByteView> for Sats {
+    fn from(value: ByteView) -> Self {
+        let bytes = copy_first_8bytes(&value).unwrap();
+        Self::from(u64::from_be_bytes(bytes))
+    }
+}
+
+impl From<&Sats> for ByteView {
+    fn from(value: &Sats) -> Self {
+        Self::new(&value.0.to_be_bytes())
+    }
+}
+
+impl From<Sats> for ByteView {
+    fn from(value: Sats) -> Self {
+        Self::from(&value)
+    }
+}
+
+impl Mul<Sats> for usize {
+    type Output = Sats;
+    fn mul(self, rhs: Sats) -> Self::Output {
+        Self::Output::from(rhs.0 * self as u64)
+    }
+}

crates/brk_core/src/structs/stored_f32.rs

@@ -0,0 +1,150 @@
+use core::panic;
+use std::{
+    cmp::Ordering,
+    ops::{Add, Div, Mul, Sub},
+};
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::CheckedSub;
+
+use super::{Dollars, StoredF64};
+
+#[derive(
+    Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
+)]
+pub struct StoredF32(f32);
+
+impl From<f32> for StoredF32 {
+    fn from(value: f32) -> Self {
+        Self(value)
+    }
+}
+
+impl From<f64> for StoredF32 {
+    fn from(value: f64) -> Self {
+        if value > f32::MAX as f64 {
+            panic!("f64 is too big")
+        }
+        Self(value as f32)
+    }
+}
+
+impl From<StoredF32> for f64 {
+    fn from(value: StoredF32) -> Self {
+        value.0 as f64
+    }
+}
+
+impl From<StoredF64> for StoredF32 {
+    fn from(value: StoredF64) -> Self {
+        Self(*value as f32)
+    }
+}
+
+impl From<usize> for StoredF32 {
+    fn from(value: usize) -> Self {
+        Self(value as f32)
+    }
+}
+
+impl CheckedSub<StoredF32> for StoredF32 {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        Some(Self(self.0 - rhs.0))
+    }
+}
+
+impl Div<usize> for StoredF32 {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self(self.0 / rhs as f32)
+    }
+}
+
+impl Add for StoredF32 {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl From<StoredF32> for f32 {
+    fn from(value: StoredF32) -> Self {
+        value.0
+    }
+}
+
+impl From<Dollars> for StoredF32 {
+    fn from(value: Dollars) -> Self {
+        StoredF32::from(f64::from(value))
+    }
+}
+
+impl Div<Dollars> for StoredF32 {
+    type Output = Self;
+    fn div(self, rhs: Dollars) -> Self::Output {
+        Self::from(self.0 as f64 / *rhs)
+    }
+}
+
+impl Div<StoredF32> for StoredF32 {
+    type Output = Self;
+    fn div(self, rhs: StoredF32) -> Self::Output {
+        Self::from(self.0 / rhs.0)
+    }
+}
+
+impl Mul<usize> for StoredF32 {
+    type Output = Self;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Self(self.0 * rhs as f32)
+    }
+}
+
+impl Mul<StoredF32> for usize {
+    type Output = StoredF32;
+    fn mul(self, rhs: StoredF32) -> Self::Output {
+        StoredF32(self as f32 * rhs.0)
+    }
+}
+
+impl Sub<StoredF32> for StoredF32 {
+    type Output = Self;
+    fn sub(self, rhs: StoredF32) -> Self::Output {
+        Self(self.0 - rhs.0)
+    }
+}
+
+impl PartialEq for StoredF32 {
+    fn eq(&self, other: &Self) -> bool {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => true,
+            (true, false) => false,
+            (false, true) => false,
+            (false, false) => self.0 == other.0,
+        }
+    }
+}
+
+impl Eq for StoredF32 {}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl PartialOrd for StoredF32 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for StoredF32 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => Ordering::Equal,
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            (false, false) => self.0.partial_cmp(&other.0).unwrap(),
+        }
+    }
+}

crates/brk_core/src/structs/stored_f64.rs

@@ -1,27 +1,24 @@
-use std::ops::{Add, Div};
+use std::{
+    cmp::Ordering,
+    f64,
+    ops::{Add, Div, Mul},
+};
 
 use derive_deref::Deref;
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::CheckedSub;
+use crate::{Bitcoin, CheckedSub, Dollars};
 
 #[derive(
-    Debug,
-    Deref,
-    Default,
-    Clone,
-    Copy,
-    PartialEq,
-    PartialOrd,
-    FromBytes,
-    Immutable,
-    IntoBytes,
-    KnownLayout,
-    Serialize,
+    Debug, Deref, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize,
 )]
 pub struct StoredF64(f64);
 
+impl StoredF64 {
+    pub const NAN: Self = Self(f64::NAN);
+}
+
 impl From<f64> for StoredF64 {
     fn from(value: f64) -> Self {
         Self(value)
@@ -40,6 +37,13 @@ impl CheckedSub<StoredF64> for StoredF64 {
     }
 }
 
+impl Mul<usize> for StoredF64 {
+    type Output = Self;
+    fn mul(self, rhs: usize) -> Self::Output {
+        Self(self.0 * rhs as f64)
+    }
+}
+
 impl Div<usize> for StoredF64 {
     type Output = Self;
     fn div(self, rhs: usize) -> Self::Output {
@@ -60,11 +64,52 @@ impl From<StoredF64> for f64 {
     }
 }
 
+impl From<Dollars> for StoredF64 {
+    fn from(value: Dollars) -> Self {
+        Self(f64::from(value))
+    }
+}
+
+impl CheckedSub<usize> for StoredF64 {
+    fn checked_sub(self, rhs: usize) -> Option<Self> {
+        Some(Self(self.0 - rhs as f64))
+    }
+}
+
+impl PartialEq for StoredF64 {
+    fn eq(&self, other: &Self) -> bool {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => true,
+            (true, false) => false,
+            (false, true) => false,
+            (false, false) => self.0 == other.0,
+        }
+    }
+}
+
 impl Eq for StoredF64 {}
 
 #[allow(clippy::derive_ord_xor_partial_ord)]
-impl Ord for StoredF64 {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.0.partial_cmp(&other.0).unwrap()
+impl PartialOrd for StoredF64 {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for StoredF64 {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self.0.is_nan(), other.0.is_nan()) {
+            (true, true) => Ordering::Equal,
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            (false, false) => self.0.partial_cmp(&other.0).unwrap(),
+        }
+    }
+}
+
+impl From<Bitcoin> for StoredF64 {
+    fn from(value: Bitcoin) -> Self {
+        Self(f64::from(value))
     }
 }

crates/brk_core/src/structs/stored_u32.rs

@@ -15,6 +15,7 @@ use super::{
     Debug,
     Deref,
     Clone,
+    Default,
     Copy,
     PartialEq,
     Eq,

crates/brk_core/src/structs/stored_usize.rs

@@ -1,6 +1,6 @@
 use std::ops::{Add, Div};
 
-use derive_deref::Deref;
+use derive_deref::{Deref, DerefMut};
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
@@ -15,6 +15,7 @@ use super::{
 #[derive(
     Debug,
     Deref,
+    DerefMut,
     Clone,
     Default,
     Copy,

crates/brk_core/src/structs/timestamp.rs

@@ -1,4 +1,7 @@
-use std::ops::{Add, Div};
+use std::{
+    cmp::Ordering,
+    ops::{Add, Div},
+};
 
 use derive_deref::Deref;
 use jiff::{civil::date, tz::TimeZone};
@@ -26,6 +29,8 @@ use super::Date;
 )]
 pub struct Timestamp(u32);
 
+const ONE_DAY_IN_SEC: i64 = 24 * 60 * 60;
+
 impl Timestamp {
     pub const ZERO: Self = Self(0);
 
@@ -34,10 +39,41 @@ impl Timestamp {
     }
 
     pub fn floor_seconds(self) -> Self {
-        let t = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC);
-        let d = jiff::civil::DateTime::from(t);
-        let d = date(d.year(), d.month(), d.day()).at(d.hour(), d.minute(), 0, 0);
-        Self::from(d.to_zoned(TimeZone::UTC).unwrap().timestamp())
+        let zoned = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC);
+        let date_time = jiff::civil::DateTime::from(zoned);
+        let trunc_date_time = date(date_time.year(), date_time.month(), date_time.day()).at(
+            date_time.hour(),
+            date_time.minute(),
+            0,
+            0,
+        );
+        Self::from(trunc_date_time.to_zoned(TimeZone::UTC).unwrap().timestamp())
+    }
+
+    pub fn difference_in_days_between(&self, other: Self) -> usize {
+        match self.cmp(&other) {
+            Ordering::Equal => 0,
+            Ordering::Greater => other.difference_in_days_between(*self),
+            Ordering::Less => {
+                (jiff::Timestamp::from(*self)
+                    .duration_until(jiff::Timestamp::from(other))
+                    .as_secs()
+                    / ONE_DAY_IN_SEC) as usize
+            }
+        }
+    }
+
+    pub fn difference_in_days_between_float(&self, other: Self) -> f64 {
+        match self.cmp(&other) {
+            Ordering::Equal => 0.0,
+            Ordering::Greater => other.difference_in_days_between_float(*self),
+            Ordering::Less => {
+                jiff::Timestamp::from(*self)
+                    .duration_until(jiff::Timestamp::from(other))
+                    .as_secs() as f64
+                    / ONE_DAY_IN_SEC as f64
+            }
+        }
     }
 }
 

crates/brk_core/src/structs/txidprefix.rs

@@ -3,7 +3,7 @@ use derive_deref::Deref;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::{Error, copy_first_8bytes};
+use crate::copy_first_8bytes;
 
 use super::Txid;
 
@@ -35,10 +35,9 @@ impl From<&Txid> for TxidPrefix {
     }
 }
 
-impl TryFrom<ByteView> for TxidPrefix {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for TxidPrefix {
+    fn from(value: ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 

crates/brk_core/src/structs/txindex.rs

@@ -6,7 +6,7 @@ use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::{CheckedSub, Error};
+use crate::CheckedSub;
 
 use super::StoredU32;
 
@@ -95,10 +95,9 @@ impl From<TxIndex> for usize {
     }
 }
 
-impl TryFrom<ByteView> for TxIndex {
-    type Error = Error;
-    fn try_from(value: ByteView) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
+impl From<ByteView> for TxIndex {
+    fn from(value: ByteView) -> Self {
+        Self::read_from_bytes(&value).unwrap()
     }
 }
 impl From<TxIndex> for ByteView {

crates/brk_core/src/structs/version.rs

@@ -1,6 +1,7 @@
 use std::{
     fs,
     io::{self, Read},
+    iter::Sum,
     ops::Add,
     path::Path,
 };
@@ -13,13 +14,17 @@ use crate::{Error, Result};
 #[derive(
     Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout,
 )]
-pub struct Version(u32);
+pub struct Version(u64);
 
 impl Version {
     pub const ZERO: Self = Self(0);
     pub const ONE: Self = Self(1);
     pub const TWO: Self = Self(2);
 
+    pub const fn new(v: u64) -> Self {
+        Self(v)
+    }
+
     pub fn write(&self, path: &Path) -> Result<(), io::Error> {
         fs::write(path, self.as_bytes())
     }
@@ -45,8 +50,8 @@ impl Version {
     }
 }
 
-impl From<u32> for Version {
-    fn from(value: u32) -> Self {
+impl From<u64> for Version {
+    fn from(value: u64) -> Self {
         Self(value)
     }
 }
@@ -54,7 +59,7 @@ impl From<u32> for Version {
 impl TryFrom<&Path> for Version {
     type Error = Error;
     fn try_from(value: &Path) -> Result<Self, Self::Error> {
-        let mut buf = [0; 4];
+        let mut buf = [0; 8];
         fs::read(value)?.as_slice().read_exact(&mut buf)?;
         Ok(*(Self::ref_from_bytes(&buf)?))
     }
@@ -66,3 +71,9 @@ impl Add<Version> for Version {
         Self(self.0 + rhs.0)
     }
 }
+
+impl Sum for Version {
+    fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
+        iter.fold(Self::ZERO, Add::add)
+    }
+}

crates/brk_core/src/utils/bytes.rs

@@ -1,9 +1,11 @@
+use crate::{Error, Result};
+
 #[allow(clippy::result_unit_err)]
-pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
+pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
     let mut buf: [u8; 8] = [0; 8];
     let buf_len = buf.len();
     if slice.len() < buf_len {
-        return Err(());
+        return Err(Error::String("Buffer is too small to convert to 8 bytes"));
     }
     slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
         buf[i] = *r;

crates/brk_core/src/utils/rlimit.rs

@@ -4,7 +4,14 @@ use rlimit::{Resource, getrlimit};
 
 pub fn setrlimit() -> io::Result<()> {
     let no_file_limit = getrlimit(Resource::NOFILE)?;
-    rlimit::setrlimit(Resource::NOFILE, no_file_limit.0.max(210_000), no_file_limit.1)?;
+    rlimit::setrlimit(
+        Resource::NOFILE,
+        no_file_limit.0.max(210_000),
+        no_file_limit.1,
+    )?;
+
+    // let no_stack = getrlimit(Resource::STACK)?;
+    // rlimit::setrlimit(Resource::STACK, no_stack.1, no_stack.1)?;
 
     Ok(())
 }

crates/brk_exit/Cargo.toml

@@ -4,9 +4,10 @@ description = "An exit blocker built on top of ctrlc"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
 brk_logger = { workspace = true }
-ctrlc = { version = "3.4.6", features = ["termination"] }
+ctrlc = { version = "3.4.7", features = ["termination"] }
 log = { workspace = true }

crates/brk_exit/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>

crates/brk_exit/src/lib.rs

@@ -57,6 +57,10 @@ impl Exit {
         self.blocking.store(true, Ordering::SeqCst);
     }
 
+    pub fn blocked(&self) -> bool {
+        self.blocking.load(Ordering::SeqCst)
+    }
+
     pub fn release(&self) {
         self.blocking.store(false, Ordering::SeqCst);
     }

crates/brk_fetcher/Cargo.toml

@@ -4,6 +4,7 @@ description = "A Bitcoin price fetcher"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]

crates/brk_fetcher/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>
@@ -34,4 +31,4 @@
   </a>
 </p>
 
-A crate that can fetch the Bitcoin price, either by date or height, from multiple APIs such Kraken, Binance and Kibo.money.
+A crate that can fetch the Bitcoin price, either by date or height, from Binance and Kibo.

crates/brk_fetcher/examples/main.rs

@@ -8,11 +8,11 @@ fn main() -> color_eyre::Result<()> {
 
     let mut fetcher = Fetcher::import(None)?;
 
-    dbg!(fetcher.get_date(Date::new(2025, 1, 1))?);
+    dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);
     dbg!(fetcher.get_height(
-        881_000_u32.into(),
-        1740683986_u32.into(),
-        Some(1740683000_u32.into())
+        899911_u32.into(),
+        1749133056_u32.into(),
+        Some(1749132055_u32.into())
     )?);
 
     Ok(())

crates/brk_fetcher/src/fetchers/binance.rs

@@ -17,7 +17,7 @@ use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, fetchers::retry};
 pub struct Binance {
     path: Option<PathBuf>,
     _1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
-    _1d: Option<BTreeMap<Date, OHLCCents>>,
+    pub _1d: Option<BTreeMap<Date, OHLCCents>>,
     har: Option<BTreeMap<Timestamp, OHLCCents>>,
 }
 
@@ -95,7 +95,7 @@ impl Binance {
     }
 
     pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
-        info!("Fetching daily prices from Kraken...");
+        info!("Fetching daily prices from Binance...");
 
         retry(
             |_| Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?),
@@ -238,4 +238,9 @@ impl Binance {
     fn url(query: &str) -> String {
         format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}")
     }
+
+    pub fn clear(&mut self) {
+        self._1d.take();
+        self._1mn.take();
+    }
 }

crates/brk_fetcher/src/fetchers/kibo.rs

@@ -38,7 +38,7 @@ impl Kibo {
             .unwrap()
             .get(usize::from(height.checked_sub(key).unwrap()))
             .cloned()
-            .ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo"))
+            .ok_or(eyre!("Couldn't find height in kibo"))
     }
 
     fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
@@ -147,4 +147,9 @@ impl Kibo {
             Close::new(get_value("close")?),
         )))
     }
+
+    pub fn clear(&mut self) {
+        self.height_to_ohlc_vec.clear();
+        self.year_to_date_to_ohlc.clear();
+    }
 }

crates/brk_fetcher/src/fetchers/kraken.rs

@@ -129,4 +129,9 @@ impl Kraken {
     fn url(interval: usize) -> String {
         format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}")
     }
+
+    pub fn clear(&mut self) {
+        self._1d.take();
+        self._1mn.take();
+    }
 }

crates/brk_fetcher/src/fetchers/mod.rs

@@ -1,9 +1,9 @@
 mod binance;
-mod kibo;
+// mod kibo;
 mod kraken;
 mod retry;
 
 pub use binance::*;
-pub use kibo::*;
+// pub use kibo::*;
 pub use kraken::*;
 use retry::*;

crates/brk_fetcher/src/lib.rs

@@ -3,20 +3,23 @@
 #![doc = include_str!("../examples/main.rs")]
 #![doc = "```"]
 
-use std::{collections::BTreeMap, fs, path::Path};
+use std::{collections::BTreeMap, fs, path::Path, thread::sleep, time::Duration};
 
-use brk_core::{Cents, Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
+use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
 use color_eyre::eyre::Error;
 
 mod fetchers;
 
 use fetchers::*;
+use log::info;
+
+const TRIES: usize = 12 * 60;
 
 #[derive(Clone)]
 pub struct Fetcher {
     binance: Binance,
     kraken: Kraken,
-    kibo: Kibo,
+    // kibo: Kibo,
 }
 
 impl Fetcher {
@@ -28,15 +31,34 @@ impl Fetcher {
         Ok(Self {
             binance: Binance::init(hars_path),
             kraken: Kraken::default(),
-            kibo: Kibo::default(),
+            // kibo: Kibo::default(),
         })
     }
 
     pub fn get_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
-        self.kraken
+        self.get_date_(date, 0)
+    }
+
+    fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
+        self.binance
             .get_from_1d(&date)
-            .or_else(|_| self.binance.get_from_1d(&date))
-            .or_else(|_| self.kibo.get_from_date(&date))
+            .or_else(|_| {
+                // eprintln!("{e}");
+                self.kraken.get_from_1d(&date)
+            })
+            .or_else(|e| {
+                sleep(Duration::from_secs(60));
+
+                if tries < TRIES {
+                    self.clear();
+                    // dbg!(e, date, &self.binance._1d);
+                    info!("Retrying to fetch date price...");
+                    self.get_date_(date, tries + 1)
+                } else {
+                    info!("Failed to fetch date prices...");
+                    Err(e)
+                }
+            })
     }
 
     pub fn get_height(
@@ -44,6 +66,16 @@ impl Fetcher {
         height: Height,
         timestamp: Timestamp,
         previous_timestamp: Option<Timestamp>,
+    ) -> color_eyre::Result<OHLCCents> {
+        self.get_height_(height, timestamp, previous_timestamp, 0)
+    }
+
+    fn get_height_(
+        &mut self,
+        height: Height,
+        timestamp: Timestamp,
+        previous_timestamp: Option<Timestamp>,
+        tries: usize,
     ) -> color_eyre::Result<OHLCCents> {
         let timestamp = timestamp.floor_seconds();
 
@@ -54,17 +86,36 @@ impl Fetcher {
         let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
 
         let ohlc = self
-            .kraken
+            .binance
             .get_from_1mn(timestamp, previous_timestamp)
-            .unwrap_or_else(|_| {
-                self.binance
+            .unwrap_or_else(|_report| {
+                // eprintln!("{_report}");
+                self.kraken
                     .get_from_1mn(timestamp, previous_timestamp)
-                    .unwrap_or_else(|_| {
-                        self.kibo.get_from_height(height).unwrap_or_else(|e| {
-                            let date = Date::from(timestamp);
-                            eprintln!("{e}");
-                            panic!(
-                                "
+                    .unwrap_or_else(|_report| {
+                        //         // eprintln!("{_report}");
+                        // self.kibo.get_from_height(height).unwrap_or_else(|_report| {
+                        // eprintln!("{_report}");
+
+                        sleep(Duration::from_secs(60));
+
+                        if tries < TRIES {
+                            self.clear();
+
+                            info!("Retrying to fetch height prices...");
+                            // dbg!((height, timestamp, previous_timestamp));
+
+                            return self
+                                .get_height_(height, timestamp, previous_timestamp, tries + 1)
+                                .unwrap();
+                        }
+
+                        info!("Failed to fetch height prices");
+
+                        let date = Date::from(timestamp);
+                        // eprintln!("{e}");
+                        panic!(
+                            "
 Can't find the price for: height: {height} - date: {date}
 1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
 How to fix this:
@@ -79,13 +130,11 @@ How to fix this:
 8. Export to a har file (if there is no explicit button, click on the cog button)
 9. Move the file to 'parser/imports/binance.har'
         "
-                            )
-                        })
+                        )
+                        // })
                     })
             });
 
-        // self.ohlc.height.insert(height, ohlc);
-
         Ok(ohlc)
     }
 
@@ -130,4 +179,10 @@ How to fix this:
 
         Ok(final_ohlc)
     }
+
+    pub fn clear(&mut self) {
+        self.binance.clear();
+        // self.kibo.clear();
+        self.kraken.clear();
+    }
 }

crates/brk_indexer/Cargo.toml

@@ -4,6 +4,7 @@ description = "A Bitcoin Core indexer built on top of brk_parser"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
@@ -11,12 +12,11 @@ bitcoin = { workspace = true }
 bitcoincore-rpc = { workspace = true }
 brk_core = { workspace = true }
 brk_exit = { workspace = true }
-brk_parser = { workspace = true }
 brk_logger = { workspace = true }
+brk_parser = { workspace = true }
+brk_store = { workspace = true }
 brk_vec = { workspace = true }
-byteview = { workspace = true }
 color-eyre = { workspace = true }
 fjall = { workspace = true }
 log = { workspace = true }
 rayon = { workspace = true }
-zerocopy = { workspace = true }

crates/brk_indexer/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>

crates/brk_indexer/examples/main.rs

@@ -3,7 +3,8 @@ use std::{path::Path, time::Instant};
 use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_parser::{Parser, rpc};
+use brk_parser::Parser;
+use brk_vec::Format;
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -14,9 +15,9 @@ fn main() -> color_eyre::Result<()> {
 
     let bitcoin_dir = default_bitcoin_path();
 
-    let rpc = Box::leak(Box::new(rpc::Client::new(
+    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
         "http://localhost:8332",
-        rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
+        bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
     )?));
     let exit = Exit::new();
 
@@ -24,7 +25,7 @@ fn main() -> color_eyre::Result<()> {
 
     let outputs = Path::new("../../_outputs");
 
-    let mut indexer = Indexer::new(outputs, false, false)?;
+    let mut indexer = Indexer::new(outputs, Format::Raw, false)?;
 
     indexer.import_stores()?;
     indexer.import_vecs()?;

crates/brk_indexer/src/indexes.rs

@@ -2,7 +2,7 @@ use bitcoincore_rpc::Client;
 use brk_core::{
     BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
     OutputType, OutputTypeIndex, P2AIndex, P2MSIndex, P2PK33Index, P2PK65Index, P2PKHIndex,
-    P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
+    P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, Result, TxIndex, UnknownOutputIndex,
 };
 use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
 use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
@@ -48,7 +48,7 @@ impl Indexes {
         }
     }
 
-    pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
+    pub fn push_if_needed(&self, vecs: &mut Vecs) -> Result<()> {
         let height = self.height;
         vecs.height_to_first_txindex
             .push_if_needed(height, self.txindex)?;

crates/brk_indexer/src/lib.rs

@@ -12,13 +12,14 @@ use std::{
 
 use brk_core::{
     AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
-    OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Vin, Vout, setrlimit,
+    OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Version, Vin, Vout,
+    setrlimit,
 };
-pub use brk_parser::*;
 
 use bitcoin::{Transaction, TxIn, TxOut};
 use brk_exit::Exit;
-use brk_vec::{AnyVec, Compressed, VecIterator};
+use brk_parser::Parser;
+use brk_vec::{AnyVec, Format, VecIterator};
 use color_eyre::eyre::{ContextCompat, eyre};
 use fjall::TransactionalKeyspace;
 use log::{error, info};
@@ -33,6 +34,7 @@ pub use vecs::*;
 
 const SNAPSHOT_BLOCK_RANGE: usize = 1000;
 const COLLISIONS_CHECKED_UP_TO: u32 = 893_000;
+const VERSION: Version = Version::ONE;
 
 #[derive(Clone)]
 pub struct Indexer {
@@ -40,13 +42,13 @@ pub struct Indexer {
     vecs: Option<Vecs>,
     stores: Option<Stores>,
     check_collisions: bool,
-    compressed: Compressed,
+    format: Format,
 }
 
 impl Indexer {
     pub fn new(
         outputs_dir: &Path,
-        compressed: bool,
+        format: Format,
         check_collisions: bool,
     ) -> color_eyre::Result<Self> {
         setrlimit()?;
@@ -54,7 +56,7 @@ impl Indexer {
             path: outputs_dir.to_owned(),
             vecs: None,
             stores: None,
-            compressed: Compressed::from(compressed),
+            format,
             check_collisions,
         })
     }
@@ -62,7 +64,8 @@ impl Indexer {
     pub fn import_vecs(&mut self) -> color_eyre::Result<()> {
         self.vecs = Some(Vecs::forced_import(
             &self.path.join("vecs/indexed"),
-            self.compressed,
+            VERSION + Version::ZERO,
+            self.format,
         )?);
         Ok(())
     }
@@ -70,14 +73,17 @@ impl Indexer {
     /// Do NOT import multiple times are things will break !!!
     /// Clone struct instead
     pub fn import_stores(&mut self) -> color_eyre::Result<()> {
-        self.stores = Some(Stores::forced_import(&self.path.join("stores"))?);
+        self.stores = Some(Stores::forced_import(
+            &self.path.join("stores"),
+            VERSION + Version::ZERO,
+        )?);
         Ok(())
     }
 
     pub fn index(
         &mut self,
         parser: &Parser,
-        rpc: &'static rpc::Client,
+        rpc: &'static bitcoincore_rpc::Client,
         exit: &Exit,
     ) -> color_eyre::Result<Indexes> {
         let starting_indexes = Indexes::try_from((
@@ -114,6 +120,7 @@ impl Indexer {
         if starting_indexes.height > Height::try_from(rpc)?
             || end.is_some_and(|end| starting_indexes.height > end)
         {
+            info!("Up to date, nothing to index.");
             return Ok(starting_indexes);
         }
 
@@ -744,18 +751,10 @@ impl Indexer {
         self.vecs.as_ref().unwrap()
     }
 
-    pub fn mut_vecs(&mut self) -> &mut Vecs {
-        self.vecs.as_mut().unwrap()
-    }
-
     pub fn stores(&self) -> &Stores {
         self.stores.as_ref().unwrap()
     }
 
-    pub fn mut_stores(&mut self) -> &mut Stores {
-        self.stores.as_mut().unwrap()
-    }
-
     pub fn keyspace(&self) -> &TransactionalKeyspace {
         &self.stores().keyspace
     }

crates/brk_indexer/src/stores.rs

@@ -1,20 +1,15 @@
 use std::{fs, path::Path, thread};
 
 use brk_core::{
-    AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
-    TxidPrefix,
+    AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, Result,
+    TxIndex, TxidPrefix, Value, Version,
 };
-use brk_vec::{AnyIterableVec, Value, Version};
+use brk_store::Store;
+use brk_vec::AnyIterableVec;
 use fjall::{PersistMode, TransactionalKeyspace};
 
 use crate::Indexes;
 
-mod base;
-mod meta;
-
-pub use base::*;
-pub use meta::*;
-
 use super::Vecs;
 
 #[derive(Clone)]
@@ -25,41 +20,46 @@ pub struct Stores {
     pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
 }
 
+const VERSION: Version = Version::ZERO;
+
 impl Stores {
-    pub fn forced_import(path: &Path) -> color_eyre::Result<Self> {
+    pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
-        let keyspace = match Self::open_keyspace(path) {
+        let keyspace = match brk_store::open_keyspace(path) {
             Ok(keyspace) => keyspace,
             Err(_) => {
                 fs::remove_dir_all(path)?;
-                return Self::forced_import(path);
+                return Self::forced_import(path, version);
             }
         };
 
         thread::scope(|scope| {
             let addressbyteshash_to_outputtypeindex = scope.spawn(|| {
                 Store::import(
-                    keyspace.clone(),
+                    &keyspace,
                     path,
                     "addressbyteshash_to_outputtypeindex",
-                    Version::ZERO,
+                    version + VERSION + Version::ZERO,
+                    None,
                 )
             });
             let blockhashprefix_to_height = scope.spawn(|| {
                 Store::import(
-                    keyspace.clone(),
+                    &keyspace,
                     path,
                     "blockhashprefix_to_height",
-                    Version::ZERO,
+                    version + VERSION + Version::ZERO,
+                    None,
                 )
             });
             let txidprefix_to_txindex = scope.spawn(|| {
                 Store::import(
-                    keyspace.clone(),
+                    &keyspace,
                     path,
                     "txidprefix_to_txindex",
-                    Version::ZERO,
+                    version + VERSION + Version::ZERO,
+                    None,
                 )
             });
 
@@ -288,8 +288,8 @@ impl Stores {
         .unwrap()
     }
 
-    pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
-        thread::scope(|scope| -> fjall::Result<()> {
+    pub fn commit(&mut self, height: Height) -> Result<()> {
+        thread::scope(|scope| -> Result<()> {
             let addressbyteshash_to_outputtypeindex_commit_handle =
                 scope.spawn(|| self.addressbyteshash_to_outputtypeindex.commit(height));
             let blockhashprefix_to_height_commit_handle =
@@ -306,7 +306,9 @@ impl Stores {
             Ok(())
         })?;
 
-        self.keyspace.persist(PersistMode::SyncAll)
+        self.keyspace
+            .persist(PersistMode::SyncAll)
+            .map_err(|e| e.into())
     }
 
     pub fn rotate_memtables(&self) {
@@ -314,10 +316,4 @@ impl Stores {
         self.blockhashprefix_to_height.rotate_memtable();
         self.txidprefix_to_txindex.rotate_memtable();
     }
-
-    fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
-        fjall::Config::new(path.join("fjall"))
-            .max_write_buffer_size(32 * 1024 * 1024)
-            .open_transactional()
-    }
 }

crates/brk_indexer/src/stores/base.rs

@@ -1,207 +0,0 @@
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    error,
-    fmt::Debug,
-    mem,
-    path::Path,
-};
-
-use brk_core::Height;
-use brk_vec::{Value, Version};
-use byteview::ByteView;
-use fjall::{
-    PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
-    TransactionalPartitionHandle,
-};
-use zerocopy::{Immutable, IntoBytes};
-
-use super::StoreMeta;
-
-pub struct Store<Key, Value> {
-    meta: StoreMeta,
-    name: String,
-    keyspace: TransactionalKeyspace,
-    partition: TransactionalPartitionHandle,
-    rtx: ReadTransaction,
-    puts: BTreeMap<Key, Value>,
-    dels: BTreeSet<Key>,
-}
-
-const CHECK_COLLISISONS: bool = true;
-const MAJOR_FJALL_VERSION: Version = Version::TWO;
-
-impl<K, V> Store<K, V>
-where
-    K: Debug + Clone + Into<ByteView> + Ord + Immutable + IntoBytes,
-    V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
-    <V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
-{
-    pub fn import(
-        keyspace: TransactionalKeyspace,
-        path: &Path,
-        name: &str,
-        version: Version,
-    ) -> color_eyre::Result<Self> {
-        let version = MAJOR_FJALL_VERSION + version;
-
-        let (meta, partition) = StoreMeta::checked_open(
-            &keyspace,
-            &path.join(format!("meta/{name}")),
-            version,
-            || {
-                Self::open_partition_handle(&keyspace, name).inspect_err(|_| {
-                    eprintln!("Delete {path:?} and try again");
-                })
-            },
-        )?;
-
-        let rtx = keyspace.read_tx();
-
-        Ok(Self {
-            meta,
-            name: name.to_owned(),
-            keyspace,
-            partition,
-            rtx,
-            puts: BTreeMap::new(),
-            dels: BTreeSet::new(),
-        })
-    }
-
-    pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
-        if let Some(v) = self.puts.get(key) {
-            Ok(Some(Value::Ref(v)))
-        } else if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes())? {
-            Ok(Some(Value::Owned(V::try_from(slice.as_bytes().into())?)))
-        } else {
-            Ok(None)
-        }
-    }
-
-    pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
-        if self.needs(height) {
-            if !self.dels.is_empty() {
-                // self.dels.remove(&key);
-                unreachable!("Shouldn't reach this");
-            }
-            self.puts.insert(key, value);
-        }
-    }
-
-    pub fn remove(&mut self, key: K) {
-        if self.is_empty() {
-            return;
-        }
-
-        if !self.puts.is_empty() {
-            unreachable!("Shouldn't reach this");
-        }
-
-        if !self.dels.insert(key.clone()) {
-            dbg!(key, &self.meta.path());
-            unreachable!();
-        }
-    }
-
-    pub fn commit(&mut self, height: Height) -> Result<()> {
-        if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
-            return Ok(());
-        }
-
-        self.meta.export(self.len(), height)?;
-
-        let mut wtx = self.keyspace.write_tx();
-
-        mem::take(&mut self.dels)
-            .into_iter()
-            .for_each(|key| wtx.remove(&self.partition, key.as_bytes()));
-
-        mem::take(&mut self.puts)
-            .into_iter()
-            .for_each(|(key, value)| {
-                if CHECK_COLLISISONS {
-                    #[allow(unused_must_use)]
-                    if let Ok(Some(value)) = wtx.get(&self.partition, key.as_bytes()) {
-                        dbg!(
-                            &key,
-                            V::try_from(value.as_bytes().into()).unwrap(),
-                            &self.meta,
-                            self.rtx.get(&self.partition, key.as_bytes())
-                        );
-                        unreachable!();
-                    }
-                }
-                wtx.insert(
-                    &self.partition,
-                    key.as_bytes(),
-                    &*ByteView::try_from(value).unwrap(),
-                )
-            });
-
-        wtx.commit()?;
-
-        self.rtx = self.keyspace.read_tx();
-
-        Ok(())
-    }
-
-    pub fn rotate_memtable(&self) {
-        let _ = self.partition.inner().rotate_memtable();
-    }
-
-    pub fn height(&self) -> Option<Height> {
-        self.meta.height()
-    }
-
-    pub fn len(&self) -> usize {
-        self.meta.len() + self.puts.len() - self.dels.len()
-    }
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    pub fn has(&self, height: Height) -> bool {
-        self.meta.has(height)
-    }
-    pub fn needs(&self, height: Height) -> bool {
-        self.meta.needs(height)
-    }
-
-    fn open_partition_handle(
-        keyspace: &TransactionalKeyspace,
-        name: &str,
-    ) -> Result<TransactionalPartitionHandle> {
-        keyspace.open_partition(
-            name,
-            PartitionCreateOptions::default()
-                .bloom_filter_bits(Some(5))
-                .max_memtable_size(8 * 1024 * 1024)
-                .manual_journal_persist(true),
-        )
-    }
-
-    pub fn reset_partition(&mut self) -> Result<()> {
-        self.keyspace.delete_partition(self.partition.clone())?;
-        self.keyspace.persist(PersistMode::SyncAll)?;
-        self.partition = Self::open_partition_handle(&self.keyspace, &self.name)?;
-        Ok(())
-    }
-}
-
-impl<Key, Value> Clone for Store<Key, Value>
-where
-    Key: Clone,
-    Value: Clone,
-{
-    fn clone(&self) -> Self {
-        Self {
-            meta: self.meta.clone(),
-            name: self.name.clone(),
-            keyspace: self.keyspace.clone(),
-            partition: self.partition.clone(),
-            rtx: self.keyspace.read_tx(),
-            puts: self.puts.clone(),
-            dels: self.dels.clone(),
-        }
-    }
-}

crates/brk_indexer/src/vecs.rs

@@ -4,14 +4,17 @@ use brk_core::{
     AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
     OutputType, OutputTypeIndex, P2ABytes, P2AIndex, P2MSIndex, P2PK33Bytes, P2PK33Index,
     P2PK65Bytes, P2PK65Index, P2PKHBytes, P2PKHIndex, P2SHBytes, P2SHIndex, P2TRBytes, P2TRIndex,
-    P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32,
-    StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight,
+    P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Result, Sats, StoredF64,
+    StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Version,
+    Weight,
 };
-use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec, Result, Version};
+use brk_vec::{AnyCollectableVec, AnyIndexedVec, Format, IndexedVec};
 use rayon::prelude::*;
 
 use crate::Indexes;
 
+const VERSION: Version = Version::ZERO;
+
 #[derive(Clone)]
 pub struct Vecs {
     pub emptyoutputindex_to_txindex: IndexedVec<EmptyOutputIndex, TxIndex>,
@@ -63,234 +66,282 @@ pub struct Vecs {
 }
 
 impl Vecs {
-    pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
+    pub fn forced_import(
+        path: &Path,
+        version: Version,
+        format: Format,
+    ) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
         Ok(Self {
             emptyoutputindex_to_txindex: IndexedVec::forced_import(
-                &path.join("emptyoutputindex_to_txindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "txindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_blockhash: IndexedVec::forced_import(
-                &path.join("height_to_blockhash"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "blockhash",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             height_to_difficulty: IndexedVec::forced_import(
-                &path.join("height_to_difficulty"),
-                Version::ZERO,
-                compressed,
+                path,
+                "difficulty",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_emptyoutputindex: IndexedVec::forced_import(
-                &path.join("height_to_first_emptyoutputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_emptyoutputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_inputindex: IndexedVec::forced_import(
-                &path.join("height_to_first_inputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_inputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_opreturnindex: IndexedVec::forced_import(
-                &path.join("height_to_first_opreturnindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_opreturnindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_outputindex: IndexedVec::forced_import(
-                &path.join("height_to_first_outputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_outputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2aindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2aindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2aindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2msindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2msindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2msindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2pk33index: IndexedVec::forced_import(
-                &path.join("height_to_first_p2pk33index"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2pk33index",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2pk65index: IndexedVec::forced_import(
-                &path.join("height_to_first_p2pk65index"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2pk65index",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2pkhindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2pkhindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2pkhindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2shindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2shindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2shindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2trindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2trindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2trindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2wpkhindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2wpkhindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2wpkhindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_p2wshindex: IndexedVec::forced_import(
-                &path.join("height_to_first_p2wshindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_p2wshindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_txindex: IndexedVec::forced_import(
-                &path.join("height_to_first_txindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_txindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_first_unknownoutputindex: IndexedVec::forced_import(
-                &path.join("height_to_first_unknownoutputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_unknownoutputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_timestamp: IndexedVec::forced_import(
-                &path.join("height_to_timestamp"),
-                Version::ZERO,
-                compressed,
+                path,
+                "timestamp",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_total_size: IndexedVec::forced_import(
-                &path.join("height_to_total_size"),
-                Version::ZERO,
-                compressed,
+                path,
+                "total_size",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             height_to_weight: IndexedVec::forced_import(
-                &path.join("height_to_weight"),
-                Version::ZERO,
-                compressed,
+                path,
+                "weight",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             inputindex_to_outputindex: IndexedVec::forced_import(
-                &path.join("inputindex_to_outputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "outputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             opreturnindex_to_txindex: IndexedVec::forced_import(
-                &path.join("opreturnindex_to_txindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "txindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             outputindex_to_outputtype: IndexedVec::forced_import(
-                &path.join("outputindex_to_outputtype"),
-                Version::ZERO,
-                compressed,
+                path,
+                "outputtype",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             outputindex_to_outputtypeindex: IndexedVec::forced_import(
-                &path.join("outputindex_to_outputtypeindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "outputtypeindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             outputindex_to_value: IndexedVec::forced_import(
-                &path.join("outputindex_to_value"),
-                Version::ZERO,
-                compressed,
+                path,
+                "value",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             p2aindex_to_p2abytes: IndexedVec::forced_import(
-                &path.join("p2aindex_to_p2abytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2abytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2msindex_to_txindex: IndexedVec::forced_import(
-                &path.join("p2msindex_to_txindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "txindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
-                &path.join("p2pk33index_to_p2pk33bytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2pk33bytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2pk65index_to_p2pk65bytes: IndexedVec::forced_import(
-                &path.join("p2pk65index_to_p2pk65bytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2pk65bytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import(
-                &path.join("p2pkhindex_to_p2pkhbytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2pkhbytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2shindex_to_p2shbytes: IndexedVec::forced_import(
-                &path.join("p2shindex_to_p2shbytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2shbytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2trindex_to_p2trbytes: IndexedVec::forced_import(
-                &path.join("p2trindex_to_p2trbytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2trbytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import(
-                &path.join("p2wpkhindex_to_p2wpkhbytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2wpkhbytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             p2wshindex_to_p2wshbytes: IndexedVec::forced_import(
-                &path.join("p2wshindex_to_p2wshbytes"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "p2wshbytes",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             txindex_to_base_size: IndexedVec::forced_import(
-                &path.join("txindex_to_base_size"),
-                Version::ZERO,
-                compressed,
+                path,
+                "base_size",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             txindex_to_first_inputindex: IndexedVec::forced_import(
-                &path.join("txindex_to_first_inputindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "first_inputindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             txindex_to_first_outputindex: IndexedVec::forced_import(
-                &path.join("txindex_to_first_outputindex"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "first_outputindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
-                &path.join("txindex_to_is_explicitly_rbf"),
-                Version::ZERO,
-                compressed,
+                path,
+                "is_explicitly_rbf",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             txindex_to_rawlocktime: IndexedVec::forced_import(
-                &path.join("txindex_to_rawlocktime"),
-                Version::ZERO,
-                compressed,
+                path,
+                "rawlocktime",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             txindex_to_total_size: IndexedVec::forced_import(
-                &path.join("txindex_to_total_size"),
-                Version::ZERO,
-                compressed,
+                path,
+                "total_size",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             txindex_to_txid: IndexedVec::forced_import(
-                &path.join("txindex_to_txid"),
-                Version::ZERO,
-                Compressed::NO,
+                path,
+                "txid",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
             )?,
             txindex_to_txversion: IndexedVec::forced_import(
-                &path.join("txindex_to_txversion"),
-                Version::ZERO,
-                compressed,
+                path,
+                "txversion",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
             unknownoutputindex_to_txindex: IndexedVec::forced_import(
-                &path.join("unknownoutputindex_to_txindex"),
-                Version::ZERO,
-                compressed,
+                path,
+                "txindex",
+                version + VERSION + Version::ZERO,
+                format,
             )?,
         })
     }
 
-    pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
+    pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> Result<()> {
         let saved_height = starting_indexes.height.decremented().unwrap_or_default();
 
         let &Indexes {
@@ -408,7 +459,7 @@ impl Vecs {
         &mut self,
         index: OutputTypeIndex,
         bytes: AddressBytes,
-    ) -> brk_vec::Result<()> {
+    ) -> Result<()> {
         match bytes {
             AddressBytes::P2PK65(bytes) => self
                 .p2pk65index_to_p2pk65bytes

crates/brk_logger/Cargo.toml

@@ -4,6 +4,7 @@ description = "A clean logger used in the Bitcoin Research Kit"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
+homepage.workspace = true
 repository.workspace = true
 
 [dependencies]

crates/brk_logger/README.md

@@ -4,9 +4,6 @@
   <a href="https://github.com/bitcoinresearchkit/brk">
     <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
   </a>
-  <a href="https://kibo.money">
-    <img alt="kibo.money" src="https://img.shields.io/badge/showcase-kib%C5%8D.money-orange">
-  </a>
   <a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
     <img src="https://img.shields.io/crates/l/brk" alt="License" />
   </a>