release: v0.0.81

vec: fix holes export
mcp: use rust-rmcp instead of brk-rmcp
2026-06-26 14:14:30 -07:00 · 2025-07-17 23:45:01 +02:00 · 2025-07-17 17:29:53 +02:00 · 2025-07-17 16:34:29 +02:00 · 2025-07-17 16:17:21 +02:00 · 2025-07-17 11:39:25 +02:00
104 changed files with 2457 additions and 856 deletions
@@ -19,6 +19,9 @@ _*
 # Logs
 .log

+# Environment variables/configs
+.env
+
 # Profiling
 profile.json.gz
 flamegraph.svg
@@ -170,9 +170,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"

 [[package]]
 name = "async-compression"
-version = "0.4.26"
+version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "937f41778d8baa0b8984a101f48ec5d2e5b0d23a3f9235b2066eef29c3472bb9"
+checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8"
 dependencies = [
 "brotli",
 "flate2",
@@ -477,7 +477,7 @@ dependencies = [

 [[package]]
 name = "brk"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_bundler",
 "brk_cli",
@@ -497,7 +497,7 @@ dependencies = [

 [[package]]
 name = "brk_bundler"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_rolldown",
 "log",
@@ -508,7 +508,7 @@ dependencies = [

 [[package]]
 name = "brk_cli"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "bitcoincore-rpc",
 "brk_computer",
@@ -531,7 +531,7 @@ dependencies = [

 [[package]]
 name = "brk_computer"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -557,7 +557,7 @@ dependencies = [

 [[package]]
 name = "brk_core"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -578,7 +578,7 @@ dependencies = [

 [[package]]
 name = "brk_exit"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_logger",
 "ctrlc",
@@ -587,7 +587,7 @@ dependencies = [

 [[package]]
 name = "brk_fetcher"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_core",
 "brk_logger",
@@ -599,7 +599,7 @@ dependencies = [

 [[package]]
 name = "brk_indexer"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -617,15 +617,15 @@ dependencies = [

 [[package]]
 name = "brk_interface"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_computer",
 "brk_core",
 "brk_indexer",
- "brk_rmcp",
 "brk_vec",
 "color-eyre",
 "derive_deref",
+ "rmcp",
 "schemars 1.0.4",
 "serde",
 "serde_json",
@@ -635,7 +635,7 @@ dependencies = [

 [[package]]
 name = "brk_logger"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "color-eyre",
 "env_logger",
@@ -645,17 +645,17 @@ dependencies = [

 [[package]]
 name = "brk_mcp"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "axum",
 "brk_interface",
- "brk_rmcp",
 "log",
+ "rmcp",
 ]

 [[package]]
 name = "brk_parser"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -668,49 +668,6 @@ dependencies = [
 "zerocopy",
 ]

-[[package]]
-name = "brk_rmcp"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dde96460b07f97f8522cef62fa5e9d5271a5183b13e6d75379ab43c1fb0b106"
-dependencies = [
- "base64 0.22.1",
- "brk_rmcp-macros",
- "bytes",
- "chrono",
- "futures",
- "http",
- "http-body",
- "http-body-util",
- "paste",
- "pin-project-lite",
- "rand 0.9.1",
- "schemars 1.0.4",
- "serde",
- "serde_json",
- "sse-stream",
- "thiserror 2.0.12",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tower-service",
- "tracing",
- "uuid",
-]
-
-[[package]]
-name = "brk_rmcp-macros"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48a3cb1aadff2f3aaaed76d17e71d6b828cafbd3fe9fbaba07d31b8ed10e63c2"
-dependencies = [
- "darling",
- "proc-macro2",
- "quote",
- "serde_json",
- "syn 2.0.104",
-]
-
 [[package]]
 name = "brk_rolldown"
 version = "0.1.1"
@@ -1024,7 +981,7 @@ dependencies = [

 [[package]]
 name = "brk_server"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "axum",
 "bitcoincore-rpc",
@@ -1046,6 +1003,7 @@ dependencies = [
 "log",
 "minreq",
 "serde",
+ "serde_json",
 "tokio",
 "tower-http",
 "tracing",
@@ -1054,7 +1012,7 @@ dependencies = [

 [[package]]
 name = "brk_store"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "brk_core",
 "byteview",
@@ -1076,7 +1034,7 @@ dependencies = [

 [[package]]
 name = "brk_vec"
-version = "0.0.80"
+version = "0.0.81"
 dependencies = [
 "arc-swap",
 "brk_core",
@@ -1478,8 +1436,18 @@ version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
 dependencies = [
- "darling_core",
- "darling_macro",
+ "darling_core 0.20.11",
+ "darling_macro 0.20.11",
+]
+
+[[package]]
+name = "darling"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a79c4acb1fd5fa3d9304be4c76e031c54d2e92d172a393e24b19a14fe8532fe9"
+dependencies = [
+ "darling_core 0.21.0",
+ "darling_macro 0.21.0",
 ]

 [[package]]
@@ -1496,13 +1464,38 @@ dependencies = [
 "syn 2.0.104",
 ]

+[[package]]
+name = "darling_core"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74875de90daf30eb59609910b84d4d368103aaec4c924824c6799b28f77d6a1d"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.11",
+ "quote",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e79f8e61677d5df9167cd85265f8e5f64b215cdea3fb55eebc3e622e44c7a146"
+dependencies = [
+ "darling_core 0.21.0",
 "quote",
 "syn 2.0.104",
 ]
@@ -3583,6 +3576,49 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "rmcp"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b008b927a85d514699ff304be84c5084598596e6cad4a6f5bc67207715fafe5f"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "chrono",
+ "futures",
+ "http",
+ "http-body",
+ "http-body-util",
+ "paste",
+ "pin-project-lite",
+ "rand 0.9.1",
+ "rmcp-macros",
+ "schemars 1.0.4",
+ "serde",
+ "serde_json",
+ "sse-stream",
+ "thiserror 2.0.12",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower-service",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "rmcp-macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7465280d5f73f2c5c99017a04af407b2262455a149f255ad22f2b0b29087695c"
+dependencies = [
+ "darling 0.21.0",
+ "proc-macro2",
+ "quote",
+ "serde_json",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "rolldown-ariadne"
 version = "0.5.2"
@@ -3617,15 +3653,15 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"

 [[package]]
 name = "rustix"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
 dependencies = [
 "bitflags 2.9.1",
 "errno",
 "libc",
 "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]

 [[package]]
@@ -3695,6 +3731,7 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
 dependencies = [
+ "chrono",
 "dyn-clone",
 "ref-cast",
 "schemars_derive",
@@ -3883,7 +3920,7 @@ version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
 dependencies = [
- "darling",
+ "darling 0.20.11",
 "proc-macro2",
 "quote",
 "syn 2.0.104",
@@ -4,9 +4,10 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.80"
+package.version = "0.0.81"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
+package.readme = "README.md"

 [profile.release]
 lto = "fat"
@@ -26,22 +27,20 @@ axum = "0.8.4"
 bincode = { version = "2.0.1", features = ["serde"] }
 bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_bundler = { version = "0.0.80", path = "crates/brk_bundler" }
-brk_cli = { version = "0.0.80", path = "crates/brk_cli" }
-brk_computer = { version = "0.0.80", path = "crates/brk_computer" }
-brk_core = { version = "0.0.80", path = "crates/brk_core" }
-brk_exit = { version = "0.0.80", path = "crates/brk_exit" }
-brk_fetcher = { version = "0.0.80", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0.0.80", path = "crates/brk_indexer" }
-brk_interface = { version = "0.0.80", path = "crates/brk_interface" }
-brk_logger = { version = "0.0.80", path = "crates/brk_logger" }
-brk_mcp = { version = "0.0.80", path = "crates/brk_mcp" }
-brk_parser = { version = "0.0.80", path = "crates/brk_parser" }
-brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
-# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
-brk_server = { version = "0.0.80", path = "crates/brk_server" }
-brk_store = { version = "0.0.80", path = "crates/brk_store" }
-brk_vec = { version = "0.0.80", path = "crates/brk_vec" }
+brk_bundler = { version = "0.0.81", path = "crates/brk_bundler" }
+brk_cli = { version = "0.0.81", path = "crates/brk_cli" }
+brk_computer = { version = "0.0.81", path = "crates/brk_computer" }
+brk_core = { version = "0.0.81", path = "crates/brk_core" }
+brk_exit = { version = "0.0.81", path = "crates/brk_exit" }
+brk_fetcher = { version = "0.0.81", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0.0.81", path = "crates/brk_indexer" }
+brk_interface = { version = "0.0.81", path = "crates/brk_interface" }
+brk_logger = { version = "0.0.81", path = "crates/brk_logger" }
+brk_mcp = { version = "0.0.81", path = "crates/brk_mcp" }
+brk_parser = { version = "0.0.81", path = "crates/brk_parser" }
+brk_server = { version = "0.0.81", path = "crates/brk_server" }
+brk_store = { version = "0.0.81", path = "crates/brk_store" }
+brk_vec = { version = "0.0.81", path = "crates/brk_vec" }
 byteview = "=0.6.1"
 clap = { version = "4.5.41", features = ["string"] }
 clap_derive = "4.5.41"
@@ -52,6 +51,7 @@ jiff = "0.2.15"
 log = { version = "0.4.27" }
 minreq = { version = "2.14.0", features = ["https", "serde_json"] }
 rayon = "1.10.0"
+rmcp = { version = "0.3.0", features = ["transport-worker", "transport-streamable-http-server" ] }
 schemars = "1.0.4"
 serde = { version = "1.0.219" }
 serde_bytes = "0.11.17"
@@ -40,7 +40,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible
 - **[Website](https://bitcoinresearchkit.org)** \
  Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
  It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
-  Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
+  Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
 - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
  Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
  Just like the website, it's entirely free, with no authentication or rate-limiting.
@@ -73,7 +73,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
 - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
 - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
 - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
+- [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec
 - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)

 ## Hosting as a service
@@ -10,8 +10,12 @@
    - pull latest version and notify is out of date
  - _computer_
    - **add rollback of states (in stateful)**
+    - remove configurable format (raw/compressed) and chose sane ones instead
+      - linear reads: compressed (height/date/... + txindex_to_height + txindex_to_version + ...)
+      - random reads: raw (outputindex_to_value + ...)
+    - add prices paid by percentile (percentile cost basis) back
    - add support for per index computation
-    - fix feerate which is always ZERO due to coinbase transaction
+    - fix min feerate which is always ZERO due to coinbase transaction
    - before computing multiple sources check their length, panic if not equal
    - add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
    - add address counts relative to all datasets
@@ -56,6 +60,7 @@
      - add extensions support (.json .csv …)
      - if format instead of extension then don't download file
    - add support for https (rustls)
+    - lru cache
  - _vec_
    - add native lock file support (once it's available in stable rust)
    - improve compressed mode (slow reads)
@@ -1,2 +1 @@
-cargo build --profile profiling
-flamegraph -- ../../target/profiling/brk
+sudo cargo flamegraph --profile profiling --root
@@ -13,7 +13,6 @@ use clap_derive::Parser;
 use color_eyre::eyre::eyre;
 use serde::{Deserialize, Serialize};

-use crate::services::Services;

 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
 #[command(version, about)]
@@ -33,10 +32,6 @@ pub struct Config {
    #[arg(long, value_name = "PATH")]
    brkdir: Option<String>,

-    /// Activated services, default: all, saved
-    #[serde(default, deserialize_with = "default_on_error")]
-    #[arg(short, long)]
-    services: Option<Services>,

    /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
    #[serde(default, deserialize_with = "default_on_error")]
@@ -129,9 +124,6 @@ impl Config {
                config_saved.brkdir = Some(brkdir);
            }

-            if let Some(services) = config_args.services.take() {
-                config_saved.services = Some(services);
-            }

            if let Some(computation) = config_args.computation.take() {
                config_saved.computation = Some(computation);
@@ -306,15 +298,6 @@ impl Config {
        self.outputsdir().join("hars")
    }

-    pub fn process(&self) -> bool {
-        self.services
-            .is_none_or(|m| m == Services::All || m == Services::Processor)
-    }
-
-    pub fn serve(&self) -> bool {
-        self.services
-            .is_none_or(|m| m == Services::All || m == Services::Server)
-    }

    fn path_cookiefile(&self) -> PathBuf {
        self.rpccookiefile.as_ref().map_or_else(
@@ -1,10 +1,11 @@
+#![doc = include_str!("../README.md")]
+
 use std::{fs, thread};

 use brk_core::{dot_brk_log_path, dot_brk_path};

 mod config;
 mod run;
-mod services;

 use run::*;

@@ -13,18 +13,16 @@ pub fn run() -> color_eyre::Result<()> {
    let config = Config::import()?;

    let rpc = config.rpc()?;
-
    let exit = Exit::new();
-
-    let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
+    let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);

    let format = config.format();

    let mut indexer = Indexer::forced_import(&config.outputsdir())?;

-    let wait_for_synced_node = || -> color_eyre::Result<()> {
+    let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
        let is_synced = || -> color_eyre::Result<bool> {
-            let info = rpc.get_blockchain_info()?;
+            let info = rpc_client.get_blockchain_info()?;
            Ok(info.headers == info.blocks)
        };

@@ -50,54 +48,41 @@ pub fn run() -> color_eyre::Result<()> {
        .enable_all()
        .build()?
        .block_on(async {
-            let server = if config.serve() {
-                let served_indexer = indexer.clone();
-                let served_computer = computer.clone();
+            let served_indexer = indexer.clone();
+            let served_computer = computer.clone();

-                let server = Server::new(served_indexer, served_computer, config.website())?;
+            let server = Server::new(served_indexer, served_computer, config.website())?;

-                let watch = config.watch();
-                let mcp = config.mcp();
-                let opt = Some(tokio::spawn(async move {
-                    server.serve(watch, mcp).await.unwrap();
-                }));
+            let watch = config.watch();
+            let mcp = config.mcp();

-                sleep(Duration::from_secs(1));
+            tokio::spawn(async move {
+                server.serve(watch, mcp).await.unwrap();
+            });

-                opt
-            } else {
-                None
-            };
+            sleep(Duration::from_secs(1));

-            if config.process() {
-                loop {
-                    wait_for_synced_node()?;
+            loop {
+                wait_for_synced_node(rpc)?;

-                    let block_count = rpc.get_block_count()?;
+                let block_count = rpc.get_block_count()?;

-                    info!("{} blocks found.", block_count + 1);
+                info!("{} blocks found.", block_count + 1);

-                    let starting_indexes =
-                        indexer.index(&parser, rpc, &exit, config.check_collisions())?;
+                let starting_indexes =
+                    indexer.index(&parser, rpc, &exit, config.check_collisions())?;

-                    computer.compute(&mut indexer, starting_indexes, &exit)?;
+                computer.compute(&mut indexer, starting_indexes, &exit)?;

-                    if let Some(delay) = config.delay() {
-                        sleep(Duration::from_secs(delay))
-                    }
+                if let Some(delay) = config.delay() {
+                    sleep(Duration::from_secs(delay))
+                }

-                    info!("Waiting for new blocks...");
+                info!("Waiting for new blocks...");

-                    while block_count == rpc.get_block_count()? {
-                        sleep(Duration::from_secs(1))
-                    }
+                while block_count == rpc.get_block_count()? {
+                    sleep(Duration::from_secs(1))
                }
            }
-
-            if let Some(handle) = server {
-                handle.await.unwrap();
-            }
-
-            Ok(())
        })
 }
@@ -1,23 +0,0 @@
-use clap_derive::{Parser, ValueEnum};
-use serde::{Deserialize, Serialize};
-
-#[derive(
-    Default,
-    Debug,
-    Clone,
-    Copy,
-    Parser,
-    ValueEnum,
-    Serialize,
-    Deserialize,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-)]
-pub enum Services {
-    #[default]
-    All,
-    Processor,
-    Server,
-}
@@ -1,7 +1,6 @@
 use std::{path::Path, thread};

 use brk_computer::Computer;
-use brk_core::{default_bitcoin_path, default_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -13,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {

    brk_logger::init(Some(Path::new(".log")));

-    // let bitcoin_dir = default_bitcoin_path();
+    // let bitcoin_dir = brk_core::default_bitcoin_path();
    let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
@@ -26,7 +25,11 @@ pub fn main() -> color_eyre::Result<()> {
    thread::Builder::new()
        .stack_size(256 * 1024 * 1024)
        .spawn(move || -> color_eyre::Result<()> {
-            let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+            let parser = Parser::new(
+                bitcoin_dir.join("blocks"),
+                brk_core::default_brk_path(),
+                rpc,
+            );

            let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
            let outputs_dir = _outputs_dir.as_path();
@@ -5,22 +5,11 @@ use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format};
-
-pub mod blocks;
-pub mod cointime;
-pub mod constants;
-pub mod fetched;
-pub mod grouped;
-pub mod indexes;
-pub mod market;
-pub mod mining;
-pub mod stateful;
-pub mod transactions;
-
-pub use indexes::Indexes;
 use log::info;

-use crate::stores::Stores;
+use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
+
+use super::stateful;

 const VERSION: Version = Version::ONE;

@@ -130,7 +119,6 @@ impl Vecs {
        starting_indexes: brk_indexer::Indexes,
        fetcher: Option<&mut Fetcher>,
        exit: &Exit,
-        stores: &mut Stores,
    ) -> color_eyre::Result<()> {
        info!("Computing indexes...");
        let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
@@ -188,7 +176,6 @@ impl Vecs {
            &self.market,
            &mut starting_indexes,
            exit,
-            stores,
        )?;

        self.cointime.compute(
@@ -8,7 +8,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -5,16 +5,13 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};

-use crate::vecs::{
-    fetched,
-    grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source},
-    stateful, transactions,
-};
-
 use super::{
-    Indexes,
-    grouped::{ComputedVecsFromHeight, VecBuilderOptions},
-    indexes,
+    Indexes, fetched,
+    grouped::{
+        ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
+        Source, VecBuilderOptions,
+    },
+    indexes, stateful, transactions,
 };

 const VERSION: Version = Version::ZERO;
@@ -305,7 +302,7 @@ impl Vecs {
        stateful: &stateful::Vecs,
        exit: &Exit,
    ) -> color_eyre::Result<()> {
-        let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply;
+        let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;

        self.indexes_to_coinblocks_created.compute_all(
            indexer,
@@ -323,7 +320,7 @@ impl Vecs {
        )?;

        let indexes_to_coinblocks_destroyed =
-            &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed;
+            &stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed;

        self.indexes_to_coinblocks_stored.compute_all(
            indexer,
@@ -433,7 +430,7 @@ impl Vecs {

        if let Some(fetched) = fetched {
            let realized_cap = stateful
-                .utxo_vecs
+                .utxo_cohorts
                .all
                .1
                .height_to_realized_cap
@@ -441,7 +438,7 @@ impl Vecs {
                .unwrap();

            let realized_price = stateful
-                .utxo_vecs
+                .utxo_cohorts
                .all
                .1
                .indexes_to_realized_price
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -13,7 +13,7 @@ use brk_vec::{
    VecIterator,
 };

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -7,7 +7,7 @@ use brk_vec::{
    ComputedVec, ComputedVecFrom2, Format, StoredIndex,
 };

-use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions};
+use crate::grouped::{EagerVecBuilder, VecBuilderOptions};

 use super::ComputedType;

@@ -10,7 +10,7 @@ use brk_vec::{
    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };

-use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes};
+use crate::{Indexes, grouped::ComputedVecBuilder, indexes};

 use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};

@@ -10,7 +10,7 @@ use brk_vec::{
    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };

-use crate::vecs::{
+use crate::{
    Indexes,
    grouped::{ComputedVecBuilder, Source},
    indexes,
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, EagerVec, Format};

-use crate::vecs::{Indexes, indexes};
+use crate::{Indexes, indexes};

 use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};

@@ -11,7 +11,7 @@ use brk_vec::{
    Format, StoredIndex, VecIterator,
 };

-use crate::vecs::{
+use crate::{
    Indexes, fetched,
    grouped::{ComputedVecBuilder, Source},
    indexes,
@@ -8,10 +8,7 @@ use brk_vec::{
    StoredIndex, VecIterator,
 };

-use crate::{
-    utils::get_percentile,
-    vecs::{Indexes, fetched, grouped::source::Source, indexes},
-};
+use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile};

 use super::{ComputedVecsFromDateIndex, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
+use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};

 use super::{Source, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 use super::{ComputedVecsFromHeight, VecBuilderOptions};

@@ -8,7 +8,7 @@ use brk_vec::{
    Format, LazyVecFrom1, StoredIndex, StoredVec,
 };

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 use super::{ComputedVecsFromTxindex, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 #[derive(Clone)]
 pub struct ComputedHeightValueVecs {
@@ -15,8 +15,6 @@ use brk_vec::{
    ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
 };

-use crate::vecs::indexes;
-
 const VERSION: Version = Version::ZERO;

 #[derive(Clone)]
@@ -1206,7 +1204,7 @@ pub struct Indexes {
 }

 impl Indexes {
-    pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) {
+    pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
        self.indexes.height = height;
        self.dateindex = DateIndex::try_from(
            indexes
@@ -12,20 +12,28 @@ use brk_indexer::Indexer;
 use brk_vec::{Computation, Format};
 use log::info;

+mod all;
+mod blocks;
+mod cointime;
+mod constants;
+mod fetched;
+mod grouped;
+mod indexes;
+mod market;
+mod mining;
+mod stateful;
 mod states;
-mod stores;
+mod transactions;
 mod utils;
-mod vecs;
+
+use indexes::Indexes;

 use states::*;
-use stores::Stores;
-use vecs::Vecs;

 #[derive(Clone)]
 pub struct Computer {
    fetcher: Option<Fetcher>,
-    pub vecs: Vecs,
-    pub stores: Stores,
+    pub vecs: all::Vecs,
 }

 const VERSION: Version = Version::ONE;
@@ -40,7 +48,7 @@ impl Computer {
        format: Format,
    ) -> color_eyre::Result<Self> {
        Ok(Self {
-            vecs: Vecs::import(
+            vecs: all::Vecs::import(
                // TODO: Give self.path, join inside import
                &outputs_dir.join("vecs/computed"),
                VERSION + Version::ZERO,
@@ -49,12 +57,6 @@ impl Computer {
                computation,
                format,
            )?,
-            stores: Stores::import(
-                // TODO: Give self.path, join inside import
-                &outputs_dir.join("stores"),
-                VERSION + Version::ZERO,
-                &indexer.stores.keyspace,
-            )?,
            fetcher,
        })
    }
@@ -68,12 +70,7 @@ impl Computer {
        exit: &Exit,
    ) -> color_eyre::Result<()> {
        info!("Computing...");
-        self.vecs.compute(
-            indexer,
-            starting_indexes,
-            self.fetcher.as_mut(),
-            exit,
-            &mut self.stores,
-        )
+        self.vecs
+            .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
    }
 }
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes, fetched,
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -8,16 +8,14 @@ use brk_vec::{
 };

 use crate::{
-    states::AddressCohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
-        indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, fetched,
+    grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
+    indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
    },
+    states::AddressCohortState,
 };

 const VERSION: Version = Version::ZERO;
@@ -9,7 +9,7 @@ use brk_vec::{Computation, Format};
 use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;

-use crate::vecs::{
+use crate::{
    Indexes, fetched, indexes,
    stateful::{
        address_cohort,
@@ -2,7 +2,7 @@ use brk_core::{ByAddressType, Height};
 use brk_vec::VecIterator;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
+use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;

 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct AddressTypeToAddressCount(ByAddressType<usize>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::EagerVec;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
+use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;

 #[derive(Debug, Clone, Deref, DerefMut)]
 pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::AnyCollectableVec;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::{
+use crate::{
    Indexes, grouped::ComputedVecsFromHeight, indexes,
    stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
 };
@@ -0,0 +1,24 @@
+use std::collections::BTreeSet;
+
+use brk_core::TypeIndex;
+use derive_deref::{Deref, DerefMut};
+
+use super::ByAddressType;
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
+
+impl Default for AddressTypeToTypeIndexSet {
+    fn default() -> Self {
+        Self(ByAddressType {
+            p2pk65: BTreeSet::default(),
+            p2pk33: BTreeSet::default(),
+            p2pkh: BTreeSet::default(),
+            p2sh: BTreeSet::default(),
+            p2wpkh: BTreeSet::default(),
+            p2wsh: BTreeSet::default(),
+            p2tr: BTreeSet::default(),
+            p2a: BTreeSet::default(),
+        })
+    }
+}
@@ -29,10 +29,6 @@ impl<T> AddressTypeToTypeIndexTree<T> {
            mem::swap(own, other);
        }
    }
-
-    pub fn unwrap(self) -> ByAddressType<BTreeMap<TypeIndex, T>> {
-        self.0
-    }
 }

 impl<T> Default for AddressTypeToTypeIndexTree<T> {
@@ -0,0 +1,57 @@
+use std::mem;
+
+use derive_deref::{Deref, DerefMut};
+
+use super::ByAddressType;
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
+
+impl<T> AddressTypeToVec<T> {
+    pub fn merge(mut self, mut other: Self) -> Self {
+        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
+        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
+        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
+        Self::merge_(&mut self.p2sh, &mut other.p2sh);
+        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
+        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
+        Self::merge_(&mut self.p2tr, &mut other.p2tr);
+        Self::merge_(&mut self.p2a, &mut other.p2a);
+        self
+    }
+
+    pub fn merge_mut(&mut self, mut other: Self) {
+        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
+        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
+        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
+        Self::merge_(&mut self.p2sh, &mut other.p2sh);
+        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
+        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
+        Self::merge_(&mut self.p2tr, &mut other.p2tr);
+        Self::merge_(&mut self.p2a, &mut other.p2a);
+    }
+
+    fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
+        if own.len() >= other.len() {
+            own.append(other);
+        } else {
+            other.append(own);
+            mem::swap(own, other);
+        }
+    }
+}
+
+impl<T> Default for AddressTypeToVec<T> {
+    fn default() -> Self {
+        Self(ByAddressType {
+            p2pk65: vec![],
+            p2pk33: vec![],
+            p2pkh: vec![],
+            p2sh: vec![],
+            p2wpkh: vec![],
+            p2wsh: vec![],
+            p2tr: vec![],
+            p2a: vec![],
+        })
+    }
+}
@@ -11,16 +11,13 @@ use brk_vec::{
 };

 use crate::{
-    states::CohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{
-            ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex,
-            ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
-            Source, VecBuilderOptions,
-        },
-        indexes, market,
+    Indexes, fetched,
+    grouped::{
+        ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
+        ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions,
    },
+    indexes, market,
+    states::CohortState,
 };

 const VERSION: Version = Version::ZERO;
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use brk_core::Height;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::AddressTypeToVec;
+use crate::stateful::AddressTypeToVec;

 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec};

-use crate::vecs::{Indexes, fetched, indexes, market};
+use crate::{Indexes, fetched, indexes, market};

 pub trait DynCohortVecs: Send + Sync {
    fn starting_height(&self) -> Height;
@@ -6,13 +6,10 @@ use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};

 use crate::{
-    UTXOCohortState,
-    vecs::{
-        Indexes, fetched, indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, UTXOCohortState, fetched, indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
    },
 };

@@ -11,8 +11,9 @@ use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;

 use crate::{
+    Indexes, fetched, indexes,
+    stateful::r#trait::DynCohortVecs,
    states::{BlockState, Transacted},
-    vecs::{Indexes, fetched, indexes, stateful::r#trait::DynCohortVecs},
 };

 use super::{r#trait::CohortVecs, utxo_cohort};
@@ -1505,13 +1506,16 @@ impl Vecs {
        let last_timestamp = chain_state.last().unwrap().timestamp;
        let current_price = chain_state.last().unwrap().price;

+        let chain_state_len = chain_state.len();
+
        height_to_sent.into_iter().for_each(|(height, sent)| {
            chain_state[height.unwrap_to_usize()].supply -= &sent.spendable_supply;

            let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
+
            let prev_price = block_state.price;

-            let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize();
+            let blocks_old = chain_state_len - 1 - height.unwrap_to_usize();

            let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
            let days_old_float =
@@ -0,0 +1,54 @@
+use brk_core::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
+
+#[derive(Debug)]
+pub enum WithAddressDataSource<T> {
+    New(T),
+    FromLoadedAddressDataVec((LoadedAddressIndex, T)),
+    FromEmptyAddressDataVec((EmptyAddressIndex, T)),
+}
+
+impl<T> WithAddressDataSource<T> {
+    pub fn is_new(&self) -> bool {
+        matches!(self, Self::New(_))
+    }
+
+    pub fn is_from_emptyaddressdata(&self) -> bool {
+        matches!(self, Self::FromEmptyAddressDataVec(_))
+    }
+
+    pub fn deref_mut(&mut self) -> &mut T {
+        match self {
+            Self::New(v) => v,
+            Self::FromLoadedAddressDataVec((_, v)) => v,
+            Self::FromEmptyAddressDataVec((_, v)) => v,
+        }
+    }
+}
+
+impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
+    fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
+        match value {
+            WithAddressDataSource::New(v) => Self::New(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
+                Self::FromLoadedAddressDataVec((i, v.into()))
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
+                Self::FromEmptyAddressDataVec((i, v.into()))
+            }
+        }
+    }
+}
+
+impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
+    fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
+        match value {
+            WithAddressDataSource::New(v) => Self::New(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
+                Self::FromLoadedAddressDataVec((i, v.into()))
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
+                Self::FromEmptyAddressDataVec((i, v.into()))
+            }
+        }
+    }
+}
@@ -1,6 +1,6 @@
 use std::path::Path;

-use brk_core::{AddressData, Dollars, Height, Result, Sats};
+use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats};

 use crate::SupplyState;

@@ -35,7 +35,7 @@ impl AddressCohortState {
    #[allow(clippy::too_many_arguments)]
    pub fn send(
        &mut self,
-        addressdata: &mut AddressData,
+        addressdata: &mut LoadedAddressData,
        value: Sats,
        current_price: Option<Dollars>,
        prev_price: Option<Dollars>,
@@ -72,7 +72,12 @@ impl AddressCohortState {
        Ok(())
    }

-    pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
+    pub fn receive(
+        &mut self,
+        address_data: &mut LoadedAddressData,
+        value: Sats,
+        price: Option<Dollars>,
+    ) {
        let compute_price = price.is_some();

        let prev_realized_price = compute_price.then(|| address_data.realized_price());
@@ -96,7 +101,7 @@ impl AddressCohortState {
        );
    }

-    pub fn add(&mut self, addressdata: &AddressData) {
+    pub fn add(&mut self, addressdata: &LoadedAddressData) {
        self.address_count += 1;
        self.inner.increment_(
            &addressdata.into(),
@@ -105,7 +110,7 @@ impl AddressCohortState {
        );
    }

-    pub fn subtract(&mut self, addressdata: &AddressData) {
+    pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
        self.address_count = self.address_count.checked_sub(1).unwrap();
        self.inner.decrement_(
            &addressdata.into(),
@@ -223,17 +223,15 @@ impl CohortState {
        let update_state =
            |price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
                match price.cmp(&current_price) {
-                    Ordering::Equal => {
-                        state.supply_even += sats;
-                    }
                    Ordering::Less => {
                        state.supply_in_profit += sats;
                        if price > Dollars::ZERO && current_price > Dollars::ZERO {
                            let diff = current_price.checked_sub(price).unwrap();
-                            if diff <= Dollars::ZERO {
-                                dbg!(price, current_price, diff, sats);
-                                panic!();
-                            }
+                            // Add back once in a while to verify, but generally not needed
+                            // if diff <= Dollars::ZERO {
+                            //     dbg!(price, current_price, diff, sats);
+                            //     panic!();
+                            // }
                            state.unrealized_profit += diff * sats;
                        }
                    }
@@ -241,13 +239,17 @@ impl CohortState {
                        state.supply_in_loss += sats;
                        if price > Dollars::ZERO && current_price > Dollars::ZERO {
                            let diff = price.checked_sub(current_price).unwrap();
-                            if diff <= Dollars::ZERO {
-                                dbg!(price, current_price, diff, sats);
-                                panic!();
-                            }
+                            // Add back once in a while to verify, but generally not needed
+                            // if diff <= Dollars::ZERO {
+                            //     dbg!(price, current_price, diff, sats);
+                            //     panic!();
+                            // }
                            state.unrealized_loss += diff * sats;
                        }
                    }
+                    Ordering::Equal => {
+                        state.supply_even += sats;
+                    }
                }
            };

@@ -1,6 +1,6 @@
 use std::ops::{Add, AddAssign, SubAssign};

-use brk_core::{AddressData, CheckedSub, Sats};
+use brk_core::{CheckedSub, LoadedAddressData, Sats};
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

@@ -40,8 +40,8 @@ impl SubAssign<&SupplyState> for SupplyState {
    }
 }

-impl From<&AddressData> for SupplyState {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for SupplyState {
+    fn from(value: &LoadedAddressData) -> Self {
        Self {
            utxos: value.outputs_len as usize,
            value: value.amount(),
@@ -9,7 +9,7 @@ use brk_store::{AnyStore, Store};
 use fjall::{PersistMode, TransactionalKeyspace};
 use log::info;

-use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
+use crate::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};

 const VERSION: Version = Version::ZERO;

@@ -597,12 +597,6 @@ impl Stores {
            .map_err(|e| e.into())
    }

-    pub fn rotate_memtables(&self) {
-        self.as_slice()
-            .into_iter()
-            .for_each(|store| store.rotate_memtable());
-    }
-
    pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
        [
            &self.p2aaddressindex_to_addressdata,
@@ -11,17 +11,13 @@ use brk_vec::{
    ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
 };

-use crate::vecs::grouped::Source;
-
-use super::{
-    Indexes, fetched,
-    grouped::{
-        ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
-        ComputedVecsFromTxindex, VecBuilderOptions,
-    },
-    indexes,
+use crate::grouped::{
+    ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
+    ComputedVecsFromTxindex, Source, VecBuilderOptions,
 };

+use super::{Indexes, fetched, indexes};
+
 const VERSION: Version = Version::ZERO;

 #[derive(Clone)]
@@ -1,47 +0,0 @@
-use std::{collections::BTreeSet, mem};
-
-use brk_core::TypeIndex;
-use derive_deref::{Deref, DerefMut};
-
-use super::ByAddressType;
-
-#[derive(Debug, Deref, DerefMut)]
-pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
-
-impl AddressTypeToTypeIndexSet {
-    pub fn merge(mut self, mut other: Self) -> Self {
-        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
-        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
-        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
-        Self::merge_(&mut self.p2sh, &mut other.p2sh);
-        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
-        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
-        Self::merge_(&mut self.p2tr, &mut other.p2tr);
-        Self::merge_(&mut self.p2a, &mut other.p2a);
-        self
-    }
-
-    fn merge_(own: &mut BTreeSet<TypeIndex>, other: &mut BTreeSet<TypeIndex>) {
-        if own.len() >= other.len() {
-            own.append(other);
-        } else {
-            other.append(own);
-            mem::swap(own, other);
-        }
-    }
-}
-
-impl Default for AddressTypeToTypeIndexSet {
-    fn default() -> Self {
-        Self(ByAddressType {
-            p2pk65: BTreeSet::default(),
-            p2pk33: BTreeSet::default(),
-            p2pkh: BTreeSet::default(),
-            p2sh: BTreeSet::default(),
-            p2wpkh: BTreeSet::default(),
-            p2wsh: BTreeSet::default(),
-            p2tr: BTreeSet::default(),
-            p2a: BTreeSet::default(),
-        })
-    }
-}
@@ -1,21 +0,0 @@
-use derive_deref::{Deref, DerefMut};
-
-use super::ByAddressType;
-
-#[derive(Debug, Deref, DerefMut)]
-pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
-
-impl<T> Default for AddressTypeToVec<T> {
-    fn default() -> Self {
-        Self(ByAddressType {
-            p2pk65: vec![],
-            p2pk33: vec![],
-            p2pkh: vec![],
-            p2sh: vec![],
-            p2wpkh: vec![],
-            p2wsh: vec![],
-            p2tr: vec![],
-            p2a: vec![],
-        })
-    }
-}
@@ -1,62 +0,0 @@
-use brk_core::{AddressData, EmptyAddressData};
-
-#[derive(Debug)]
-pub enum WithAddressDataSource<T> {
-    New(T),
-    FromAddressDataStore(T),
-    FromEmptyAddressDataStore(T),
-}
-
-impl<T> WithAddressDataSource<T> {
-    pub fn is_new(&self) -> bool {
-        matches!(self, Self::New(_))
-    }
-
-    pub fn is_from_addressdata(&self) -> bool {
-        matches!(self, Self::FromAddressDataStore(_))
-    }
-
-    pub fn is_from_emptyaddressdata(&self) -> bool {
-        matches!(self, Self::FromEmptyAddressDataStore(_))
-    }
-
-    pub fn deref(&self) -> &T {
-        match self {
-            Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
-        }
-    }
-
-    pub fn deref_mut(&mut self) -> &mut T {
-        match self {
-            Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
-        }
-    }
-}
-
-impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<AddressData> {
-    fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
-        match value {
-            WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
-                Self::FromEmptyAddressDataStore(v.into())
-            }
-        }
-    }
-}
-
-impl From<WithAddressDataSource<AddressData>> for WithAddressDataSource<EmptyAddressData> {
-    fn from(value: WithAddressDataSource<AddressData>) -> Self {
-        match value {
-            WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
-                Self::FromEmptyAddressDataStore(v.into())
-            }
-        }
-    }
-}
@@ -19,6 +19,10 @@ pub struct ByAddressType<T> {
 }

 impl<T> ByAddressType<T> {
+    pub fn get_unwrap(&self, address_type: OutputType) -> &T {
+        self.get(address_type).unwrap()
+    }
+
    pub fn get(&self, address_type: OutputType) -> Option<&T> {
        match address_type {
            OutputType::P2PK65 => Some(&self.p2pk65),
@@ -0,0 +1,5 @@
+#[derive(Debug, Default)]
+pub struct ByAnyAddress<T> {
+    pub loaded: T,
+    pub empty: T,
+}
@@ -15,12 +15,11 @@ pub enum GroupFilter {
 impl GroupFilter {
    pub fn contains(&self, value: usize) -> bool {
        match self {
-            GroupFilter::All => true,
+            GroupFilter::Range(r) => r.contains(&value),
            GroupFilter::LowerThan(max) => *max > value,
            GroupFilter::GreaterOrEqual(min) => *min <= value,
-            GroupFilter::Range(r) => r.contains(&value),
-            GroupFilter::Epoch(_) => false,
-            GroupFilter::Type(_) => false,
+            GroupFilter::All => true,
+            GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
        }
    }

@@ -28,24 +27,22 @@ impl GroupFilter {
        match self {
            GroupFilter::All => true,
            GroupFilter::LowerThan(max) => match other {
-                GroupFilter::All => false,
                GroupFilter::LowerThan(max2) => max >= max2,
                GroupFilter::Range(range) => range.end <= *max,
-                GroupFilter::GreaterOrEqual(_) => false,
-                GroupFilter::Epoch(_) => false,
-                GroupFilter::Type(_) => false,
+                GroupFilter::All
+                | GroupFilter::GreaterOrEqual(_)
+                | GroupFilter::Epoch(_)
+                | GroupFilter::Type(_) => false,
            },
            GroupFilter::GreaterOrEqual(min) => match other {
-                GroupFilter::All => false,
-                GroupFilter::LowerThan(_) => false,
                GroupFilter::Range(range) => range.start >= *min,
                GroupFilter::GreaterOrEqual(min2) => min <= min2,
-                GroupFilter::Epoch(_) => false,
-                GroupFilter::Type(_) => false,
+                GroupFilter::All
+                | GroupFilter::LowerThan(_)
+                | GroupFilter::Epoch(_)
+                | GroupFilter::Type(_) => false,
            },
-            GroupFilter::Range(_) => false,
-            GroupFilter::Epoch(_) => false,
-            GroupFilter::Type(_) => false,
+            GroupFilter::Range(_) | GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
        }
    }
 }
@@ -2,6 +2,7 @@ mod address;
 mod by_address_type;
 mod by_age_range;
 mod by_amount_range;
+mod by_any_address;
 mod by_epoch;
 mod by_ge_amount;
 mod by_lt_amount;
@@ -18,6 +19,7 @@ pub use address::*;
 pub use by_address_type::*;
 pub use by_age_range::*;
 pub use by_amount_range::*;
+pub use by_any_address::*;
 pub use by_epoch::*;
 pub use by_ge_amount::*;
 pub use by_lt_amount::*;
@@ -0,0 +1,61 @@
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{
+    TypeIndex,
+    structs::{EmptyAddressIndex, LoadedAddressIndex},
+};
+
+const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000;
+
+#[derive(
+    Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout,
+)]
+pub struct AnyAddressIndex(TypeIndex);
+
+impl AnyAddressIndex {
+    pub fn to_enum(&self) -> AnyAddressDataIndexEnum {
+        AnyAddressDataIndexEnum::from(*self)
+    }
+}
+
+impl From<LoadedAddressIndex> for AnyAddressIndex {
+    fn from(value: LoadedAddressIndex) -> Self {
+        if u32::from(value) >= MIN_EMPTY_INDEX {
+            panic!("")
+        }
+        Self(*value)
+    }
+}
+
+impl From<EmptyAddressIndex> for AnyAddressIndex {
+    fn from(value: EmptyAddressIndex) -> Self {
+        Self(*value + MIN_EMPTY_INDEX)
+    }
+}
+
+impl Serialize for AnyAddressIndex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_enum().serialize(serializer)
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub enum AnyAddressDataIndexEnum {
+    Loaded(LoadedAddressIndex),
+    Empty(EmptyAddressIndex),
+}
+
+impl From<AnyAddressIndex> for AnyAddressDataIndexEnum {
+    fn from(value: AnyAddressIndex) -> Self {
+        let uvalue = u32::from(value.0);
+        if uvalue >= MIN_EMPTY_INDEX {
+            Self::Empty(EmptyAddressIndex::from(uvalue - MIN_EMPTY_INDEX))
+        } else {
+            Self::Loaded(LoadedAddressIndex::from(value.0))
+        }
+    }
+}
@@ -1,22 +1,23 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

-use crate::{AddressData, Sats};
+use crate::{LoadedAddressData, Sats};

-#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)]
+#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
 pub struct EmptyAddressData {
    pub transfered: Sats,
 }

-impl From<AddressData> for EmptyAddressData {
-    fn from(value: AddressData) -> Self {
+impl From<LoadedAddressData> for EmptyAddressData {
+    fn from(value: LoadedAddressData) -> Self {
        Self::from(&value)
    }
 }

-impl From<&AddressData> for EmptyAddressData {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for EmptyAddressData {
+    fn from(value: &LoadedAddressData) -> Self {
        if value.sent != value.received {
            dbg!(&value);
            panic!("Trying to convert not empty wallet to empty !");
@@ -0,0 +1,69 @@
+use std::ops::Add;
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    Default,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Deref,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct EmptyAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for EmptyAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for EmptyAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+impl From<u32> for EmptyAddressIndex {
+    fn from(value: u32) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+
+impl From<EmptyAddressIndex> for usize {
+    fn from(value: EmptyAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+
+impl Add<usize> for EmptyAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<EmptyAddressIndex> for EmptyAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for EmptyAddressIndex {
+    fn to_string() -> &'static str {
+        "emptyaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["emptyaddr", "emptyaddressindex"]
+    }
+}
@@ -31,7 +31,7 @@ impl From<TypeIndex> for EmptyOutputIndex {
 }
 impl From<EmptyOutputIndex> for usize {
    fn from(value: EmptyOutputIndex) -> Self {
-        Self::from(*value)
+        Self::from(value.0)
    }
 }
 impl From<usize> for EmptyOutputIndex {
@@ -42,7 +42,7 @@ impl From<usize> for EmptyOutputIndex {
 impl Add<usize> for EmptyOutputIndex {
    type Output = Self;
    fn add(self, rhs: usize) -> Self::Output {
-        Self(*self + rhs)
+        Self(self.0 + rhs)
    }
 }
 impl CheckedSub<EmptyOutputIndex> for EmptyOutputIndex {
@@ -1,18 +1,21 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

 use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats};

-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
 #[repr(C)]
-pub struct AddressData {
+pub struct LoadedAddressData {
    pub sent: Sats,
    pub received: Sats,
    pub realized_cap: Dollars,
    pub outputs_len: u32,
+    padding: u32,
 }

-impl AddressData {
+impl LoadedAddressData {
    pub fn amount(&self) -> Sats {
        (u64::from(self.received) - u64::from(self.sent)).into()
    }
@@ -55,23 +58,24 @@ impl AddressData {
    }
 }

-impl From<EmptyAddressData> for AddressData {
+impl From<EmptyAddressData> for LoadedAddressData {
    fn from(value: EmptyAddressData) -> Self {
        Self::from(&value)
    }
 }
-impl From<&EmptyAddressData> for AddressData {
+impl From<&EmptyAddressData> for LoadedAddressData {
    fn from(value: &EmptyAddressData) -> Self {
        Self {
            sent: value.transfered,
            received: value.transfered,
            realized_cap: Dollars::ZERO,
            outputs_len: 0,
+            padding: 0,
        }
    }
 }

-impl From<ByteView> for AddressData {
+impl From<ByteView> for LoadedAddressData {
    fn from(value: ByteView) -> Self {
        Self {
            // MUST be same order as impl From<&AddressData> for ByteView
@@ -79,16 +83,17 @@ impl From<ByteView> for AddressData {
            received: Sats::read_from_bytes(&value[8..16]).unwrap(),
            realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(),
            outputs_len: u32::read_from_bytes(&value[24..]).unwrap(),
+            padding: 0,
        }
    }
 }
-impl From<AddressData> for ByteView {
-    fn from(value: AddressData) -> Self {
+impl From<LoadedAddressData> for ByteView {
+    fn from(value: LoadedAddressData) -> Self {
        Self::from(&value)
    }
 }
-impl From<&AddressData> for ByteView {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for ByteView {
+    fn from(value: &LoadedAddressData) -> Self {
        Self::new(
            &[
                value.sent.as_bytes(),
@@ -0,0 +1,67 @@
+use std::ops::Add;
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Deref,
+    Default,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct LoadedAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for LoadedAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for LoadedAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+impl From<LoadedAddressIndex> for usize {
+    fn from(value: LoadedAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+impl From<LoadedAddressIndex> for u32 {
+    fn from(value: LoadedAddressIndex) -> Self {
+        u32::from(value.0)
+    }
+}
+impl Add<usize> for LoadedAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<LoadedAddressIndex> for LoadedAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for LoadedAddressIndex {
+    fn to_string() -> &'static str {
+        "loadedaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["loadedaddr", "loadedaddressindex"]
+    }
+}
@@ -1,6 +1,6 @@
 mod addressbytes;
 mod addressbyteshash;
-mod addressdata;
+mod anyaddressindex;
 mod bitcoin;
 mod blockhash;
 mod blockhashprefix;
@@ -11,11 +11,14 @@ mod decadeindex;
 mod difficultyepoch;
 mod dollars;
 mod emptyaddressdata;
+mod emptyaddressindex;
 mod emptyoutputindex;
 mod feerate;
 mod halvingepoch;
 mod height;
 mod inputindex;
+mod loadedaddressdata;
+mod loadedaddressindex;
 mod monthindex;
 mod ohlc;
 mod opreturnindex;
@@ -58,7 +61,7 @@ mod yearindex;

 pub use addressbytes::*;
 pub use addressbyteshash::*;
-pub use addressdata::*;
+pub use anyaddressindex::*;
 pub use bitcoin::*;
 pub use blockhash::*;
 pub use blockhashprefix::*;
@@ -69,11 +72,14 @@ pub use decadeindex::*;
 pub use difficultyepoch::*;
 pub use dollars::*;
 pub use emptyaddressdata::*;
+pub use emptyaddressindex::*;
 pub use emptyoutputindex::*;
 pub use feerate::*;
 pub use halvingepoch::*;
 pub use height::*;
 pub use inputindex::*;
+pub use loadedaddressdata::*;
+pub use loadedaddressindex::*;
 pub use monthindex::*;
 pub use ohlc::*;
 pub use opreturnindex::*;
@@ -73,6 +73,12 @@ impl From<TypeIndex> for usize {
    }
 }

+impl Add<u32> for TypeIndex {
+    type Output = Self;
+    fn add(self, rhs: u32) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
 impl Add<usize> for TypeIndex {
    type Output = Self;
    fn add(self, rhs: usize) -> Self::Output {
@@ -1,6 +1,10 @@
-use std::{path::Path, time::Instant};
+use std::{
+    fs,
+    path::Path,
+    thread::sleep,
+    time::{Duration, Instant},
+};

-use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_parser::Parser;
@@ -8,11 +12,12 @@ use brk_parser::Parser;
 fn main() -> color_eyre::Result<()> {
    color_eyre::install()?;

-    let i = Instant::now();
-
    brk_logger::init(Some(Path::new(".log")));

-    let bitcoin_dir = default_bitcoin_path();
+    // let bitcoin_dir = brk_core::default_bitcoin_path();
+    let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
+    // let outputs_dir = brk_core::default_brk_path().join("outputs");
+    let outputs_dir = Path::new("/Volumes/WD_BLACK/brk/outputs");

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
        "http://localhost:8332",
@@ -20,15 +25,17 @@ fn main() -> color_eyre::Result<()> {
    )?));
    let exit = Exit::new();

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);

-    let outputs = Path::new("../../_outputs");
+    fs::create_dir_all(outputs_dir)?;

-    let mut indexer = Indexer::forced_import(outputs)?;
+    let mut indexer = Indexer::forced_import(outputs_dir)?;

-    indexer.index(&parser, rpc, &exit, true)?;
+    loop {
+        let i = Instant::now();
+        indexer.index(&parser, rpc, &exit, false)?;
+        dbg!(i.elapsed());

-    dbg!(i.elapsed());
-
-    Ok(())
+        sleep(Duration::from_secs(5 * 60));
+    }
 }
@@ -761,8 +761,6 @@ impl Indexer {

        export_if_needed(stores, vecs, idxs.height, true, exit)?;

-        stores.rotate_memtables();
-
        Ok(starting_indexes)
    }
 }
@@ -181,12 +181,6 @@ impl Stores {
            .map_err(|e| e.into())
    }

-    pub fn rotate_memtables(&self) {
-        self.as_slice()
-            .into_iter()
-            .for_each(|store| store.rotate_memtable());
-    }
-
    fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] {
        [
            &self.addressbyteshash_to_typeindex,
@@ -14,7 +14,7 @@ brk_indexer = { workspace = true }
 brk_vec = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
-brk_rmcp = { workspace = true }
+rmcp = { workspace = true }
 schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -1,5 +1,5 @@
-use brk_rmcp::schemars::JsonSchema;
 use color_eyre::eyre::eyre;
+use rmcp::schemars::JsonSchema;
 use serde::Deserialize;

 #[allow(clippy::upper_case_acronyms)]
@@ -1,6 +1,6 @@
 use std::ops::Deref;

-use brk_rmcp::schemars::{self, JsonSchema};
+use rmcp::schemars::{self, JsonSchema};
 use serde::Deserialize;

 use crate::{
@@ -101,33 +101,6 @@ impl<'a> Vecs<'a> {
    fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
        let name = vec.name();
        let serialized_index = vec.index_type_to_string();
-        // let split = name.split("_to_").collect::<Vec<_>>();
-        // if split.len() != 1
-        //     && !(split.len() == 2
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.ends_with("relative")
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s.starts_with("cumulative_start")
-        //                 || s.starts_with("cumulative_from")
-        //                 || s == &"activity"
-        //         }))
-        //     && !(split.len() == 3
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s == &"cumulative_start"
-        //                 || s.starts_with("cumulative_from")
-        //         })
-        //         && split.get(1).is_some_and(|s| s.ends_with("relative")))
-        // {
-        //     dbg!((&serialized_index, &name, &split));
-        //     unreachable!();
-        // }
        let index = Index::try_from(serialized_index)
            .inspect_err(|_| {
                dbg!(&serialized_index);
@@ -26,7 +26,7 @@ pub fn init(path: Option<&Path>) {
    });

    Builder::from_env(Env::default().default_filter_or(
-        "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,brk_rmcp=off,tracing=off",
+        "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,rmcp=off,tracing=off",
    ))
    .format(move |buf, record| {
        let date_time = Timestamp::now()
@@ -11,4 +11,4 @@ repository.workspace = true
 axum = { workspace = true }
 brk_interface = { workspace = true }
 log = { workspace = true }
-brk_rmcp = { workspace = true }
+rmcp = { workspace = true }
@@ -4,14 +4,14 @@
 // #![doc = "```"]

 use brk_interface::{IdParam, Interface, PaginatedIndexParam, PaginationParam, Params};
-use brk_rmcp::{
-    Error as McpError, RoleServer, ServerHandler,
+use log::info;
+use rmcp::{
+    ErrorData as McpError, RoleServer, ServerHandler,
    handler::server::{router::tool::ToolRouter, tool::Parameters},
    model::*,
    service::RequestContext,
    tool, tool_handler, tool_router,
 };
-use log::info;

 pub mod route;

@@ -1,6 +1,6 @@
 use axum::Router;
 use brk_interface::Interface;
-use brk_rmcp::transport::{
+use rmcp::transport::{
    StreamableHttpServerConfig,
    streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager},
 };
@@ -22,15 +22,13 @@ where
            return self;
        }

-        let config = StreamableHttpServerConfig {
-            // stateful_mode: false, // breaks Claude
-            ..Default::default()
-        };
-
        let service = StreamableHttpService::new(
            move || Ok(MCP::new(interface)),
            LocalSessionManager::default().into(),
-            config,
+            StreamableHttpServerConfig {
+                // stateful_mode: false, // breaks Claude
+                ..Default::default()
+            },
        );

        info!("Setting MCP...");
@@ -1,11 +1,12 @@
 use bitcoincore_rpc::{Auth, Client};
-use brk_core::{Height, default_bitcoin_path};
+use brk_core::{Height, default_bitcoin_path, default_brk_path};
 use brk_parser::Parser;

 fn main() {
    let i = std::time::Instant::now();

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(
        Client::new(
@@ -18,7 +19,7 @@ fn main() {
    let start = None;
    let end = None;

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    parser
        .parse(start, end)
@@ -1,11 +1,12 @@
 use bitcoincore_rpc::{Auth, Client};
-use brk_core::{Height, OutputType, default_bitcoin_path};
+use brk_core::{Height, OutputType, default_bitcoin_path, default_brk_path};
 use brk_parser::Parser;

 fn main() {
    let i = std::time::Instant::now();

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(
        Client::new(
@@ -18,7 +19,7 @@ fn main() {
    // let start = None;
    // let end = None;

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    // parser
    //     .parse(start, end)
@@ -5,7 +5,7 @@ use std::{
    path::{Path, PathBuf},
 };

-use crate::{BlkIndexToBlkPath, Height, blk_recap::BlkRecap};
+use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, Height};

 #[derive(Debug)]
 pub struct BlkIndexToBlkRecap {
@@ -15,11 +15,11 @@ pub struct BlkIndexToBlkRecap {

 impl BlkIndexToBlkRecap {
    pub fn import(
-        bitcoin_dir: &Path,
+        outputs_dir: &Path,
        blk_index_to_blk_path: &BlkIndexToBlkPath,
        start: Option<Height>,
    ) -> (Self, u16) {
-        let path = bitcoin_dir.join("blk_index_to_blk_recap.json");
+        let path = outputs_dir.join("blk_index_to_blk_recap.json");

        let tree = {
            if let Ok(file) = File::open(&path) {
@@ -38,12 +38,21 @@ const BOUND_CAP: usize = 50;

 pub struct Parser {
    blocks_dir: PathBuf,
+    outputs_dir: PathBuf,
    rpc: &'static bitcoincore_rpc::Client,
 }

 impl Parser {
-    pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
-        Self { blocks_dir, rpc }
+    pub fn new(
+        blocks_dir: PathBuf,
+        outputs_dir: PathBuf,
+        rpc: &'static bitcoincore_rpc::Client,
+    ) -> Self {
+        Self {
+            blocks_dir,
+            outputs_dir,
+            rpc,
+        }
    }

    pub fn get(&self, height: Height) -> Block {
@@ -74,7 +83,7 @@ impl Parser {
        let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);

        let (mut blk_index_to_blk_recap, blk_index) =
-            BlkIndexToBlkRecap::import(blocks_dir, &blk_index_to_blk_path, start);
+            BlkIndexToBlkRecap::import(&self.outputs_dir, &blk_index_to_blk_path, start);

        let xor_bytes = XORBytes::from(blocks_dir);

@@ -28,6 +28,7 @@ jiff = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 tokio = { workspace = true }
 tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }
 tracing = "0.1.41"
@@ -2,7 +2,7 @@ use std::{path::Path, thread::sleep, time::Duration};

 use bitcoincore_rpc::RpcApi;
 use brk_computer::Computer;
-use brk_core::default_bitcoin_path;
+use brk_core::{default_bitcoin_path, default_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -18,6 +18,7 @@ pub fn main() -> color_eyre::Result<()> {
    let process = true;

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
        "http://localhost:8332",
@@ -25,7 +26,7 @@ pub fn main() -> color_eyre::Result<()> {
    )?));
    let exit = Exit::new();

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    let outputs_dir = Path::new("../../_outputs");

@@ -107,6 +107,16 @@ impl ApiRoutes for Router<AppState> {
                },
            ),
        )
+        .route(
+            "/health",
+            get(|| async {
+                Json(serde_json::json!({
+                    "status": "healthy",
+                    "service": "brk-server",
+                    "timestamp": jiff::Timestamp::now().to_string()
+                }))
+            }),
+        )
        .route(
            "/api",
            get(|| async {
@@ -277,10 +277,6 @@ where
        Ok(())
    }

-    fn rotate_memtable(&self) {
-        let _ = self.partition.as_ref().unwrap().inner().rotate_memtable();
-    }
-
    fn height(&self) -> Option<Height> {
        self.meta.height()
    }
@@ -9,8 +9,6 @@ pub trait AnyStore {

    fn name(&self) -> &'static str;

-    fn rotate_memtable(&self);
-
    fn height(&self) -> Option<Height>;

    fn has(&self, height: Height) -> bool;
@@ -1,6 +1,6 @@
 [package]
 name = "brk_vec"
-description = "A push-only, truncable, compressable, saveable Vec"
+description = "A storeable vec"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
 version.workspace = true
@@ -1,6 +1,6 @@
 use std::{fs, path::Path};

-use brk_core::{DateIndex, Height, Printable, Version};
+use brk_core::{DateIndex, Height, Version};
 use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator};

 type I = DateIndex;
@@ -9,7 +9,6 @@ type VEC = StoredVec<I, u32>;

 fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _ = fs::remove_dir_all("./vec");
-    let _ = fs::remove_file("./vec");

    let version = Version::TWO;
    let format = Format::Raw;
@@ -102,6 +101,38 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
        dbg!(iter.get(0.into()));
        dbg!(iter.get(20.into()));
        dbg!(iter.get(21.into()));
+
+        let mmap = vec.create_mmap()?;
+        dbg!(vec.take(10.into(), &mmap)?);
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
+        dbg!(vec.holes());
+        vec.flush()?;
+        dbg!(vec.holes());
+    }
+
+    {
+        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        let mmap = vec.create_mmap()?;
+
+        dbg!(vec.holes());
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
+
+        vec.update(10.into(), 10)?;
+        vec.update(0.into(), 10)?;
+        dbg!(
+            vec.holes(),
+            vec.get_or_read(0.into(), &mmap)?,
+            vec.get_or_read(10.into(), &mmap)?
+        );
+
+        vec.flush()?;
+    }
+
+    {
+        let vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        dbg!(vec.collect()?);
    }

    Ok(())
@@ -1,11 +1,13 @@
 use std::{
    borrow::Cow,
-    fs::{File, OpenOptions},
+    cmp::Ordering,
+    collections::{BTreeMap, BTreeSet},
+    fs::{self, File, OpenOptions},
    io::{self, Seek, SeekFrom, Write},
    path::{Path, PathBuf},
 };

-use brk_core::Result;
+use brk_core::{Error, Result};
 use memmap2::Mmap;

 use crate::{AnyVec, HEADER_OFFSET, Header};
@@ -44,10 +46,22 @@ where
            if j >= pushed.len() {
                return Ok(None);
            }
-            Ok(pushed.get(j).map(Cow::Borrowed))
-        } else {
-            Ok(self.read_(index, mmap)?.map(Cow::Owned))
+            return Ok(pushed.get(j).map(Cow::Borrowed));
        }
+
+        let updated = self.updated();
+        if !updated.is_empty()
+            && let Some(updated) = updated.get(&index)
+        {
+            return Ok(Some(Cow::Borrowed(updated)));
+        }
+
+        let holes = self.holes();
+        if !holes.is_empty() && holes.contains(&index) {
+            return Ok(None);
+        }
+
+        Ok(self.read_(index, mmap)?.map(Cow::Owned))
    }

    #[inline]
@@ -72,6 +86,95 @@ where
        self.mut_pushed().push(value)
    }

+    #[inline]
+    fn update_or_push(&mut self, index: I, value: T) -> Result<()> {
+        let len = self.len();
+        match len.cmp(&index.to_usize()?) {
+            Ordering::Less => {
+                dbg!(index, value, len, self.header());
+                Err(Error::IndexTooHigh)
+            }
+            Ordering::Equal => {
+                self.push(value);
+                Ok(())
+            }
+            Ordering::Greater => self.update(index, value),
+        }
+    }
+
+    fn get_first_empty_index(&self) -> I {
+        self.holes()
+            .first()
+            .cloned()
+            .unwrap_or_else(|| self.len_())
+            .into()
+    }
+
+    #[inline]
+    fn fill_first_hole_or_push(&mut self, value: T) -> Result<I> {
+        Ok(
+            if let Some(hole) = self.mut_holes().pop_first().map(I::from) {
+                self.update(hole, value)?;
+                hole
+            } else {
+                self.push(value);
+                I::from(self.len() - 1)
+            },
+        )
+    }
+
+    fn holes(&self) -> &BTreeSet<usize>;
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize>;
+    fn take(&mut self, index: I, mmap: &Mmap) -> Result<Option<T>> {
+        let opt = self.get_or_read(index, mmap)?.map(|v| v.into_owned());
+        if opt.is_some() {
+            self.unchecked_delete(index);
+        }
+        Ok(opt)
+    }
+    #[inline]
+    fn delete(&mut self, index: I) {
+        if index.unwrap_to_usize() < self.len() {
+            self.unchecked_delete(index);
+        }
+    }
+    #[inline]
+    #[doc(hidden)]
+    fn unchecked_delete(&mut self, index: I) {
+        let uindex = index.unwrap_to_usize();
+        let updated = self.mut_updated();
+        if !updated.is_empty() {
+            updated.remove(&uindex);
+        }
+        self.mut_holes().insert(uindex);
+    }
+
+    fn updated(&self) -> &BTreeMap<usize, T>;
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T>;
+    #[inline]
+    fn update(&mut self, index: I, value: T) -> Result<()> {
+        let uindex = index.unwrap_to_usize();
+        let stored_len = self.stored_len();
+
+        if uindex >= stored_len {
+            if let Some(prev) = self.mut_pushed().get_mut(uindex - stored_len) {
+                *prev = value;
+                return Ok(());
+            } else {
+                return Err(Error::IndexTooHigh);
+            }
+        }
+
+        let holes = self.mut_holes();
+        if !holes.is_empty() {
+            holes.remove(&index.unwrap_to_usize());
+        }
+
+        self.mut_updated().insert(index.unwrap_to_usize(), value);
+
+        Ok(())
+    }
+
    fn header(&self) -> &Header;
    fn mut_header(&mut self) -> &mut Header;

@@ -85,14 +188,24 @@ where
        parent.join(name)
    }

+    #[inline]
    fn path(&self) -> PathBuf {
        Self::path_(self.parent(), self.name())
    }
-
+    #[inline]
    fn path_(parent: &Path, name: &str) -> PathBuf {
        Self::folder_(parent, name).join(I::to_string())
    }

+    #[inline]
+    fn holes_path(&self) -> PathBuf {
+        Self::holes_path_(self.parent(), self.name())
+    }
+    #[inline]
+    fn holes_path_(parent: &Path, name: &str) -> PathBuf {
+        Self::folder_(parent, name).join(format!("{}_holes", I::to_string()))
+    }
+
    // ---

    fn open_file(&self) -> io::Result<File> {
@@ -134,6 +247,10 @@ where

    #[inline]
    fn reset_(&mut self) -> Result<()> {
+        let holes_path = self.holes_path();
+        if fs::exists(&holes_path)? {
+            fs::remove_file(&holes_path)?;
+        }
        let mut file = self.open_file()?;
        self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
    }
@@ -1,6 +1,7 @@
 use std::{fmt::Debug, ops::Add};

 use brk_core::{Error, Printable, Result};
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};

 pub trait StoredIndex
 where
@@ -15,6 +16,10 @@ where
        + TryInto<usize>
        + From<usize>
        + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
        + Send
        + Sync
        + Printable,
@@ -37,6 +42,10 @@ where
        + TryInto<usize>
        + From<usize>
        + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
        + Send
        + Sync
        + Printable,
@@ -1,5 +1,6 @@
 use std::{
    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
    fs, mem,
    path::{Path, PathBuf},
    sync::Arc,
@@ -56,6 +57,7 @@ where
        }
    }

+    #[allow(unreachable_code, unused_variables)]
    pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
        let mut inner = RawVec::import(parent, name, version)?;

@@ -196,12 +198,44 @@ where
    fn mut_pushed(&mut self) -> &mut Vec<T> {
        self.inner.mut_pushed()
    }
+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        self.inner.holes()
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        self.inner.mut_holes()
+    }
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        self.inner.updated()
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        self.inner.mut_updated()
+    }

    #[inline]
    fn path(&self) -> PathBuf {
        self.inner.path()
    }

+    fn delete(&mut self, _: I) {
+        panic!("unsupported")
+    }
+
+    fn unchecked_delete(&mut self, _: I) {
+        panic!("unsupported")
+    }
+
+    fn fill_first_hole_or_push(&mut self, _: T) -> Result<I> {
+        panic!("unsupported")
+    }
+
+    fn update(&mut self, _: I, _: T) -> Result<()> {
+        panic!("unsupported")
+    }
+
    fn flush(&mut self) -> Result<()> {
        let file_opt = self.inner.write_header_if_needed()?;

@@ -38,6 +38,30 @@ where
        self.0.get_or_read(index, mmap)
    }

+    #[inline]
+    pub fn update_or_push(&mut self, index: I, value: T) -> Result<()> {
+        self.0.update_or_push(index, value)
+    }
+
+    #[inline]
+    pub fn checked_push(&mut self, index: I, value: T) -> Result<()> {
+        let len = self.0.len();
+        match len.cmp(&index.to_usize()?) {
+            Ordering::Greater => {
+                dbg!(index, value, len, self.0.header());
+                Err(Error::IndexTooLow)
+            }
+            Ordering::Equal => {
+                self.0.push(value);
+                Ok(())
+            }
+            Ordering::Less => {
+                dbg!(index, value, len, self.0.header());
+                Err(Error::IndexTooHigh)
+            }
+        }
+    }
+
    #[inline]
    pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
        let len = self.0.len();
@@ -58,10 +82,32 @@ where
        }
    }

+    #[inline]
+    pub fn fill_first_hole_or_push(&mut self, value: T) -> Result<I> {
+        self.0.fill_first_hole_or_push(value)
+    }
+
+    pub fn update(&mut self, index: I, value: T) -> Result<()> {
+        self.0.update(index, value)
+    }
+
+    pub fn take(&mut self, index: I, mmap: &Mmap) -> Result<Option<T>> {
+        self.0.take(index, mmap)
+    }
+
+    pub fn delete(&mut self, index: I) {
+        self.0.delete(index)
+    }
+
    fn update_height(&mut self, height: Height) {
        self.0.mut_header().update_height(height);
    }

+    pub fn reset(&mut self) -> Result<()> {
+        self.update_height(Height::ZERO);
+        self.0.reset()
+    }
+
    pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> {
        self.update_height(height);
        self.0.truncate_if_needed(index)?;
@@ -1,9 +1,11 @@
 use std::{
    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
    fs::{self, File},
    io,
    marker::PhantomData,
    mem,
+    os::unix::fs::FileExt,
    path::{Path, PathBuf},
    sync::{
        Arc,
@@ -24,12 +26,17 @@ const VERSION: Version = Version::ONE;

 #[derive(Debug)]
 pub struct RawVec<I, T> {
+    // --- Needed for &, TODO: Weak copy ?
    header: Header,
    parent: PathBuf,
    name: &'static str,
-    pushed: Vec<T>,
-    local_stored_len: Option<usize>,
    shared_stored_len: Arc<AtomicUsize>,
+    // --- Needed for &mut
+    pushed: Vec<T>,
+    has_stored_holes: bool,
+    holes: BTreeSet<usize>,
+    updated: BTreeMap<usize, T>,
+    local_stored_len: Option<usize>,
    phantom: PhantomData<I>,
 }

@@ -49,6 +56,10 @@ where
            | Err(Error::DifferentVersion { .. }) => {
                let path = Self::path_(parent, name);
                fs::remove_file(path)?;
+                let holes_path = Self::holes_path_(parent, name);
+                if fs::exists(&holes_path)? {
+                    fs::remove_file(holes_path)?;
+                }
                Self::import(parent, name, version)
            }
            _ => res,
@@ -90,17 +101,35 @@ where
            0
        };

+        let holes_path = Self::holes_path_(parent, name);
+        let holes = if fs::exists(&holes_path)? {
+            Some(
+                fs::read(&holes_path)?
+                    .chunks(size_of::<usize>())
+                    .map(|b| -> Result<usize> {
+                        Ok(usize::from_ne_bytes(brk_core::copy_first_8bytes(b)?))
+                    })
+                    .collect::<Result<BTreeSet<usize>>>()?,
+            )
+        } else {
+            None
+        };
+
        Ok(Self {
            header,
            name: Box::leak(Box::new(name.to_string())),
            parent: parent.to_owned(),
            pushed: vec![],
+            has_stored_holes: holes.is_some(),
+            holes: holes.unwrap_or_default(),
+            updated: BTreeMap::new(),
            local_stored_len: Some(stored_len),
            shared_stored_len: Arc::new(AtomicUsize::new(stored_len)),
            phantom: PhantomData,
        })
    }

+    #[doc(hidden)]
    pub fn set_stored_len(&mut self, len: usize) {
        self.local_stored_len.replace(len);
        self.shared_stored_len.store(len, Ordering::Relaxed);
@@ -171,6 +200,24 @@ where
        &mut self.pushed
    }

+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        &self.holes
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        &mut self.holes
+    }
+
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        &self.updated
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        &mut self.updated
+    }
+
    #[inline]
    fn parent(&self) -> &Path {
        &self.parent
@@ -181,33 +228,72 @@ where

        let pushed_len = self.pushed_len();

-        if pushed_len == 0 {
+        let has_new_data = pushed_len != 0;
+        let has_updated_data = !self.updated.is_empty();
+        let has_holes = !self.holes.is_empty();
+        let had_holes = self.has_stored_holes && !has_holes;
+
+        if !has_new_data && !has_updated_data && !has_holes && !had_holes {
            return Ok(());
        }

-        let bytes = {
-            let pushed = &mut self.pushed;
+        if has_new_data || has_updated_data {
+            let mut file = file_opt.unwrap_or(self.open_file()?);

-            let mut bytes: Vec<u8> = vec![0; pushed.len() * Self::SIZE_OF_T];
+            if has_new_data {
+                let bytes = {
+                    let mut bytes: Vec<u8> = vec![0; pushed_len * Self::SIZE_OF_T];

-            let unsafe_bytes = UnsafeSlice::new(&mut bytes);
+                    let unsafe_bytes = UnsafeSlice::new(&mut bytes);

-            mem::take(pushed)
-                .into_par_iter()
-                .enumerate()
-                .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()));
+                    mem::take(&mut self.pushed)
+                        .into_par_iter()
+                        .enumerate()
+                        .for_each(|(i, v)| {
+                            unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes())
+                        });

-            bytes
-        };
+                    bytes
+                };

-        let mut file = file_opt.unwrap_or(self.open_file()?);
-        self.file_write_all(&mut file, &bytes)?;
+                self.file_write_all(&mut file, &bytes)?;

-        if let Some(local_stored_len) = self.local_stored_len.as_mut() {
-            *local_stored_len += pushed_len;
+                if let Some(local_stored_len) = self.local_stored_len.as_mut() {
+                    *local_stored_len += pushed_len;
+                }
+                self.shared_stored_len
+                    .fetch_add(pushed_len, Ordering::Relaxed);
+            }
+
+            if has_updated_data {
+                mem::take(&mut self.updated)
+                    .into_iter()
+                    .try_for_each(|(i, v)| -> Result<()> {
+                        file.write_all_at(
+                            v.as_bytes(),
+                            ((i * Self::SIZE_OF_T) + HEADER_OFFSET) as u64,
+                        )?;
+                        Ok(())
+                    })?;
+            }
+        }
+
+        if has_holes || had_holes {
+            let holes_path = self.holes_path();
+            if has_holes {
+                self.has_stored_holes = true;
+                fs::write(
+                    &holes_path,
+                    self.holes
+                        .iter()
+                        .flat_map(|i| i.to_ne_bytes())
+                        .collect::<Vec<_>>(),
+                )?;
+            } else if had_holes {
+                self.has_stored_holes = false;
+                let _ = fs::remove_file(&holes_path);
+            }
        }
-        self.shared_stored_len
-            .fetch_add(pushed_len, Ordering::Relaxed);

        Ok(())
    }
@@ -278,9 +364,12 @@ impl<I, T> Clone for RawVec<I, T> {
            parent: self.parent.clone(),
            name: self.name,
            pushed: vec![],
-            phantom: PhantomData,
+            updated: BTreeMap::new(),
+            has_stored_holes: false,
+            holes: BTreeSet::new(),
            local_stored_len: None,
            shared_stored_len: self.shared_stored_len.clone(),
+            phantom: PhantomData,
        }
    }
 }
@@ -1,5 +1,6 @@
 use std::{
    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
    path::{Path, PathBuf},
 };

@@ -105,6 +106,36 @@ where
        }
    }

+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        match self {
+            StoredVec::Raw(v) => v.holes(),
+            StoredVec::Compressed(v) => v.holes(),
+        }
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        match self {
+            StoredVec::Raw(v) => v.mut_holes(),
+            StoredVec::Compressed(v) => v.mut_holes(),
+        }
+    }
+
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        match self {
+            StoredVec::Raw(v) => v.updated(),
+            StoredVec::Compressed(v) => v.updated(),
+        }
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        match self {
+            StoredVec::Raw(v) => v.mut_updated(),
+            StoredVec::Compressed(v) => v.mut_updated(),
+        }
+    }
+
    #[inline]
    fn path(&self) -> PathBuf {
        match self {
@@ -0,0 +1,68 @@
+# Git
+.git
+.gitignore
+
+# Build artifacts
+target/
+
+# Development files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Docker files
+Dockerfile
+docker-compose.yml
+.dockerignore
+docker-build.sh
+
+# Documentation  
+docs/
+LICENSE
+# Keep README.md for build process
+!README.md
+
+# CI/CD
+.github/
+
+# Logs and temporary files
+*.log
+tmp/
+temp/
+
+# BRK runtime data (should be in volumes)
+.brk/
+
+# Example and test data
+examples/
+tests/
+*.test
+*.example
+
+# Node modules (if any frontend deps)
+node_modules/
+
+# Python cache (if any)
+__pycache__/
+*.pyc
+*.pyo
+
+# Rust workspace cache
+**/*.rs.bk
+
+# macOS
+.AppleDouble
+.LSOverride
+
+# Windows
+Desktop.ini
+ehthumbs.db
+
+# Linux
+.directory
@@ -0,0 +1,42 @@
+# Bitcoin Core data directory
+# This should point to your Bitcoin Core data directory
+BITCOIN_DATA_DIR=/path/to/bitcoin
+
+# Bitcoin Core RPC configuration
+# If running Bitcoin Core on the same host (not in Docker), use host.docker.internal on macOS/Windows
+# or the host's IP address on Linux
+BTC_RPC_HOST=localhost
+BTC_RPC_PORT=8332
+
+# Use either cookie file authentication (recommended) or username/password
+# Cookie file is automatically created by Bitcoin Core
+# If using username/password, comment out RPCCOOKIEFILE in docker-compose.yml
+# BTC_RPC_USER=your_rpc_username
+# BTC_RPC_PASSWORD=your_rpc_password
+
+# BRK configuration
+# Services to run: all, processor, or server
+BRK_SERVICES=all
+
+# Computation mode: lazy (compute on demand) or eager (precompute and save)
+BRK_COMPUTATION=lazy
+
+# Data format: raw (faster) or compressed (saves disk space)
+BRK_FORMAT=raw
+
+# Enable price fetching from exchanges
+BRK_FETCH=true
+
+# Enable Model Context Protocol (MCP) for AI/LLM integration
+BRK_MCP=true
+
+# BRK data storage options
+# Option 1: Use a Docker named volume (default, recommended)
+# This is the default configuration - no changes needed.
+# Leave this commented to use the default named volume
+# BRK_DATA_VOLUME=brk-data
+
+# Option 2: Use a bind mount to a local directory
+# Uncomment and set this to use a specific directory on your host
+# Also uncomment the corresponding line in docker-compose.yml
+# BRK_DATA_DIR=/path/to/brk/data
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
nym21	537d98b41b	release: v0.0.81	2025-07-17 23:45:01 +02:00
nym21	9c4cadfc04	vec: fix holes export	2025-07-17 17:29:53 +02:00
nym21	2001370441	mcp: use rust-rmcp instead of brk-rmcp	2025-07-17 16:34:29 +02:00
nym21	cc87b22757	computer: perf improvements	2025-07-17 16:17:21 +02:00
nym21	c0a65b30ad	indexer: update example	2025-07-17 11:39:25 +02:00
nym21	c07e66c086	computer: fix stateful	2025-07-17 11:35:40 +02:00
nym21	a0cfc1be2b	computer: convert stores to vecs part 2	2025-07-16 16:23:40 +02:00
nym21	1505454793	computer: convert stores to vecs part 1	2025-07-15 22:47:46 +02:00
nym21	e1dff66283	pr: merge #21 from deadmanoz/dockerize Add Docker support	2025-07-15 15:51:32 +00:00
deadmanoz	5be801a086	Merge branch 'main' into dockerize	2025-07-15 08:50:22 -07:00
deadmanoz	94d4b05c29	Address review feedback	2025-07-15 08:48:39 -07:00
nym21	cebb889f7e	cargo: update	2025-07-14 16:00:31 +02:00
nym21	c4ed6ed034	store: remove rotate_memtable as could be the root cause of the issue	2025-07-14 15:48:19 +02:00
deadmanoz	5de9757d46	Remove services from docker	2025-07-04 16:37:39 +08:00
deadmanoz	f89276d7b8	Remove redundant services	2025-07-04 15:51:28 +08:00
deadmanoz	30ba034206	Move docker artefacts into /docker directory	2025-07-04 13:00:12 +08:00
deadmanoz	fa1e5aaa7f	Make Parser::new the only entrypoint	2025-07-04 12:15:32 +08:00
deadmanoz	870c70180f	Back to a single image/container setup	2025-07-04 11:40:37 +08:00
deadmanoz	d83a833b4d	Switch to multiple container setup	2025-06-27 12:56:25 +08:00
deadmanoz	ec3a2f29f0	Docker functionality, change location of 'blk_index_to_blk_recap.json'	2025-06-27 12:56:03 +08:00