release: v0.0.81

vec: fix holes export
mcp: use rust-rmcp instead of brk-rmcp
2026-06-26 22:24:30 -07:00 · 2025-07-17 23:45:01 +02:00 · 2025-07-17 17:29:53 +02:00 · 2025-07-17 16:34:29 +02:00 · 2025-07-17 16:17:21 +02:00 · 2025-07-17 11:39:25 +02:00
111 changed files with 2963 additions and 1297 deletions
@@ -19,6 +19,9 @@ _*
 # Logs
 .log

+# Environment variables/configs
+.env
+
 # Profiling
 profile.json.gz
 flamegraph.svg
@@ -170,9 +170,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"

 [[package]]
 name = "async-compression"
-version = "0.4.25"
+version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4"
+checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8"
 dependencies = [
 "brotli",
 "flate2",
@@ -477,7 +477,7 @@ dependencies = [

 [[package]]
 name = "brk"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_bundler",
 "brk_cli",
@@ -497,7 +497,7 @@ dependencies = [

 [[package]]
 name = "brk_bundler"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_rolldown",
 "log",
@@ -508,7 +508,7 @@ dependencies = [

 [[package]]
 name = "brk_cli"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "bitcoincore-rpc",
 "brk_computer",
@@ -531,7 +531,7 @@ dependencies = [

 [[package]]
 name = "brk_computer"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -548,7 +548,6 @@ dependencies = [
 "derive_deref",
 "either",
 "fjall",
- "jiff",
 "log",
 "rayon",
 "serde",
@@ -558,7 +557,7 @@ dependencies = [

 [[package]]
 name = "brk_core"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "bincode",
 "bitcoin",
@@ -579,7 +578,7 @@ dependencies = [

 [[package]]
 name = "brk_exit"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_logger",
 "ctrlc",
@@ -588,7 +587,7 @@ dependencies = [

 [[package]]
 name = "brk_fetcher"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_core",
 "brk_logger",
@@ -600,7 +599,7 @@ dependencies = [

 [[package]]
 name = "brk_indexer"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -618,15 +617,15 @@ dependencies = [

 [[package]]
 name = "brk_interface"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_computer",
 "brk_core",
 "brk_indexer",
- "brk_rmcp",
 "brk_vec",
 "color-eyre",
 "derive_deref",
+ "rmcp",
 "schemars 1.0.4",
 "serde",
 "serde_json",
@@ -636,7 +635,7 @@ dependencies = [

 [[package]]
 name = "brk_logger"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "color-eyre",
 "env_logger",
@@ -646,17 +645,17 @@ dependencies = [

 [[package]]
 name = "brk_mcp"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "axum",
 "brk_interface",
- "brk_rmcp",
 "log",
+ "rmcp",
 ]

 [[package]]
 name = "brk_parser"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "bitcoin",
 "bitcoincore-rpc",
@@ -669,49 +668,6 @@ dependencies = [
 "zerocopy",
 ]

-[[package]]
-name = "brk_rmcp"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dde96460b07f97f8522cef62fa5e9d5271a5183b13e6d75379ab43c1fb0b106"
-dependencies = [
- "base64 0.22.1",
- "brk_rmcp-macros",
- "bytes",
- "chrono",
- "futures",
- "http",
- "http-body",
- "http-body-util",
- "paste",
- "pin-project-lite",
- "rand 0.9.1",
- "schemars 1.0.4",
- "serde",
- "serde_json",
- "sse-stream",
- "thiserror 2.0.12",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tower-service",
- "tracing",
- "uuid",
-]
-
-[[package]]
-name = "brk_rmcp-macros"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48a3cb1aadff2f3aaaed76d17e71d6b828cafbd3fe9fbaba07d31b8ed10e63c2"
-dependencies = [
- "darling",
- "proc-macro2",
- "quote",
- "serde_json",
- "syn 2.0.104",
-]
-
 [[package]]
 name = "brk_rolldown"
 version = "0.1.1"
@@ -1025,7 +981,7 @@ dependencies = [

 [[package]]
 name = "brk_server"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "axum",
 "bitcoincore-rpc",
@@ -1047,6 +1003,7 @@ dependencies = [
 "log",
 "minreq",
 "serde",
+ "serde_json",
 "tokio",
 "tower-http",
 "tracing",
@@ -1055,7 +1012,7 @@ dependencies = [

 [[package]]
 name = "brk_store"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "brk_core",
 "byteview",
@@ -1077,7 +1034,7 @@ dependencies = [

 [[package]]
 name = "brk_vec"
-version = "0.0.78"
+version = "0.0.81"
 dependencies = [
 "arc-swap",
 "brk_core",
@@ -1159,9 +1116,9 @@ dependencies = [

 [[package]]
 name = "castaway"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
 dependencies = [
 "rustversion",
 ]
@@ -1371,9 +1328,9 @@ dependencies = [

 [[package]]
 name = "crc32fast"
-version = "1.4.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
 dependencies = [
 "cfg-if",
 ]
@@ -1479,8 +1436,18 @@ version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
 dependencies = [
- "darling_core",
- "darling_macro",
+ "darling_core 0.20.11",
+ "darling_macro 0.20.11",
+]
+
+[[package]]
+name = "darling"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a79c4acb1fd5fa3d9304be4c76e031c54d2e92d172a393e24b19a14fe8532fe9"
+dependencies = [
+ "darling_core 0.21.0",
+ "darling_macro 0.21.0",
 ]

 [[package]]
@@ -1497,13 +1464,38 @@ dependencies = [
 "syn 2.0.104",
 ]

+[[package]]
+name = "darling_core"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74875de90daf30eb59609910b84d4d368103aaec4c924824c6799b28f77d6a1d"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.11",
+ "quote",
+ "syn 2.0.104",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e79f8e61677d5df9167cd85265f8e5f64b215cdea3fb55eebc3e622e44c7a146"
+dependencies = [
+ "darling_core 0.21.0",
 "quote",
 "syn 2.0.104",
 ]
@@ -2431,9 +2423,9 @@ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"

 [[package]]
 name = "memmap2"
-version = "0.9.5"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
+checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28"
 dependencies = [
 "libc",
 ]
@@ -3553,9 +3545,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"

 [[package]]
 name = "regress"
-version = "0.10.3"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366"
+checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010"
 dependencies = [
 "hashbrown 0.15.4",
 "memchr",
@@ -3584,6 +3576,49 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "rmcp"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b008b927a85d514699ff304be84c5084598596e6cad4a6f5bc67207715fafe5f"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "chrono",
+ "futures",
+ "http",
+ "http-body",
+ "http-body-util",
+ "paste",
+ "pin-project-lite",
+ "rand 0.9.1",
+ "rmcp-macros",
+ "schemars 1.0.4",
+ "serde",
+ "serde_json",
+ "sse-stream",
+ "thiserror 2.0.12",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower-service",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "rmcp-macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7465280d5f73f2c5c99017a04af407b2262455a149f255ad22f2b0b29087695c"
+dependencies = [
+ "darling 0.21.0",
+ "proc-macro2",
+ "quote",
+ "serde_json",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "rolldown-ariadne"
 version = "0.5.2"
@@ -3618,15 +3653,15 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"

 [[package]]
 name = "rustix"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
 dependencies = [
 "bitflags 2.9.1",
 "errno",
 "libc",
 "linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]

 [[package]]
@@ -3696,6 +3731,7 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
 dependencies = [
+ "chrono",
 "dyn-clone",
 "ref-cast",
 "schemars_derive",
@@ -3884,7 +3920,7 @@ version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
 dependencies = [
- "darling",
+ "darling 0.20.11",
 "proc-macro2",
 "quote",
 "syn 2.0.104",
@@ -4240,9 +4276,9 @@ dependencies = [

 [[package]]
 name = "toml"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0207d6ed1852c2a124c1fbec61621acb8330d2bf969a5d0643131e9affd985a5"
+checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac"
 dependencies = [
 "indexmap 2.10.0",
 "serde",
@@ -4264,18 +4300,18 @@ dependencies = [

 [[package]]
 name = "toml_parser"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5c1c469eda89749d2230d8156a5969a69ffe0d6d01200581cdc6110674d293e"
+checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30"
 dependencies = [
 "winnow",
 ]

 [[package]]
 name = "toml_writer"
-version = "1.0.0"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b679217f2848de74cabd3e8fc5e6d66f40b7da40f8e1954d92054d9010690fd5"
+checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64"

 [[package]]
 name = "tower"
@@ -4574,9 +4610,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"

 [[package]]
 name = "vfs"
-version = "0.12.1"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ec343ec20aa715908fd028a4b8e7c99a349d13143224222e4d61c316d1e7f0a"
+checksum = "9e723b9e1c02a3cf9f9d0de6a4ddb8cdc1df859078902fe0ae0589d615711ae6"
 dependencies = [
 "filetime",
 ]
@@ -4929,9 +4965,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"

 [[package]]
 name = "winnow"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"

 [[package]]
 name = "wit-bindgen-rt"
@@ -4,9 +4,10 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.78"
+package.version = "0.0.81"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
+package.readme = "README.md"

 [profile.release]
 lto = "fat"
@@ -26,22 +27,20 @@ axum = "0.8.4"
 bincode = { version = "2.0.1", features = ["serde"] }
 bitcoin = { version = "0.32.6", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_bundler = { version = "0.0.78", path = "crates/brk_bundler" }
-brk_cli = { version = "0.0.78", path = "crates/brk_cli" }
-brk_computer = { version = "0.0.78", path = "crates/brk_computer" }
-brk_core = { version = "0.0.78", path = "crates/brk_core" }
-brk_exit = { version = "0.0.78", path = "crates/brk_exit" }
-brk_fetcher = { version = "0.0.78", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0.0.78", path = "crates/brk_indexer" }
-brk_interface = { version = "0.0.78", path = "crates/brk_interface" }
-brk_logger = { version = "0.0.78", path = "crates/brk_logger" }
-brk_mcp = { version = "0.0.78", path = "crates/brk_mcp" }
-brk_parser = { version = "0.0.78", path = "crates/brk_parser" }
-brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
-# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
-brk_server = { version = "0.0.78", path = "crates/brk_server" }
-brk_store = { version = "0.0.78", path = "crates/brk_store" }
-brk_vec = { version = "0.0.78", path = "crates/brk_vec" }
+brk_bundler = { version = "0.0.81", path = "crates/brk_bundler" }
+brk_cli = { version = "0.0.81", path = "crates/brk_cli" }
+brk_computer = { version = "0.0.81", path = "crates/brk_computer" }
+brk_core = { version = "0.0.81", path = "crates/brk_core" }
+brk_exit = { version = "0.0.81", path = "crates/brk_exit" }
+brk_fetcher = { version = "0.0.81", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0.0.81", path = "crates/brk_indexer" }
+brk_interface = { version = "0.0.81", path = "crates/brk_interface" }
+brk_logger = { version = "0.0.81", path = "crates/brk_logger" }
+brk_mcp = { version = "0.0.81", path = "crates/brk_mcp" }
+brk_parser = { version = "0.0.81", path = "crates/brk_parser" }
+brk_server = { version = "0.0.81", path = "crates/brk_server" }
+brk_store = { version = "0.0.81", path = "crates/brk_store" }
+brk_vec = { version = "0.0.81", path = "crates/brk_vec" }
 byteview = "=0.6.1"
 clap = { version = "4.5.41", features = ["string"] }
 clap_derive = "4.5.41"
@@ -52,6 +51,7 @@ jiff = "0.2.15"
 log = { version = "0.4.27" }
 minreq = { version = "2.14.0", features = ["https", "serde_json"] }
 rayon = "1.10.0"
+rmcp = { version = "0.3.0", features = ["transport-worker", "transport-streamable-http-server" ] }
 schemars = "1.0.4"
 serde = { version = "1.0.219" }
 serde_bytes = "0.11.17"
@@ -40,14 +40,13 @@ The toolkit can be used in various ways to accommodate as many needs as possible
 - **[Website](https://bitcoinresearchkit.org)** \
  Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
  It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
-  Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
+  Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
 - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
  Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
  Just like the website, it's entirely free, with no authentication or rate-limiting.
 - **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \
  LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \
-  It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. \
-  One-shot output examples: [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) // [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) // [Dashboard 2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc)
+  It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts.
 - **[CLI](https://crates.io/crates/brk_cli)** \
  Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
  The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
@@ -74,7 +73,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
 - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
 - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
 - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
+- [`brk_vec`](https://crates.io/crates/brk_vec): A storeable vec
 - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)

 ## Hosting as a service
@@ -10,8 +10,12 @@
    - pull latest version and notify is out of date
  - _computer_
    - **add rollback of states (in stateful)**
+    - remove configurable format (raw/compressed) and chose sane ones instead
+      - linear reads: compressed (height/date/... + txindex_to_height + txindex_to_version + ...)
+      - random reads: raw (outputindex_to_value + ...)
+    - add prices paid by percentile (percentile cost basis) back
    - add support for per index computation
-    - fix feerate which is always ZERO due to coinbase transaction
+    - fix min feerate which is always ZERO due to coinbase transaction
    - before computing multiple sources check their length, panic if not equal
    - add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
    - add address counts relative to all datasets
@@ -56,6 +60,7 @@
      - add extensions support (.json .csv …)
      - if format instead of extension then don't download file
    - add support for https (rustls)
+    - lru cache
  - _vec_
    - add native lock file support (once it's available in stable rust)
    - improve compressed mode (slow reads)
@@ -1,2 +1 @@
-cargo build --profile profiling
-flamegraph -- ../../target/profiling/brk
+sudo cargo flamegraph --profile profiling --root
@@ -24,7 +24,7 @@ color-eyre = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true }
 tokio = { workspace = true }
-toml = "0.9.1"
+toml = "0.9.2"

 [[bin]]
 name = "brk"
@@ -13,7 +13,6 @@ use clap_derive::Parser;
 use color_eyre::eyre::eyre;
 use serde::{Deserialize, Serialize};

-use crate::services::Services;

 #[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
 #[command(version, about)]
@@ -33,10 +32,6 @@ pub struct Config {
    #[arg(long, value_name = "PATH")]
    brkdir: Option<String>,

-    /// Activated services, default: all, saved
-    #[serde(default, deserialize_with = "default_on_error")]
-    #[arg(short, long)]
-    services: Option<Services>,

    /// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
    #[serde(default, deserialize_with = "default_on_error")]
@@ -129,9 +124,6 @@ impl Config {
                config_saved.brkdir = Some(brkdir);
            }

-            if let Some(services) = config_args.services.take() {
-                config_saved.services = Some(services);
-            }

            if let Some(computation) = config_args.computation.take() {
                config_saved.computation = Some(computation);
@@ -306,15 +298,6 @@ impl Config {
        self.outputsdir().join("hars")
    }

-    pub fn process(&self) -> bool {
-        self.services
-            .is_none_or(|m| m == Services::All || m == Services::Processor)
-    }
-
-    pub fn serve(&self) -> bool {
-        self.services
-            .is_none_or(|m| m == Services::All || m == Services::Server)
-    }

    fn path_cookiefile(&self) -> PathBuf {
        self.rpccookiefile.as_ref().map_or_else(
@@ -1,10 +1,11 @@
+#![doc = include_str!("../README.md")]
+
 use std::{fs, thread};

 use brk_core::{dot_brk_log_path, dot_brk_path};

 mod config;
 mod run;
-mod services;

 use run::*;

@@ -13,18 +13,16 @@ pub fn run() -> color_eyre::Result<()> {
    let config = Config::import()?;

    let rpc = config.rpc()?;
-
    let exit = Exit::new();
-
-    let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
+    let parser = brk_parser::Parser::new(config.blocksdir(), config.brkdir(), rpc);

    let format = config.format();

    let mut indexer = Indexer::forced_import(&config.outputsdir())?;

-    let wait_for_synced_node = || -> color_eyre::Result<()> {
+    let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
        let is_synced = || -> color_eyre::Result<bool> {
-            let info = rpc.get_blockchain_info()?;
+            let info = rpc_client.get_blockchain_info()?;
            Ok(info.headers == info.blocks)
        };

@@ -50,54 +48,41 @@ pub fn run() -> color_eyre::Result<()> {
        .enable_all()
        .build()?
        .block_on(async {
-            let server = if config.serve() {
-                let served_indexer = indexer.clone();
-                let served_computer = computer.clone();
+            let served_indexer = indexer.clone();
+            let served_computer = computer.clone();

-                let server = Server::new(served_indexer, served_computer, config.website())?;
+            let server = Server::new(served_indexer, served_computer, config.website())?;

-                let watch = config.watch();
-                let mcp = config.mcp();
-                let opt = Some(tokio::spawn(async move {
-                    server.serve(watch, mcp).await.unwrap();
-                }));
+            let watch = config.watch();
+            let mcp = config.mcp();

-                sleep(Duration::from_secs(1));
+            tokio::spawn(async move {
+                server.serve(watch, mcp).await.unwrap();
+            });

-                opt
-            } else {
-                None
-            };
+            sleep(Duration::from_secs(1));

-            if config.process() {
-                loop {
-                    wait_for_synced_node()?;
+            loop {
+                wait_for_synced_node(rpc)?;

-                    let block_count = rpc.get_block_count()?;
+                let block_count = rpc.get_block_count()?;

-                    info!("{} blocks found.", block_count + 1);
+                info!("{} blocks found.", block_count + 1);

-                    let starting_indexes =
-                        indexer.index(&parser, rpc, &exit, config.check_collisions())?;
+                let starting_indexes =
+                    indexer.index(&parser, rpc, &exit, config.check_collisions())?;

-                    computer.compute(&mut indexer, starting_indexes, &exit)?;
+                computer.compute(&mut indexer, starting_indexes, &exit)?;

-                    if let Some(delay) = config.delay() {
-                        sleep(Duration::from_secs(delay))
-                    }
+                if let Some(delay) = config.delay() {
+                    sleep(Duration::from_secs(delay))
+                }

-                    info!("Waiting for new blocks...");
+                info!("Waiting for new blocks...");

-                    while block_count == rpc.get_block_count()? {
-                        sleep(Duration::from_secs(1))
-                    }
+                while block_count == rpc.get_block_count()? {
+                    sleep(Duration::from_secs(1))
                }
            }
-
-            if let Some(handle) = server {
-                handle.await.unwrap();
-            }
-
-            Ok(())
        })
 }
@@ -1,23 +0,0 @@
-use clap_derive::{Parser, ValueEnum};
-use serde::{Deserialize, Serialize};
-
-#[derive(
-    Default,
-    Debug,
-    Clone,
-    Copy,
-    Parser,
-    ValueEnum,
-    Serialize,
-    Deserialize,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-)]
-pub enum Services {
-    #[default]
-    All,
-    Processor,
-    Server,
-}
@@ -23,7 +23,6 @@ color-eyre = { workspace = true }
 derive_deref = { workspace = true }
 either = "1.15.0"
 fjall = { workspace = true }
-jiff = { workspace = true }
 log = { workspace = true }
 rayon = { workspace = true }
 serde = { workspace = true }
@@ -1,7 +1,6 @@
 use std::{path::Path, thread};

 use brk_computer::Computer;
-use brk_core::{default_bitcoin_path, default_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -13,7 +12,7 @@ pub fn main() -> color_eyre::Result<()> {

    brk_logger::init(Some(Path::new(".log")));

-    // let bitcoin_dir = default_bitcoin_path();
+    // let bitcoin_dir = brk_core::default_bitcoin_path();
    let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
@@ -26,7 +25,11 @@ pub fn main() -> color_eyre::Result<()> {
    thread::Builder::new()
        .stack_size(256 * 1024 * 1024)
        .spawn(move || -> color_eyre::Result<()> {
-            let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+            let parser = Parser::new(
+                bitcoin_dir.join("blocks"),
+                brk_core::default_brk_path(),
+                rpc,
+            );

            let _outputs_dir = Path::new("/Volumes/WD_BLACK/brk").join("outputs");
            let outputs_dir = _outputs_dir.as_path();
@@ -5,22 +5,11 @@ use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format};
-
-pub mod blocks;
-pub mod cointime;
-pub mod constants;
-pub mod fetched;
-pub mod grouped;
-pub mod indexes;
-pub mod market;
-pub mod mining;
-pub mod stateful;
-pub mod transactions;
-
-pub use indexes::Indexes;
 use log::info;

-use crate::stores::Stores;
+use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
+
+use super::stateful;

 const VERSION: Version = Version::ONE;

@@ -130,7 +119,6 @@ impl Vecs {
        starting_indexes: brk_indexer::Indexes,
        fetcher: Option<&mut Fetcher>,
        exit: &Exit,
-        stores: &mut Stores,
    ) -> color_eyre::Result<()> {
        info!("Computing indexes...");
        let mut starting_indexes = self.indexes.compute(indexer, starting_indexes, exit)?;
@@ -188,7 +176,6 @@ impl Vecs {
            &self.market,
            &mut starting_indexes,
            exit,
-            stores,
        )?;

        self.cointime.compute(
@@ -8,7 +8,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -5,16 +5,13 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};

-use crate::vecs::{
-    fetched,
-    grouped::{ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, Source},
-    stateful, transactions,
-};
-
 use super::{
-    Indexes,
-    grouped::{ComputedVecsFromHeight, VecBuilderOptions},
-    indexes,
+    Indexes, fetched,
+    grouped::{
+        ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
+        Source, VecBuilderOptions,
+    },
+    indexes, stateful, transactions,
 };

 const VERSION: Version = Version::ZERO;
@@ -305,7 +302,7 @@ impl Vecs {
        stateful: &stateful::Vecs,
        exit: &Exit,
    ) -> color_eyre::Result<()> {
-        let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply;
+        let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;

        self.indexes_to_coinblocks_created.compute_all(
            indexer,
@@ -323,7 +320,7 @@ impl Vecs {
        )?;

        let indexes_to_coinblocks_destroyed =
-            &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed;
+            &stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed;

        self.indexes_to_coinblocks_stored.compute_all(
            indexer,
@@ -433,7 +430,7 @@ impl Vecs {

        if let Some(fetched) = fetched {
            let realized_cap = stateful
-                .utxo_vecs
+                .utxo_cohorts
                .all
                .1
                .height_to_realized_cap
@@ -441,7 +438,7 @@ impl Vecs {
                .unwrap();

            let realized_price = stateful
-                .utxo_vecs
+                .utxo_cohorts
                .all
                .1
                .indexes_to_realized_price
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -10,9 +10,10 @@ use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
 use brk_vec::{
    AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex,
+    VecIterator,
 };

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -456,33 +457,34 @@ impl Vecs {
            exit,
        )?;

+        let mut prev = None;
        self.dateindex_to_ohlc_in_cents.compute_transform(
            starting_indexes.dateindex,
            &indexes.dateindex_to_date,
            |(di, d, this)| {
-                let get_prev = || {
-                    this.get_or_read(di, &this.mmap().load())
-                        .unwrap()
-                        .unwrap()
-                        .into_owned()
-                };
+                if prev.is_none() {
+                    let i = di.unwrap_to_usize();
+                    prev.replace(if i > 0 {
+                        this.into_iter().unwrap_get_inner_(i - 1)
+                    } else {
+                        OHLCCents::default()
+                    });
+                }

-                let mut ohlc = if di.unwrap_to_usize() + 100 >= this.len() {
-                    fetcher.get_date(d).unwrap_or_else(|_| get_prev())
-                } else {
-                    get_prev()
-                };
-
-                if let Some(prev) = di.decremented() {
-                    let prev_open = *this
-                        .get_or_read(prev, &this.mmap().load())
-                        .unwrap()
-                        .unwrap()
-                        .close;
+                let ohlc = if di.unwrap_to_usize() + 100 >= this.len()
+                    && let Ok(mut ohlc) = fetcher.get_date(d)
+                {
+                    let prev_open = *prev.as_ref().unwrap().close;
                    *ohlc.open = prev_open;
                    *ohlc.high = (*ohlc.high).max(prev_open);
                    *ohlc.low = (*ohlc.low).min(prev_open);
-                }
+                    ohlc
+                } else {
+                    prev.clone().unwrap()
+                };
+
+                prev.replace(ohlc.clone());
+
                (di, ohlc)
            },
            exit,
@@ -7,7 +7,7 @@ use brk_vec::{
    ComputedVec, ComputedVecFrom2, Format, StoredIndex,
 };

-use crate::vecs::grouped::{EagerVecBuilder, VecBuilderOptions};
+use crate::grouped::{EagerVecBuilder, VecBuilderOptions};

 use super::ComputedType;

@@ -10,7 +10,7 @@ use brk_vec::{
    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };

-use crate::vecs::{Indexes, grouped::ComputedVecBuilder, indexes};
+use crate::{Indexes, grouped::ComputedVecBuilder, indexes};

 use super::{ComputedType, EagerVecBuilder, Source, VecBuilderOptions};

@@ -10,7 +10,7 @@ use brk_vec::{
    AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
 };

-use crate::vecs::{
+use crate::{
    Indexes,
    grouped::{ComputedVecBuilder, Source},
    indexes,
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, EagerVec, Format};

-use crate::vecs::{Indexes, indexes};
+use crate::{Indexes, indexes};

 use super::{ComputedType, EagerVecBuilder, VecBuilderOptions};

@@ -11,7 +11,7 @@ use brk_vec::{
    Format, StoredIndex, VecIterator,
 };

-use crate::vecs::{
+use crate::{
    Indexes, fetched,
    grouped::{ComputedVecBuilder, Source},
    indexes,
@@ -8,10 +8,7 @@ use brk_vec::{
    StoredIndex, VecIterator,
 };

-use crate::{
-    utils::get_percentile,
-    vecs::{Indexes, fetched, grouped::source::Source, indexes},
-};
+use crate::{Indexes, fetched, grouped::source::Source, indexes, utils::get_percentile};

 use super::{ComputedVecsFromDateIndex, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
+use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};

 use super::{Source, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 use super::{ComputedVecsFromHeight, VecBuilderOptions};

@@ -8,7 +8,7 @@ use brk_vec::{
    Format, LazyVecFrom1, StoredIndex, StoredVec,
 };

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 use super::{ComputedVecsFromTxindex, VecBuilderOptions};

@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};

-use crate::vecs::{Indexes, fetched, grouped::Source, indexes};
+use crate::{Indexes, fetched, grouped::Source, indexes};

 #[derive(Clone)]
 pub struct ComputedHeightValueVecs {
@@ -15,8 +15,6 @@ use brk_vec::{
    ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
 };

-use crate::vecs::indexes;
-
 const VERSION: Version = Version::ZERO;

 #[derive(Clone)]
@@ -1206,7 +1204,7 @@ pub struct Indexes {
 }

 impl Indexes {
-    pub fn update_from_height(&mut self, height: Height, indexes: &indexes::Vecs) {
+    pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
        self.indexes.height = height;
        self.dateindex = DateIndex::try_from(
            indexes
@@ -12,20 +12,28 @@ use brk_indexer::Indexer;
 use brk_vec::{Computation, Format};
 use log::info;

+mod all;
+mod blocks;
+mod cointime;
+mod constants;
+mod fetched;
+mod grouped;
+mod indexes;
+mod market;
+mod mining;
+mod stateful;
 mod states;
-mod stores;
+mod transactions;
 mod utils;
-mod vecs;
+
+use indexes::Indexes;

 use states::*;
-use stores::Stores;
-use vecs::Vecs;

 #[derive(Clone)]
 pub struct Computer {
    fetcher: Option<Fetcher>,
-    pub vecs: Vecs,
-    pub stores: Stores,
+    pub vecs: all::Vecs,
 }

 const VERSION: Version = Version::ONE;
@@ -40,7 +48,7 @@ impl Computer {
        format: Format,
    ) -> color_eyre::Result<Self> {
        Ok(Self {
-            vecs: Vecs::import(
+            vecs: all::Vecs::import(
                // TODO: Give self.path, join inside import
                &outputs_dir.join("vecs/computed"),
                VERSION + Version::ZERO,
@@ -49,12 +57,6 @@ impl Computer {
                computation,
                format,
            )?,
-            stores: Stores::import(
-                // TODO: Give self.path, join inside import
-                &outputs_dir.join("stores"),
-                VERSION + Version::ZERO,
-                &indexer.stores.keyspace,
-            )?,
            fetcher,
        })
    }
@@ -68,12 +70,7 @@ impl Computer {
        exit: &Exit,
    ) -> color_eyre::Result<()> {
        info!("Computing...");
-        self.vecs.compute(
-            indexer,
-            starting_indexes,
-            self.fetcher.as_mut(),
-            exit,
-            &mut self.stores,
-        )
+        self.vecs
+            .compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
    }
 }
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, EagerVec, Format, StoredIndex, VecIterator};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes, fetched,
@@ -5,7 +5,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};

-use crate::vecs::grouped::Source;
+use crate::grouped::Source;

 use super::{
    Indexes,
@@ -8,16 +8,14 @@ use brk_vec::{
 };

 use crate::{
-    states::AddressCohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
-        indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, fetched,
+    grouped::{ComputedVecsFromHeight, Source, VecBuilderOptions},
+    indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
    },
+    states::AddressCohortState,
 };

 const VERSION: Version = Version::ZERO;
@@ -9,7 +9,7 @@ use brk_vec::{Computation, Format};
 use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;

-use crate::vecs::{
+use crate::{
    Indexes, fetched, indexes,
    stateful::{
        address_cohort,
@@ -2,7 +2,7 @@ use brk_core::{ByAddressType, Height};
 use brk_vec::VecIterator;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
+use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;

 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct AddressTypeToAddressCount(ByAddressType<usize>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::EagerVec;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
+use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;

 #[derive(Debug, Clone, Deref, DerefMut)]
 pub struct AddressTypeToHeightToAddressCount(ByAddressType<EagerVec<Height, StoredUsize>>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_vec::AnyCollectableVec;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::{
+use crate::{
    Indexes, grouped::ComputedVecsFromHeight, indexes,
    stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
 };
@@ -0,0 +1,24 @@
+use std::collections::BTreeSet;
+
+use brk_core::TypeIndex;
+use derive_deref::{Deref, DerefMut};
+
+use super::ByAddressType;
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
+
+impl Default for AddressTypeToTypeIndexSet {
+    fn default() -> Self {
+        Self(ByAddressType {
+            p2pk65: BTreeSet::default(),
+            p2pk33: BTreeSet::default(),
+            p2pkh: BTreeSet::default(),
+            p2sh: BTreeSet::default(),
+            p2wpkh: BTreeSet::default(),
+            p2wsh: BTreeSet::default(),
+            p2tr: BTreeSet::default(),
+            p2a: BTreeSet::default(),
+        })
+    }
+}
@@ -29,10 +29,6 @@ impl<T> AddressTypeToTypeIndexTree<T> {
            mem::swap(own, other);
        }
    }
-
-    pub fn unwrap(self) -> ByAddressType<BTreeMap<TypeIndex, T>> {
-        self.0
-    }
 }

 impl<T> Default for AddressTypeToTypeIndexTree<T> {
@@ -0,0 +1,57 @@
+use std::mem;
+
+use derive_deref::{Deref, DerefMut};
+
+use super::ByAddressType;
+
+#[derive(Debug, Deref, DerefMut)]
+pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
+
+impl<T> AddressTypeToVec<T> {
+    pub fn merge(mut self, mut other: Self) -> Self {
+        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
+        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
+        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
+        Self::merge_(&mut self.p2sh, &mut other.p2sh);
+        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
+        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
+        Self::merge_(&mut self.p2tr, &mut other.p2tr);
+        Self::merge_(&mut self.p2a, &mut other.p2a);
+        self
+    }
+
+    pub fn merge_mut(&mut self, mut other: Self) {
+        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
+        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
+        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
+        Self::merge_(&mut self.p2sh, &mut other.p2sh);
+        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
+        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
+        Self::merge_(&mut self.p2tr, &mut other.p2tr);
+        Self::merge_(&mut self.p2a, &mut other.p2a);
+    }
+
+    fn merge_(own: &mut Vec<T>, other: &mut Vec<T>) {
+        if own.len() >= other.len() {
+            own.append(other);
+        } else {
+            other.append(own);
+            mem::swap(own, other);
+        }
+    }
+}
+
+impl<T> Default for AddressTypeToVec<T> {
+    fn default() -> Self {
+        Self(ByAddressType {
+            p2pk65: vec![],
+            p2pk33: vec![],
+            p2pkh: vec![],
+            p2sh: vec![],
+            p2wpkh: vec![],
+            p2wsh: vec![],
+            p2tr: vec![],
+            p2a: vec![],
+        })
+    }
+}
@@ -11,16 +11,13 @@ use brk_vec::{
 };

 use crate::{
-    states::CohortState,
-    vecs::{
-        Indexes, fetched,
-        grouped::{
-            ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex,
-            ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight,
-            Source, VecBuilderOptions,
-        },
-        indexes, market,
+    Indexes, fetched,
+    grouped::{
+        ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex,
+        ComputedVecsFromDateIndex, ComputedVecsFromHeight, Source, VecBuilderOptions,
    },
+    indexes, market,
+    states::CohortState,
 };

 const VERSION: Version = Version::ZERO;
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use brk_core::Height;
 use derive_deref::{Deref, DerefMut};

-use crate::vecs::stateful::AddressTypeToVec;
+use crate::stateful::AddressTypeToVec;

 #[derive(Debug, Default, Deref, DerefMut)]
 pub struct HeightToAddressTypeToVec<T>(pub BTreeMap<Height, AddressTypeToVec<T>>);
@@ -3,7 +3,7 @@ use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec};

-use crate::vecs::{Indexes, fetched, indexes, market};
+use crate::{Indexes, fetched, indexes, market};

 pub trait DynCohortVecs: Send + Sync {
    fn starting_height(&self) -> Height;
@@ -6,13 +6,10 @@ use brk_indexer::Indexer;
 use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};

 use crate::{
-    UTXOCohortState,
-    vecs::{
-        Indexes, fetched, indexes, market,
-        stateful::{
-            common,
-            r#trait::{CohortVecs, DynCohortVecs},
-        },
+    Indexes, UTXOCohortState, fetched, indexes, market,
+    stateful::{
+        common,
+        r#trait::{CohortVecs, DynCohortVecs},
    },
 };

@@ -11,8 +11,9 @@ use derive_deref::{Deref, DerefMut};
 use rayon::prelude::*;

 use crate::{
+    Indexes, fetched, indexes,
+    stateful::r#trait::DynCohortVecs,
    states::{BlockState, Transacted},
-    vecs::{Indexes, fetched, indexes, stateful::r#trait::DynCohortVecs},
 };

 use super::{r#trait::CohortVecs, utxo_cohort};
@@ -1505,13 +1506,16 @@ impl Vecs {
        let last_timestamp = chain_state.last().unwrap().timestamp;
        let current_price = chain_state.last().unwrap().price;

+        let chain_state_len = chain_state.len();
+
        height_to_sent.into_iter().for_each(|(height, sent)| {
            chain_state[height.unwrap_to_usize()].supply -= &sent.spendable_supply;

            let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
+
            let prev_price = block_state.price;

-            let blocks_old = chain_state.len() - 1 - height.unwrap_to_usize();
+            let blocks_old = chain_state_len - 1 - height.unwrap_to_usize();

            let days_old = last_timestamp.difference_in_days_between(block_state.timestamp);
            let days_old_float =
@@ -0,0 +1,54 @@
+use brk_core::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
+
+#[derive(Debug)]
+pub enum WithAddressDataSource<T> {
+    New(T),
+    FromLoadedAddressDataVec((LoadedAddressIndex, T)),
+    FromEmptyAddressDataVec((EmptyAddressIndex, T)),
+}
+
+impl<T> WithAddressDataSource<T> {
+    pub fn is_new(&self) -> bool {
+        matches!(self, Self::New(_))
+    }
+
+    pub fn is_from_emptyaddressdata(&self) -> bool {
+        matches!(self, Self::FromEmptyAddressDataVec(_))
+    }
+
+    pub fn deref_mut(&mut self) -> &mut T {
+        match self {
+            Self::New(v) => v,
+            Self::FromLoadedAddressDataVec((_, v)) => v,
+            Self::FromEmptyAddressDataVec((_, v)) => v,
+        }
+    }
+}
+
+impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
+    fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
+        match value {
+            WithAddressDataSource::New(v) => Self::New(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
+                Self::FromLoadedAddressDataVec((i, v.into()))
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
+                Self::FromEmptyAddressDataVec((i, v.into()))
+            }
+        }
+    }
+}
+
+impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
+    fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
+        match value {
+            WithAddressDataSource::New(v) => Self::New(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec((i, v)) => {
+                Self::FromLoadedAddressDataVec((i, v.into()))
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec((i, v)) => {
+                Self::FromEmptyAddressDataVec((i, v.into()))
+            }
+        }
+    }
+}
@@ -1,6 +1,6 @@
 use std::path::Path;

-use brk_core::{AddressData, Dollars, Height, Result, Sats};
+use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats};

 use crate::SupplyState;

@@ -35,7 +35,7 @@ impl AddressCohortState {
    #[allow(clippy::too_many_arguments)]
    pub fn send(
        &mut self,
-        addressdata: &mut AddressData,
+        addressdata: &mut LoadedAddressData,
        value: Sats,
        current_price: Option<Dollars>,
        prev_price: Option<Dollars>,
@@ -72,7 +72,12 @@ impl AddressCohortState {
        Ok(())
    }

-    pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
+    pub fn receive(
+        &mut self,
+        address_data: &mut LoadedAddressData,
+        value: Sats,
+        price: Option<Dollars>,
+    ) {
        let compute_price = price.is_some();

        let prev_realized_price = compute_price.then(|| address_data.realized_price());
@@ -96,7 +101,7 @@ impl AddressCohortState {
        );
    }

-    pub fn add(&mut self, addressdata: &AddressData) {
+    pub fn add(&mut self, addressdata: &LoadedAddressData) {
        self.address_count += 1;
        self.inner.increment_(
            &addressdata.into(),
@@ -105,7 +110,7 @@ impl AddressCohortState {
        );
    }

-    pub fn subtract(&mut self, addressdata: &AddressData) {
+    pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
        self.address_count = self.address_count.checked_sub(1).unwrap();
        self.inner.decrement_(
            &addressdata.into(),
@@ -223,17 +223,15 @@ impl CohortState {
        let update_state =
            |price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
                match price.cmp(&current_price) {
-                    Ordering::Equal => {
-                        state.supply_even += sats;
-                    }
                    Ordering::Less => {
                        state.supply_in_profit += sats;
                        if price > Dollars::ZERO && current_price > Dollars::ZERO {
                            let diff = current_price.checked_sub(price).unwrap();
-                            if diff <= Dollars::ZERO {
-                                dbg!(price, current_price, diff, sats);
-                                panic!();
-                            }
+                            // Add back once in a while to verify, but generally not needed
+                            // if diff <= Dollars::ZERO {
+                            //     dbg!(price, current_price, diff, sats);
+                            //     panic!();
+                            // }
                            state.unrealized_profit += diff * sats;
                        }
                    }
@@ -241,13 +239,17 @@ impl CohortState {
                        state.supply_in_loss += sats;
                        if price > Dollars::ZERO && current_price > Dollars::ZERO {
                            let diff = price.checked_sub(current_price).unwrap();
-                            if diff <= Dollars::ZERO {
-                                dbg!(price, current_price, diff, sats);
-                                panic!();
-                            }
+                            // Add back once in a while to verify, but generally not needed
+                            // if diff <= Dollars::ZERO {
+                            //     dbg!(price, current_price, diff, sats);
+                            //     panic!();
+                            // }
                            state.unrealized_loss += diff * sats;
                        }
                    }
+                    Ordering::Equal => {
+                        state.supply_even += sats;
+                    }
                }
            };

@@ -1,6 +1,6 @@
 use std::ops::{Add, AddAssign, SubAssign};

-use brk_core::{AddressData, CheckedSub, Sats};
+use brk_core::{CheckedSub, LoadedAddressData, Sats};
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

@@ -40,8 +40,8 @@ impl SubAssign<&SupplyState> for SupplyState {
    }
 }

-impl From<&AddressData> for SupplyState {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for SupplyState {
+    fn from(value: &LoadedAddressData) -> Self {
        Self {
            utxos: value.outputs_len as usize,
            value: value.amount(),
@@ -9,7 +9,7 @@ use brk_store::{AnyStore, Store};
 use fjall::{PersistMode, TransactionalKeyspace};
 use log::info;

-use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
+use crate::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};

 const VERSION: Version = Version::ZERO;

@@ -597,12 +597,6 @@ impl Stores {
            .map_err(|e| e.into())
    }

-    pub fn rotate_memtables(&self) {
-        self.as_slice()
-            .into_iter()
-            .for_each(|store| store.rotate_memtable());
-    }
-
    pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
        [
            &self.p2aaddressindex_to_addressdata,
@@ -11,17 +11,13 @@ use brk_vec::{
    ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
 };

-use crate::vecs::grouped::Source;
-
-use super::{
-    Indexes, fetched,
-    grouped::{
-        ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
-        ComputedVecsFromTxindex, VecBuilderOptions,
-    },
-    indexes,
+use crate::grouped::{
+    ComputedValueVecsFromHeight, ComputedValueVecsFromTxindex, ComputedVecsFromHeight,
+    ComputedVecsFromTxindex, Source, VecBuilderOptions,
 };

+use super::{Indexes, fetched, indexes};
+
 const VERSION: Version = Version::ZERO;

 #[derive(Clone)]
@@ -1,47 +0,0 @@
-use std::{collections::BTreeSet, mem};
-
-use brk_core::TypeIndex;
-use derive_deref::{Deref, DerefMut};
-
-use super::ByAddressType;
-
-#[derive(Debug, Deref, DerefMut)]
-pub struct AddressTypeToTypeIndexSet(ByAddressType<BTreeSet<TypeIndex>>);
-
-impl AddressTypeToTypeIndexSet {
-    pub fn merge(mut self, mut other: Self) -> Self {
-        Self::merge_(&mut self.p2pk65, &mut other.p2pk65);
-        Self::merge_(&mut self.p2pk33, &mut other.p2pk33);
-        Self::merge_(&mut self.p2pkh, &mut other.p2pkh);
-        Self::merge_(&mut self.p2sh, &mut other.p2sh);
-        Self::merge_(&mut self.p2wpkh, &mut other.p2wpkh);
-        Self::merge_(&mut self.p2wsh, &mut other.p2wsh);
-        Self::merge_(&mut self.p2tr, &mut other.p2tr);
-        Self::merge_(&mut self.p2a, &mut other.p2a);
-        self
-    }
-
-    fn merge_(own: &mut BTreeSet<TypeIndex>, other: &mut BTreeSet<TypeIndex>) {
-        if own.len() >= other.len() {
-            own.append(other);
-        } else {
-            other.append(own);
-            mem::swap(own, other);
-        }
-    }
-}
-
-impl Default for AddressTypeToTypeIndexSet {
-    fn default() -> Self {
-        Self(ByAddressType {
-            p2pk65: BTreeSet::default(),
-            p2pk33: BTreeSet::default(),
-            p2pkh: BTreeSet::default(),
-            p2sh: BTreeSet::default(),
-            p2wpkh: BTreeSet::default(),
-            p2wsh: BTreeSet::default(),
-            p2tr: BTreeSet::default(),
-            p2a: BTreeSet::default(),
-        })
-    }
-}
@@ -1,21 +0,0 @@
-use derive_deref::{Deref, DerefMut};
-
-use super::ByAddressType;
-
-#[derive(Debug, Deref, DerefMut)]
-pub struct AddressTypeToVec<T>(ByAddressType<Vec<T>>);
-
-impl<T> Default for AddressTypeToVec<T> {
-    fn default() -> Self {
-        Self(ByAddressType {
-            p2pk65: vec![],
-            p2pk33: vec![],
-            p2pkh: vec![],
-            p2sh: vec![],
-            p2wpkh: vec![],
-            p2wsh: vec![],
-            p2tr: vec![],
-            p2a: vec![],
-        })
-    }
-}
@@ -1,62 +0,0 @@
-use brk_core::{AddressData, EmptyAddressData};
-
-#[derive(Debug)]
-pub enum WithAddressDataSource<T> {
-    New(T),
-    FromAddressDataStore(T),
-    FromEmptyAddressDataStore(T),
-}
-
-impl<T> WithAddressDataSource<T> {
-    pub fn is_new(&self) -> bool {
-        matches!(self, Self::New(_))
-    }
-
-    pub fn is_from_addressdata(&self) -> bool {
-        matches!(self, Self::FromAddressDataStore(_))
-    }
-
-    pub fn is_from_emptyaddressdata(&self) -> bool {
-        matches!(self, Self::FromEmptyAddressDataStore(_))
-    }
-
-    pub fn deref(&self) -> &T {
-        match self {
-            Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
-        }
-    }
-
-    pub fn deref_mut(&mut self) -> &mut T {
-        match self {
-            Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
-        }
-    }
-}
-
-impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<AddressData> {
-    fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
-        match value {
-            WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
-                Self::FromEmptyAddressDataStore(v.into())
-            }
-        }
-    }
-}
-
-impl From<WithAddressDataSource<AddressData>> for WithAddressDataSource<EmptyAddressData> {
-    fn from(value: WithAddressDataSource<AddressData>) -> Self {
-        match value {
-            WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
-                Self::FromEmptyAddressDataStore(v.into())
-            }
-        }
-    }
-}
@@ -19,6 +19,10 @@ pub struct ByAddressType<T> {
 }

 impl<T> ByAddressType<T> {
+    pub fn get_unwrap(&self, address_type: OutputType) -> &T {
+        self.get(address_type).unwrap()
+    }
+
    pub fn get(&self, address_type: OutputType) -> Option<&T> {
        match address_type {
            OutputType::P2PK65 => Some(&self.p2pk65),
@@ -0,0 +1,5 @@
+#[derive(Debug, Default)]
+pub struct ByAnyAddress<T> {
+    pub loaded: T,
+    pub empty: T,
+}
@@ -15,12 +15,11 @@ pub enum GroupFilter {
 impl GroupFilter {
    pub fn contains(&self, value: usize) -> bool {
        match self {
-            GroupFilter::All => true,
+            GroupFilter::Range(r) => r.contains(&value),
            GroupFilter::LowerThan(max) => *max > value,
            GroupFilter::GreaterOrEqual(min) => *min <= value,
-            GroupFilter::Range(r) => r.contains(&value),
-            GroupFilter::Epoch(_) => false,
-            GroupFilter::Type(_) => false,
+            GroupFilter::All => true,
+            GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
        }
    }

@@ -28,24 +27,22 @@ impl GroupFilter {
        match self {
            GroupFilter::All => true,
            GroupFilter::LowerThan(max) => match other {
-                GroupFilter::All => false,
                GroupFilter::LowerThan(max2) => max >= max2,
                GroupFilter::Range(range) => range.end <= *max,
-                GroupFilter::GreaterOrEqual(_) => false,
-                GroupFilter::Epoch(_) => false,
-                GroupFilter::Type(_) => false,
+                GroupFilter::All
+                | GroupFilter::GreaterOrEqual(_)
+                | GroupFilter::Epoch(_)
+                | GroupFilter::Type(_) => false,
            },
            GroupFilter::GreaterOrEqual(min) => match other {
-                GroupFilter::All => false,
-                GroupFilter::LowerThan(_) => false,
                GroupFilter::Range(range) => range.start >= *min,
                GroupFilter::GreaterOrEqual(min2) => min <= min2,
-                GroupFilter::Epoch(_) => false,
-                GroupFilter::Type(_) => false,
+                GroupFilter::All
+                | GroupFilter::LowerThan(_)
+                | GroupFilter::Epoch(_)
+                | GroupFilter::Type(_) => false,
            },
-            GroupFilter::Range(_) => false,
-            GroupFilter::Epoch(_) => false,
-            GroupFilter::Type(_) => false,
+            GroupFilter::Range(_) | GroupFilter::Epoch(_) | GroupFilter::Type(_) => false,
        }
    }
 }
@@ -2,6 +2,7 @@ mod address;
 mod by_address_type;
 mod by_age_range;
 mod by_amount_range;
+mod by_any_address;
 mod by_epoch;
 mod by_ge_amount;
 mod by_lt_amount;
@@ -18,6 +19,7 @@ pub use address::*;
 pub use by_address_type::*;
 pub use by_age_range::*;
 pub use by_amount_range::*;
+pub use by_any_address::*;
 pub use by_epoch::*;
 pub use by_ge_amount::*;
 pub use by_lt_amount::*;
@@ -0,0 +1,61 @@
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{
+    TypeIndex,
+    structs::{EmptyAddressIndex, LoadedAddressIndex},
+};
+
+const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000;
+
+#[derive(
+    Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout,
+)]
+pub struct AnyAddressIndex(TypeIndex);
+
+impl AnyAddressIndex {
+    pub fn to_enum(&self) -> AnyAddressDataIndexEnum {
+        AnyAddressDataIndexEnum::from(*self)
+    }
+}
+
+impl From<LoadedAddressIndex> for AnyAddressIndex {
+    fn from(value: LoadedAddressIndex) -> Self {
+        if u32::from(value) >= MIN_EMPTY_INDEX {
+            panic!("")
+        }
+        Self(*value)
+    }
+}
+
+impl From<EmptyAddressIndex> for AnyAddressIndex {
+    fn from(value: EmptyAddressIndex) -> Self {
+        Self(*value + MIN_EMPTY_INDEX)
+    }
+}
+
+impl Serialize for AnyAddressIndex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_enum().serialize(serializer)
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub enum AnyAddressDataIndexEnum {
+    Loaded(LoadedAddressIndex),
+    Empty(EmptyAddressIndex),
+}
+
+impl From<AnyAddressIndex> for AnyAddressDataIndexEnum {
+    fn from(value: AnyAddressIndex) -> Self {
+        let uvalue = u32::from(value.0);
+        if uvalue >= MIN_EMPTY_INDEX {
+            Self::Empty(EmptyAddressIndex::from(uvalue - MIN_EMPTY_INDEX))
+        } else {
+            Self::Loaded(LoadedAddressIndex::from(value.0))
+        }
+    }
+}
@@ -1,22 +1,23 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

-use crate::{AddressData, Sats};
+use crate::{LoadedAddressData, Sats};

-#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)]
+#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
 pub struct EmptyAddressData {
    pub transfered: Sats,
 }

-impl From<AddressData> for EmptyAddressData {
-    fn from(value: AddressData) -> Self {
+impl From<LoadedAddressData> for EmptyAddressData {
+    fn from(value: LoadedAddressData) -> Self {
        Self::from(&value)
    }
 }

-impl From<&AddressData> for EmptyAddressData {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for EmptyAddressData {
+    fn from(value: &LoadedAddressData) -> Self {
        if value.sent != value.received {
            dbg!(&value);
            panic!("Trying to convert not empty wallet to empty !");
@@ -0,0 +1,69 @@
+use std::ops::Add;
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    Default,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Deref,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct EmptyAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for EmptyAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for EmptyAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+impl From<u32> for EmptyAddressIndex {
+    fn from(value: u32) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+
+impl From<EmptyAddressIndex> for usize {
+    fn from(value: EmptyAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+
+impl Add<usize> for EmptyAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<EmptyAddressIndex> for EmptyAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for EmptyAddressIndex {
+    fn to_string() -> &'static str {
+        "emptyaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["emptyaddr", "emptyaddressindex"]
+    }
+}
@@ -31,7 +31,7 @@ impl From<TypeIndex> for EmptyOutputIndex {
 }
 impl From<EmptyOutputIndex> for usize {
    fn from(value: EmptyOutputIndex) -> Self {
-        Self::from(*value)
+        Self::from(value.0)
    }
 }
 impl From<usize> for EmptyOutputIndex {
@@ -42,7 +42,7 @@ impl From<usize> for EmptyOutputIndex {
 impl Add<usize> for EmptyOutputIndex {
    type Output = Self;
    fn add(self, rhs: usize) -> Self::Output {
-        Self(*self + rhs)
+        Self(self.0 + rhs)
    }
 }
 impl CheckedSub<EmptyOutputIndex> for EmptyOutputIndex {
@@ -1,18 +1,21 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

 use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats};

-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
 #[repr(C)]
-pub struct AddressData {
+pub struct LoadedAddressData {
    pub sent: Sats,
    pub received: Sats,
    pub realized_cap: Dollars,
    pub outputs_len: u32,
+    padding: u32,
 }

-impl AddressData {
+impl LoadedAddressData {
    pub fn amount(&self) -> Sats {
        (u64::from(self.received) - u64::from(self.sent)).into()
    }
@@ -55,23 +58,24 @@ impl AddressData {
    }
 }

-impl From<EmptyAddressData> for AddressData {
+impl From<EmptyAddressData> for LoadedAddressData {
    fn from(value: EmptyAddressData) -> Self {
        Self::from(&value)
    }
 }
-impl From<&EmptyAddressData> for AddressData {
+impl From<&EmptyAddressData> for LoadedAddressData {
    fn from(value: &EmptyAddressData) -> Self {
        Self {
            sent: value.transfered,
            received: value.transfered,
            realized_cap: Dollars::ZERO,
            outputs_len: 0,
+            padding: 0,
        }
    }
 }

-impl From<ByteView> for AddressData {
+impl From<ByteView> for LoadedAddressData {
    fn from(value: ByteView) -> Self {
        Self {
            // MUST be same order as impl From<&AddressData> for ByteView
@@ -79,16 +83,17 @@ impl From<ByteView> for AddressData {
            received: Sats::read_from_bytes(&value[8..16]).unwrap(),
            realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(),
            outputs_len: u32::read_from_bytes(&value[24..]).unwrap(),
+            padding: 0,
        }
    }
 }
-impl From<AddressData> for ByteView {
-    fn from(value: AddressData) -> Self {
+impl From<LoadedAddressData> for ByteView {
+    fn from(value: LoadedAddressData) -> Self {
        Self::from(&value)
    }
 }
-impl From<&AddressData> for ByteView {
-    fn from(value: &AddressData) -> Self {
+impl From<&LoadedAddressData> for ByteView {
+    fn from(value: &LoadedAddressData) -> Self {
        Self::new(
            &[
                value.sent.as_bytes(),
@@ -0,0 +1,67 @@
+use std::ops::Add;
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Deref,
+    Default,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct LoadedAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for LoadedAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for LoadedAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+impl From<LoadedAddressIndex> for usize {
+    fn from(value: LoadedAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+impl From<LoadedAddressIndex> for u32 {
+    fn from(value: LoadedAddressIndex) -> Self {
+        u32::from(value.0)
+    }
+}
+impl Add<usize> for LoadedAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<LoadedAddressIndex> for LoadedAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for LoadedAddressIndex {
+    fn to_string() -> &'static str {
+        "loadedaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["loadedaddr", "loadedaddressindex"]
+    }
+}
@@ -1,6 +1,6 @@
 mod addressbytes;
 mod addressbyteshash;
-mod addressdata;
+mod anyaddressindex;
 mod bitcoin;
 mod blockhash;
 mod blockhashprefix;
@@ -11,11 +11,14 @@ mod decadeindex;
 mod difficultyepoch;
 mod dollars;
 mod emptyaddressdata;
+mod emptyaddressindex;
 mod emptyoutputindex;
 mod feerate;
 mod halvingepoch;
 mod height;
 mod inputindex;
+mod loadedaddressdata;
+mod loadedaddressindex;
 mod monthindex;
 mod ohlc;
 mod opreturnindex;
@@ -58,7 +61,7 @@ mod yearindex;

 pub use addressbytes::*;
 pub use addressbyteshash::*;
-pub use addressdata::*;
+pub use anyaddressindex::*;
 pub use bitcoin::*;
 pub use blockhash::*;
 pub use blockhashprefix::*;
@@ -69,11 +72,14 @@ pub use decadeindex::*;
 pub use difficultyepoch::*;
 pub use dollars::*;
 pub use emptyaddressdata::*;
+pub use emptyaddressindex::*;
 pub use emptyoutputindex::*;
 pub use feerate::*;
 pub use halvingepoch::*;
 pub use height::*;
 pub use inputindex::*;
+pub use loadedaddressdata::*;
+pub use loadedaddressindex::*;
 pub use monthindex::*;
 pub use ohlc::*;
 pub use opreturnindex::*;
@@ -73,6 +73,12 @@ impl From<TypeIndex> for usize {
    }
 }

+impl Add<u32> for TypeIndex {
+    type Output = Self;
+    fn add(self, rhs: u32) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
 impl Add<usize> for TypeIndex {
    type Output = Self;
    fn add(self, rhs: usize) -> Self::Output {
@@ -7,7 +7,7 @@ pub fn setrlimit() -> io::Result<()> {

    rlimit::setrlimit(
        Resource::NOFILE,
-        no_file_limit.0.max(250_000),
+        no_file_limit.0.max(10_000),
        no_file_limit.1,
    )?;

@@ -1,6 +1,10 @@
-use std::{path::Path, time::Instant};
+use std::{
+    fs,
+    path::Path,
+    thread::sleep,
+    time::{Duration, Instant},
+};

-use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_parser::Parser;
@@ -8,11 +12,12 @@ use brk_parser::Parser;
 fn main() -> color_eyre::Result<()> {
    color_eyre::install()?;

-    let i = Instant::now();
-
    brk_logger::init(Some(Path::new(".log")));

-    let bitcoin_dir = default_bitcoin_path();
+    // let bitcoin_dir = brk_core::default_bitcoin_path();
+    let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
+    // let outputs_dir = brk_core::default_brk_path().join("outputs");
+    let outputs_dir = Path::new("/Volumes/WD_BLACK/brk/outputs");

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
        "http://localhost:8332",
@@ -20,15 +25,17 @@ fn main() -> color_eyre::Result<()> {
    )?));
    let exit = Exit::new();

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), outputs_dir.to_path_buf(), rpc);

-    let outputs = Path::new("../../_outputs");
+    fs::create_dir_all(outputs_dir)?;

-    let mut indexer = Indexer::forced_import(outputs)?;
+    let mut indexer = Indexer::forced_import(outputs_dir)?;

-    indexer.index(&parser, rpc, &exit, true)?;
+    loop {
+        let i = Instant::now();
+        indexer.index(&parser, rpc, &exit, false)?;
+        dbg!(i.elapsed());

-    dbg!(i.elapsed());
-
-    Ok(())
+        sleep(Duration::from_secs(5 * 60));
+    }
 }
@@ -15,7 +15,7 @@ use bitcoin::{Transaction, TxIn, TxOut};
 use brk_exit::Exit;
 use brk_parser::Parser;
 use brk_store::AnyStore;
-use brk_vec::{AnyVec, VecIterator};
+use brk_vec::{AnyVec, Mmap, VecIterator};
 use color_eyre::eyre::{ContextCompat, eyre};
 use log::{error, info};
 use rayon::prelude::*;
@@ -40,6 +40,7 @@ pub struct Indexer {
 impl Indexer {
    pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
        setrlimit()?;
+
        Ok(Self {
            vecs: Vecs::forced_import(&outputs_dir.join("vecs/indexed"), VERSION + Version::ZERO)?,
            stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?,
@@ -88,9 +89,9 @@ impl Indexer {
                                height: Height,
                                rem: bool,
                                exit: &Exit|
-         -> color_eyre::Result<()> {
+         -> color_eyre::Result<bool> {
            if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() {
-                return Ok(());
+                return Ok(false);
            }

            info!("Exporting...");
@@ -98,15 +99,88 @@ impl Indexer {
            stores.commit(height)?;
            vecs.flush(height)?;
            exit.release();
-            Ok(())
+            Ok(true)
        };

+        let mut txindex_to_first_outputindex_mmap_opt = None;
+        let mut p2pk65addressindex_to_p2pk65bytes_mmap_opt = None;
+        let mut p2pk33addressindex_to_p2pk33bytes_mmap_opt = None;
+        let mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt = None;
+        let mut p2shaddressindex_to_p2shbytes_mmap_opt = None;
+        let mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt = None;
+        let mut p2wshaddressindex_to_p2wshbytes_mmap_opt = None;
+        let mut p2traddressindex_to_p2trbytes_mmap_opt = None;
+        let mut p2aaddressindex_to_p2abytes_mmap_opt = None;
+
+        let reset_mmaps_options =
+            |vecs: &mut Vecs,
+             txindex_to_first_outputindex_mmap_opt: &mut Option<Mmap>,
+             p2pk65addressindex_to_p2pk65bytes_mmap_opt: &mut Option<Mmap>,
+             p2pk33addressindex_to_p2pk33bytes_mmap_opt: &mut Option<Mmap>,
+             p2pkhaddressindex_to_p2pkhbytes_mmap_opt: &mut Option<Mmap>,
+             p2shaddressindex_to_p2shbytes_mmap_opt: &mut Option<Mmap>,
+             p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt: &mut Option<Mmap>,
+             p2wshaddressindex_to_p2wshbytes_mmap_opt: &mut Option<Mmap>,
+             p2traddressindex_to_p2trbytes_mmap_opt: &mut Option<Mmap>,
+             p2aaddressindex_to_p2abytes_mmap_opt: &mut Option<Mmap>| {
+                txindex_to_first_outputindex_mmap_opt
+                    .replace(vecs.txindex_to_first_outputindex.create_mmap().unwrap());
+                p2pk65addressindex_to_p2pk65bytes_mmap_opt.replace(
+                    vecs.p2pk65addressindex_to_p2pk65bytes
+                        .create_mmap()
+                        .unwrap(),
+                );
+                p2pk33addressindex_to_p2pk33bytes_mmap_opt.replace(
+                    vecs.p2pk33addressindex_to_p2pk33bytes
+                        .create_mmap()
+                        .unwrap(),
+                );
+                p2pkhaddressindex_to_p2pkhbytes_mmap_opt
+                    .replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_mmap().unwrap());
+                p2shaddressindex_to_p2shbytes_mmap_opt
+                    .replace(vecs.p2shaddressindex_to_p2shbytes.create_mmap().unwrap());
+                p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.replace(
+                    vecs.p2wpkhaddressindex_to_p2wpkhbytes
+                        .create_mmap()
+                        .unwrap(),
+                );
+                p2wshaddressindex_to_p2wshbytes_mmap_opt
+                    .replace(vecs.p2wshaddressindex_to_p2wshbytes.create_mmap().unwrap());
+                p2traddressindex_to_p2trbytes_mmap_opt
+                    .replace(vecs.p2traddressindex_to_p2trbytes.create_mmap().unwrap());
+                p2aaddressindex_to_p2abytes_mmap_opt
+                    .replace(vecs.p2aaddressindex_to_p2abytes.create_mmap().unwrap());
+            };
+
+        reset_mmaps_options(
+            vecs,
+            &mut txindex_to_first_outputindex_mmap_opt,
+            &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt,
+            &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt,
+            &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt,
+            &mut p2shaddressindex_to_p2shbytes_mmap_opt,
+            &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt,
+            &mut p2wshaddressindex_to_p2wshbytes_mmap_opt,
+            &mut p2traddressindex_to_p2trbytes_mmap_opt,
+            &mut p2aaddressindex_to_p2abytes_mmap_opt,
+        );
+
        parser.parse(start, end).iter().try_for_each(
            |(height, block, blockhash)| -> color_eyre::Result<()> {
                info!("Indexing block {height}...");

                idxs.height = height;

+                let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_mmap_opt.as_ref().unwrap();
+                let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_mmap_opt.as_ref().unwrap();
+                let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_mmap_opt.as_ref().unwrap();
+                let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_mmap_opt.as_ref().unwrap();
+                let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_mmap_opt.as_ref().unwrap();
+                let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.as_ref().unwrap();
+                let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_mmap_opt.as_ref().unwrap();
+                let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_mmap_opt.as_ref().unwrap();
+                let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_mmap_opt.as_ref().unwrap();
+
                // Used to check rapidhash collisions
                let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO);

@@ -166,9 +240,6 @@ impl Indexer {
                        });

                    let input_source_vec_handle = scope.spawn(|| {
-                        let txindex_to_first_outputindex_mmap = vecs
-                            .txindex_to_first_outputindex.mmap().load();
-
                        let inputs = block
                            .txdata
                            .iter()
@@ -211,7 +282,7 @@ impl Indexer {

                                let vout = Vout::from(outpoint.vout);

-                                let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)?
+                                let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_mmap)?
                                    .context("Expect outputindex to not be none")
                                    .inspect_err(|_| {
                                        dbg!(outpoint.txid, prev_txindex, vout);
@@ -240,16 +311,6 @@ impl Indexer {
                            })
                    });

-                    let p2pk65addressindex_to_p2pk65bytes_mmap = vecs
-                        .p2pk65addressindex_to_p2pk65bytes.mmap().load();
-                    let p2pk33addressindex_to_p2pk33bytes_mmap = vecs.p2pk33addressindex_to_p2pk33bytes.mmap().load();
-                    let p2pkhaddressindex_to_p2pkhbytes_mmap = vecs.p2pkhaddressindex_to_p2pkhbytes.mmap().load();
-                    let p2shaddressindex_to_p2shbytes_mmap = vecs.p2shaddressindex_to_p2shbytes.mmap().load();
-                    let p2wpkhaddressindex_to_p2wpkhbytes_mmap = vecs.p2wpkhaddressindex_to_p2wpkhbytes.mmap().load();
-                    let p2wshaddressindex_to_p2wshbytes_mmap = vecs.p2wshaddressindex_to_p2wshbytes.mmap().load();
-                    let p2traddressindex_to_p2trbytes_mmap = vecs.p2traddressindex_to_p2trbytes.mmap().load();
-                    let p2aaddressindex_to_p2abytes_mmap = vecs.p2aaddressindex_to_p2abytes.mmap().load();
-
                    let outputs = block
                        .txdata
                        .iter()
@@ -307,35 +368,35 @@ impl Indexer {
                                    let prev_addressbytes_opt = match outputtype {
                                        OutputType::P2PK65 => vecs
                                            .p2pk65addressindex_to_p2pk65bytes
-                                            .get_or_read(typeindex.into(), &p2pk65addressindex_to_p2pk65bytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2pk65addressindex_to_p2pk65bytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2PK33 => vecs
                                            .p2pk33addressindex_to_p2pk33bytes
-                                            .get_or_read(typeindex.into(), &p2pk33addressindex_to_p2pk33bytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2pk33addressindex_to_p2pk33bytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2PKH => vecs
                                            .p2pkhaddressindex_to_p2pkhbytes
-                                            .get_or_read(typeindex.into(), &p2pkhaddressindex_to_p2pkhbytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2pkhaddressindex_to_p2pkhbytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2SH => vecs
                                            .p2shaddressindex_to_p2shbytes
-                                            .get_or_read(typeindex.into(), &p2shaddressindex_to_p2shbytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2shaddressindex_to_p2shbytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2WPKH => vecs
                                            .p2wpkhaddressindex_to_p2wpkhbytes
-                                            .get_or_read(typeindex.into(), &p2wpkhaddressindex_to_p2wpkhbytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2wpkhaddressindex_to_p2wpkhbytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2WSH => vecs
                                            .p2wshaddressindex_to_p2wshbytes
-                                            .get_or_read(typeindex.into(), &p2wshaddressindex_to_p2wshbytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2wshaddressindex_to_p2wshbytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2TR => vecs
                                            .p2traddressindex_to_p2trbytes
-                                            .get_or_read(typeindex.into(), &p2traddressindex_to_p2trbytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2traddressindex_to_p2trbytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::P2A => vecs
                                            .p2aaddressindex_to_p2abytes
-                                            .get_or_read(typeindex.into(), &p2aaddressindex_to_p2abytes_mmap)?
+                                            .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_mmap)?
                                            .map(|v| AddressBytes::from(v.into_owned())),
                                        OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
                                            unreachable!()
@@ -677,7 +738,22 @@ impl Indexer {
                idxs.inputindex += InputIndex::from(inputs_len);
                idxs.outputindex += OutputIndex::from(outputs_len);

-                export_if_needed(stores, vecs, height, false, exit)?;
+                let exported = export_if_needed(stores, vecs, height, false, exit)?;
+
+                if exported {
+                    reset_mmaps_options(
+                        vecs,
+                        &mut txindex_to_first_outputindex_mmap_opt,
+                        &mut p2pk65addressindex_to_p2pk65bytes_mmap_opt,
+                        &mut p2pk33addressindex_to_p2pk33bytes_mmap_opt,
+                        &mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt,
+                        &mut p2shaddressindex_to_p2shbytes_mmap_opt,
+                        &mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt,
+                        &mut p2wshaddressindex_to_p2wshbytes_mmap_opt,
+                        &mut p2traddressindex_to_p2trbytes_mmap_opt,
+                        &mut p2aaddressindex_to_p2abytes_mmap_opt,
+                    );
+                }

                Ok(())
            },
@@ -685,8 +761,6 @@ impl Indexer {

        export_if_needed(stores, vecs, idxs.height, true, exit)?;

-        stores.rotate_memtables();
-
        Ok(starting_indexes)
    }
 }
@@ -72,7 +72,7 @@ impl Stores {
                    path,
                    "p2aaddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2pk33addressindex_with_outputindex = scope.spawn(|| {
@@ -81,7 +81,7 @@ impl Stores {
                    path,
                    "p2pk33addressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2pk65addressindex_with_outputindex = scope.spawn(|| {
@@ -90,7 +90,7 @@ impl Stores {
                    path,
                    "p2pk65addressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2pkhaddressindex_with_outputindex = scope.spawn(|| {
@@ -99,7 +99,7 @@ impl Stores {
                    path,
                    "p2pkhaddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2shaddressindex_with_outputindex = scope.spawn(|| {
@@ -108,7 +108,7 @@ impl Stores {
                    path,
                    "p2shaddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2traddressindex_with_outputindex = scope.spawn(|| {
@@ -117,7 +117,7 @@ impl Stores {
                    path,
                    "p2traddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2wpkhaddressindex_with_outputindex = scope.spawn(|| {
@@ -126,7 +126,7 @@ impl Stores {
                    path,
                    "p2wpkhaddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });
            let p2wshaddressindex_with_outputindex = scope.spawn(|| {
@@ -135,7 +135,7 @@ impl Stores {
                    path,
                    "p2wshaddressindex_with_outputindex",
                    version + VERSION + Version::ZERO,
-                    Some(None),
+                    Some(false),
                )
            });

@@ -181,12 +181,6 @@ impl Stores {
            .map_err(|e| e.into())
    }

-    pub fn rotate_memtables(&self) {
-        self.as_slice()
-            .into_iter()
-            .for_each(|store| store.rotate_memtable());
-    }
-
    fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] {
        [
            &self.addressbyteshash_to_typeindex,
@@ -14,7 +14,7 @@ brk_indexer = { workspace = true }
 brk_vec = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
-brk_rmcp = { workspace = true }
+rmcp = { workspace = true }
 schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -1,5 +1,5 @@
-use brk_rmcp::schemars::JsonSchema;
 use color_eyre::eyre::eyre;
+use rmcp::schemars::JsonSchema;
 use serde::Deserialize;

 #[allow(clippy::upper_case_acronyms)]
@@ -1,6 +1,6 @@
 use std::ops::Deref;

-use brk_rmcp::schemars::{self, JsonSchema};
+use rmcp::schemars::{self, JsonSchema};
 use serde::Deserialize;

 use crate::{
@@ -101,33 +101,6 @@ impl<'a> Vecs<'a> {
    fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
        let name = vec.name();
        let serialized_index = vec.index_type_to_string();
-        // let split = name.split("_to_").collect::<Vec<_>>();
-        // if split.len() != 1
-        //     && !(split.len() == 2
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.ends_with("relative")
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s.starts_with("cumulative_start")
-        //                 || s.starts_with("cumulative_from")
-        //                 || s == &"activity"
-        //         }))
-        //     && !(split.len() == 3
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s == &"cumulative_start"
-        //                 || s.starts_with("cumulative_from")
-        //         })
-        //         && split.get(1).is_some_and(|s| s.ends_with("relative")))
-        // {
-        //     dbg!((&serialized_index, &name, &split));
-        //     unreachable!();
-        // }
        let index = Index::try_from(serialized_index)
            .inspect_err(|_| {
                dbg!(&serialized_index);
@@ -26,7 +26,7 @@ pub fn init(path: Option<&Path>) {
    });

    Builder::from_env(Env::default().default_filter_or(
-        "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,brk_rmcp=off,tracing=off",
+        "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm_tree=off,rolldown=off,brk_rolldown=off,rmcp=off,rmcp=off,tracing=off",
    ))
    .format(move |buf, record| {
        let date_time = Timestamp::now()
@@ -11,4 +11,4 @@ repository.workspace = true
 axum = { workspace = true }
 brk_interface = { workspace = true }
 log = { workspace = true }
-brk_rmcp = { workspace = true }
+rmcp = { workspace = true }
@@ -47,11 +47,3 @@ Verify that it has access to BRK's tools.
 Optionally and highly recommended, giving it unsupervised access gives a more fluid experience and prevents possible issues and errors.

 ![Image of edit integration meny on Claude Desktop](https://github.com/bitcoinresearchkit/brk/blob/main/assets/claude-step4.png)
-
-#### Results
-
-Some examples of dashboard and documents generated by Claude using BRK's tools:
-
- [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5)
- [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42)
- [Dashboard2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc)
@@ -4,14 +4,14 @@
 // #![doc = "```"]

 use brk_interface::{IdParam, Interface, PaginatedIndexParam, PaginationParam, Params};
-use brk_rmcp::{
-    Error as McpError, RoleServer, ServerHandler,
+use log::info;
+use rmcp::{
+    ErrorData as McpError, RoleServer, ServerHandler,
    handler::server::{router::tool::ToolRouter, tool::Parameters},
    model::*,
    service::RequestContext,
    tool, tool_handler, tool_router,
 };
-use log::info;

 pub mod route;

@@ -1,6 +1,6 @@
 use axum::Router;
 use brk_interface::Interface;
-use brk_rmcp::transport::{
+use rmcp::transport::{
    StreamableHttpServerConfig,
    streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager},
 };
@@ -22,15 +22,13 @@ where
            return self;
        }

-        let config = StreamableHttpServerConfig {
-            // stateful_mode: false, // breaks Claude
-            ..Default::default()
-        };
-
        let service = StreamableHttpService::new(
            move || Ok(MCP::new(interface)),
            LocalSessionManager::default().into(),
-            config,
+            StreamableHttpServerConfig {
+                // stateful_mode: false, // breaks Claude
+                ..Default::default()
+            },
        );

        info!("Setting MCP...");
@@ -1,11 +1,12 @@
 use bitcoincore_rpc::{Auth, Client};
-use brk_core::{Height, default_bitcoin_path};
+use brk_core::{Height, default_bitcoin_path, default_brk_path};
 use brk_parser::Parser;

 fn main() {
    let i = std::time::Instant::now();

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(
        Client::new(
@@ -18,7 +19,7 @@ fn main() {
    let start = None;
    let end = None;

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    parser
        .parse(start, end)
@@ -1,11 +1,12 @@
 use bitcoincore_rpc::{Auth, Client};
-use brk_core::{Height, OutputType, default_bitcoin_path};
+use brk_core::{Height, OutputType, default_bitcoin_path, default_brk_path};
 use brk_parser::Parser;

 fn main() {
    let i = std::time::Instant::now();

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(
        Client::new(
@@ -18,7 +19,7 @@ fn main() {
    // let start = None;
    // let end = None;

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    // parser
    //     .parse(start, end)
@@ -5,7 +5,7 @@ use std::{
    path::{Path, PathBuf},
 };

-use crate::{BlkIndexToBlkPath, Height, blk_recap::BlkRecap};
+use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, Height};

 #[derive(Debug)]
 pub struct BlkIndexToBlkRecap {
@@ -15,11 +15,11 @@ pub struct BlkIndexToBlkRecap {

 impl BlkIndexToBlkRecap {
    pub fn import(
-        bitcoin_dir: &Path,
+        outputs_dir: &Path,
        blk_index_to_blk_path: &BlkIndexToBlkPath,
        start: Option<Height>,
    ) -> (Self, u16) {
-        let path = bitcoin_dir.join("blk_index_to_blk_recap.json");
+        let path = outputs_dir.join("blk_index_to_blk_recap.json");

        let tree = {
            if let Ok(file) = File::open(&path) {
@@ -38,12 +38,21 @@ const BOUND_CAP: usize = 50;

 pub struct Parser {
    blocks_dir: PathBuf,
+    outputs_dir: PathBuf,
    rpc: &'static bitcoincore_rpc::Client,
 }

 impl Parser {
-    pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
-        Self { blocks_dir, rpc }
+    pub fn new(
+        blocks_dir: PathBuf,
+        outputs_dir: PathBuf,
+        rpc: &'static bitcoincore_rpc::Client,
+    ) -> Self {
+        Self {
+            blocks_dir,
+            outputs_dir,
+            rpc,
+        }
    }

    pub fn get(&self, height: Height) -> Block {
@@ -74,7 +83,7 @@ impl Parser {
        let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir);

        let (mut blk_index_to_blk_recap, blk_index) =
-            BlkIndexToBlkRecap::import(blocks_dir, &blk_index_to_blk_path, start);
+            BlkIndexToBlkRecap::import(&self.outputs_dir, &blk_index_to_blk_path, start);

        let xor_bytes = XORBytes::from(blocks_dir);

@@ -28,6 +28,7 @@ jiff = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 tokio = { workspace = true }
 tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }
 tracing = "0.1.41"
@@ -2,7 +2,7 @@ use std::{path::Path, thread::sleep, time::Duration};

 use bitcoincore_rpc::RpcApi;
 use brk_computer::Computer;
-use brk_core::default_bitcoin_path;
+use brk_core::{default_bitcoin_path, default_brk_path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
@@ -18,6 +18,7 @@ pub fn main() -> color_eyre::Result<()> {
    let process = true;

    let bitcoin_dir = default_bitcoin_path();
+    let brk_dir = default_brk_path();

    let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
        "http://localhost:8332",
@@ -25,7 +26,7 @@ pub fn main() -> color_eyre::Result<()> {
    )?));
    let exit = Exit::new();

-    let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
+    let parser = Parser::new(bitcoin_dir.join("blocks"), brk_dir, rpc);

    let outputs_dir = Path::new("../../_outputs");

@@ -107,6 +107,16 @@ impl ApiRoutes for Router<AppState> {
                },
            ),
        )
+        .route(
+            "/health",
+            get(|| async {
+                Json(serde_json::json!({
+                    "status": "healthy",
+                    "service": "brk-server",
+                    "timestamp": jiff::Timestamp::now().to_string()
+                }))
+            }),
+        )
        .route(
            "/api",
            get(|| async {
@@ -34,11 +34,9 @@ pub struct Store<Key, Value> {
    rtx: ReadTransaction,
    puts: BTreeMap<Key, Value>,
    dels: BTreeSet<Key>,
-    bloom_filter_bits: Option<Option<u8>>,
+    bloom_filters: Option<bool>,
 }

-/// Use default if will read
-const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
 // const CHECK_COLLISIONS: bool = true;
 const MAJOR_FJALL_VERSION: Version = Version::TWO;

@@ -59,7 +57,7 @@ where
        path: &Path,
        name: &str,
        version: Version,
-        bloom_filter_bits: Option<Option<u8>>,
+        bloom_filters: Option<bool>,
    ) -> Result<Self> {
        fs::create_dir_all(path)?;

@@ -68,7 +66,7 @@ where
            &path.join(format!("meta/{name}")),
            MAJOR_FJALL_VERSION + version,
            || {
-                Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| {
+                Self::open_partition_handle(keyspace, name, bloom_filters).inspect_err(|e| {
                    eprintln!("{e}");
                    eprintln!("Delete {path:?} and try again");
                })
@@ -85,7 +83,7 @@ where
            rtx,
            puts: BTreeMap::new(),
            dels: BTreeSet::new(),
-            bloom_filter_bits,
+            bloom_filters,
        })
    }

@@ -180,17 +178,17 @@ where
    fn open_partition_handle(
        keyspace: &TransactionalKeyspace,
        name: &str,
-        bloom_filter_bits: Option<Option<u8>>,
+        bloom_filters: Option<bool>,
    ) -> Result<TransactionalPartitionHandle> {
-        keyspace
-            .open_partition(
-                name,
-                PartitionCreateOptions::default()
-                    .bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS))
-                    .max_memtable_size(8 * 1024 * 1024)
-                    .manual_journal_persist(true),
-            )
-            .map_err(|e| e.into())
+        let mut options = PartitionCreateOptions::default()
+            .max_memtable_size(8 * 1024 * 1024)
+            .manual_journal_persist(true);
+
+        if bloom_filters.is_some_and(|b| !b) {
+            options = options.bloom_filter_bits(None);
+        }
+
+        keyspace.open_partition(name, options).map_err(|e| e.into())
    }

    pub fn commit_(
@@ -272,18 +270,13 @@ where

        self.meta.reset();

-        let partition =
-            Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filter_bits)?;
+        let partition = Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filters)?;

        self.partition.replace(partition);

        Ok(())
    }

-    fn rotate_memtable(&self) {
-        let _ = self.partition.as_ref().unwrap().inner().rotate_memtable();
-    }
-
    fn height(&self) -> Option<Height> {
        self.meta.height()
    }
@@ -314,7 +307,7 @@ where
            rtx: self.keyspace.read_tx(),
            puts: self.puts.clone(),
            dels: self.dels.clone(),
-            bloom_filter_bits: self.bloom_filter_bits,
+            bloom_filters: self.bloom_filters,
        }
    }
 }
@@ -9,8 +9,6 @@ pub trait AnyStore {

    fn name(&self) -> &'static str;

-    fn rotate_memtable(&self);
-
    fn height(&self) -> Option<Height>;

    fn has(&self, height: Height) -> bool;
@@ -1,6 +1,6 @@
 [package]
 name = "brk_vec"
-description = "A push-only, truncable, compressable, saveable Vec"
+description = "A storeable vec"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
 version.workspace = true
@@ -16,7 +16,7 @@ brk_exit = { workspace = true }
 clap = { workspace = true }
 clap_derive = { workspace = true }
 log = { workspace = true }
-memmap2 = "0.9.5"
+memmap2 = "0.9.7"
 rayon = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -3,14 +3,15 @@ use std::{fs, path::Path};
 use brk_core::{DateIndex, Height, Version};
 use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator};

+type I = DateIndex;
 #[allow(clippy::upper_case_acronyms)]
-type VEC = StoredVec<DateIndex, u32>;
+type VEC = StoredVec<I, u32>;

 fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _ = fs::remove_dir_all("./vec");

    let version = Version::TWO;
-    let format = Format::Compressed;
+    let format = Format::Raw;

    {
        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
@@ -21,6 +22,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {

        let mut iter = vec.into_iter();
        dbg!(iter.get(0.into()));
+        dbg!(iter.get(1.into()));
+        dbg!(iter.get(2.into()));
        dbg!(iter.get(20.into()));
        dbg!(iter.get(21.into()));

@@ -38,6 +41,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
        dbg!(iter.get(0.into()));
        dbg!(iter.get(1.into()));
        dbg!(iter.get(2.into()));
+        dbg!(iter.get(3.into()));
+        dbg!(iter.get(4.into()));
+        dbg!(iter.get(5.into()));
        dbg!(iter.get(20.into()));
        dbg!(iter.get(20.into()));
        dbg!(iter.get(0.into()));
@@ -95,6 +101,38 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
        dbg!(iter.get(0.into()));
        dbg!(iter.get(20.into()));
        dbg!(iter.get(21.into()));
+
+        let mmap = vec.create_mmap()?;
+        dbg!(vec.take(10.into(), &mmap)?);
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
+        dbg!(vec.holes());
+        vec.flush()?;
+        dbg!(vec.holes());
+    }
+
+    {
+        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        let mmap = vec.create_mmap()?;
+
+        dbg!(vec.holes());
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
+
+        vec.update(10.into(), 10)?;
+        vec.update(0.into(), 10)?;
+        dbg!(
+            vec.holes(),
+            vec.get_or_read(0.into(), &mmap)?,
+            vec.get_or_read(10.into(), &mmap)?
+        );
+
+        vec.flush()?;
+    }
+
+    {
+        let vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+
+        dbg!(vec.collect()?);
    }

    Ok(())
@@ -7,7 +7,6 @@ use std::{

 use arc_swap::ArcSwap;
 use brk_core::{Error, Height, Result, Version};
-use memmap2::Mmap;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

@@ -31,8 +30,12 @@ impl Header {
        })
    }

-    pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result<Self> {
-        let inner = HeaderInner::import_and_verify(mmap, vec_version, format)?;
+    pub fn import_and_verify(
+        file: &mut File,
+        vec_version: Version,
+        format: Format,
+    ) -> Result<Self> {
+        let inner = HeaderInner::import_and_verify(file, vec_version, format)?;
        Ok(Self {
            inner: Arc::new(ArcSwap::from_pointee(inner)),
            modified: false,
@@ -100,7 +103,6 @@ impl HeaderInner {
            compressed: ZeroCopyBool::from(format),
        };
        header.write(file)?;
-        // dbg!(file.bytes().map(|b| b.unwrap()).collect::<Vec<_>>());
        file.seek(SeekFrom::End(0))?;
        Ok(header)
    }
@@ -109,13 +111,22 @@ impl HeaderInner {
        file.write_all_at(self.as_bytes(), 0)
    }

-    pub fn import_and_verify(mmap: &Mmap, vec_version: Version, format: Format) -> Result<Self> {
-        if mmap.len() < HEADER_OFFSET {
+    pub fn import_and_verify(
+        file: &mut File,
+        vec_version: Version,
+        format: Format,
+    ) -> Result<Self> {
+        let len = file.metadata()?.len();
+
+        if len < HEADER_OFFSET as u64 {
            return Err(Error::WrongLength);
        }
-        // dbg!(mmap.len());
-        let header = HeaderInner::read_from_bytes(&mmap[..HEADER_OFFSET])?;
-        // dbg!(&header);
+
+        let mut buf = [0; HEADER_OFFSET];
+        file.read_exact_at(&mut buf, 0)?;
+
+        let header = HeaderInner::read_from_bytes(&buf)?;
+
        if header.header_version != HEADER_VERSION {
            return Err(Error::DifferentVersion {
                found: header.header_version,
@@ -136,6 +147,7 @@ impl HeaderInner {
        {
            return Err(Error::DifferentCompressionMode);
        }
+
        Ok(header)
    }
 }
@@ -1,13 +1,13 @@
 use std::{
    borrow::Cow,
-    fs::{File, OpenOptions},
+    cmp::Ordering,
+    collections::{BTreeMap, BTreeSet},
+    fs::{self, File, OpenOptions},
    io::{self, Seek, SeekFrom, Write},
    path::{Path, PathBuf},
-    sync::Arc,
 };

-use arc_swap::ArcSwap;
-use brk_core::Result;
+use brk_core::{Error, Result};
 use memmap2::Mmap;

 use crate::{AnyVec, HEADER_OFFSET, Header};
@@ -38,7 +38,7 @@ where
    }
    #[inline]
    fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Cow<T>>> {
-        let stored_len = self.stored_len_(mmap);
+        let stored_len = self.stored_len();

        if index >= stored_len {
            let pushed = self.pushed();
@@ -46,10 +46,22 @@ where
            if j >= pushed.len() {
                return Ok(None);
            }
-            Ok(pushed.get(j).map(Cow::Borrowed))
-        } else {
-            Ok(self.read_(index, mmap)?.map(Cow::Owned))
+            return Ok(pushed.get(j).map(Cow::Borrowed));
        }
+
+        let updated = self.updated();
+        if !updated.is_empty()
+            && let Some(updated) = updated.get(&index)
+        {
+            return Ok(Some(Cow::Borrowed(updated)));
+        }
+
+        let holes = self.holes();
+        if !holes.is_empty() && holes.contains(&index) {
+            return Ok(None);
+        }
+
+        Ok(self.read_(index, mmap)?.map(Cow::Owned))
    }

    #[inline]
@@ -61,10 +73,7 @@ where
        format!("{}_to_{}", I::to_string(), self.name())
    }

-    fn mmap(&self) -> &ArcSwap<Mmap>;
-
    fn stored_len(&self) -> usize;
-    fn stored_len_(&self, mmap: &Mmap) -> usize;

    fn pushed(&self) -> &[T];
    #[inline]
@@ -77,6 +86,95 @@ where
        self.mut_pushed().push(value)
    }

+    #[inline]
+    fn update_or_push(&mut self, index: I, value: T) -> Result<()> {
+        let len = self.len();
+        match len.cmp(&index.to_usize()?) {
+            Ordering::Less => {
+                dbg!(index, value, len, self.header());
+                Err(Error::IndexTooHigh)
+            }
+            Ordering::Equal => {
+                self.push(value);
+                Ok(())
+            }
+            Ordering::Greater => self.update(index, value),
+        }
+    }
+
+    fn get_first_empty_index(&self) -> I {
+        self.holes()
+            .first()
+            .cloned()
+            .unwrap_or_else(|| self.len_())
+            .into()
+    }
+
+    #[inline]
+    fn fill_first_hole_or_push(&mut self, value: T) -> Result<I> {
+        Ok(
+            if let Some(hole) = self.mut_holes().pop_first().map(I::from) {
+                self.update(hole, value)?;
+                hole
+            } else {
+                self.push(value);
+                I::from(self.len() - 1)
+            },
+        )
+    }
+
+    fn holes(&self) -> &BTreeSet<usize>;
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize>;
+    fn take(&mut self, index: I, mmap: &Mmap) -> Result<Option<T>> {
+        let opt = self.get_or_read(index, mmap)?.map(|v| v.into_owned());
+        if opt.is_some() {
+            self.unchecked_delete(index);
+        }
+        Ok(opt)
+    }
+    #[inline]
+    fn delete(&mut self, index: I) {
+        if index.unwrap_to_usize() < self.len() {
+            self.unchecked_delete(index);
+        }
+    }
+    #[inline]
+    #[doc(hidden)]
+    fn unchecked_delete(&mut self, index: I) {
+        let uindex = index.unwrap_to_usize();
+        let updated = self.mut_updated();
+        if !updated.is_empty() {
+            updated.remove(&uindex);
+        }
+        self.mut_holes().insert(uindex);
+    }
+
+    fn updated(&self) -> &BTreeMap<usize, T>;
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T>;
+    #[inline]
+    fn update(&mut self, index: I, value: T) -> Result<()> {
+        let uindex = index.unwrap_to_usize();
+        let stored_len = self.stored_len();
+
+        if uindex >= stored_len {
+            if let Some(prev) = self.mut_pushed().get_mut(uindex - stored_len) {
+                *prev = value;
+                return Ok(());
+            } else {
+                return Err(Error::IndexTooHigh);
+            }
+        }
+
+        let holes = self.mut_holes();
+        if !holes.is_empty() {
+            holes.remove(&index.unwrap_to_usize());
+        }
+
+        self.mut_updated().insert(index.unwrap_to_usize(), value);
+
+        Ok(())
+    }
+
    fn header(&self) -> &Header;
    fn mut_header(&mut self) -> &mut Header;

@@ -90,14 +188,24 @@ where
        parent.join(name)
    }

+    #[inline]
    fn path(&self) -> PathBuf {
        Self::path_(self.parent(), self.name())
    }
-
+    #[inline]
    fn path_(parent: &Path, name: &str) -> PathBuf {
        Self::folder_(parent, name).join(I::to_string())
    }

+    #[inline]
+    fn holes_path(&self) -> PathBuf {
+        Self::holes_path_(self.parent(), self.name())
+    }
+    #[inline]
+    fn holes_path_(parent: &Path, name: &str) -> PathBuf {
+        Self::folder_(parent, name).join(format!("{}_holes", I::to_string()))
+    }
+
    // ---

    fn open_file(&self) -> io::Result<File> {
@@ -116,7 +224,7 @@ where

    fn file_set_len(&mut self, file: &mut File, len: u64) -> Result<()> {
        Self::file_set_len_(file, len)?;
-        self.update_mmap(file)
+        Ok(())
    }
    fn file_set_len_(file: &mut File, len: u64) -> Result<()> {
        file.set_len(len)?;
@@ -127,7 +235,7 @@ where
    fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> {
        file.write_all(buf)?;
        file.flush()?;
-        self.update_mmap(file)
+        Ok(())
    }

    fn file_truncate_and_write_all(&mut self, file: &mut File, len: u64, buf: &[u8]) -> Result<()> {
@@ -139,18 +247,18 @@ where

    #[inline]
    fn reset_(&mut self) -> Result<()> {
+        let holes_path = self.holes_path();
+        if fs::exists(&holes_path)? {
+            fs::remove_file(&holes_path)?;
+        }
        let mut file = self.open_file()?;
        self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
    }

-    fn new_mmap(file: &File) -> Result<Arc<Mmap>> {
-        Ok(Arc::new(unsafe { Mmap::map(file)? }))
-    }
-
-    fn update_mmap(&mut self, file: &File) -> Result<()> {
-        let mmap = Self::new_mmap(file)?;
-        self.mmap().store(mmap);
-        Ok(())
+    #[inline]
+    fn create_mmap(&self) -> Result<Mmap> {
+        let file = self.open_file()?;
+        unsafe { Mmap::map(&file).map_err(|e| e.into()) }
    }

    #[inline]
@@ -1,6 +1,7 @@
 use std::{fmt::Debug, ops::Add};

 use brk_core::{Error, Printable, Result};
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};

 pub trait StoredIndex
 where
@@ -15,6 +16,10 @@ where
        + TryInto<usize>
        + From<usize>
        + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
        + Send
        + Sync
        + Printable,
@@ -37,6 +42,10 @@ where
        + TryInto<usize>
        + From<usize>
        + Add<usize, Output = Self>
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
        + Send
        + Sync
        + Printable,
@@ -1,5 +1,6 @@
 use std::{
    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
    fs, mem,
    path::{Path, PathBuf},
    sync::Arc,
@@ -56,8 +57,9 @@ where
        }
    }

+    #[allow(unreachable_code, unused_variables)]
    pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
-        let inner = RawVec::import(parent, name, version)?;
+        let mut inner = RawVec::import(parent, name, version)?;

        let pages_meta = {
            let path = inner
@@ -66,12 +68,19 @@ where
            if inner.is_empty() {
                let _ = fs::remove_file(&path);
            }
-            Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(
-                &path,
-            )?)))
+            CompressedPagesMetadata::read(&path)?
        };

-        Ok(Self { inner, pages_meta })
+        inner.set_stored_len(if let Some(last) = pages_meta.last() {
+            (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
+        } else {
+            0
+        });
+
+        Ok(Self {
+            inner,
+            pages_meta: Arc::new(ArcSwap::new(Arc::new(pages_meta))),
+        })
    }

    fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result<Vec<T>> {
@@ -130,14 +139,6 @@ where
        page_index * Self::PER_PAGE
    }

-    fn stored_len__(pages_meta: &Guard<Arc<CompressedPagesMetadata>>) -> usize {
-        if let Some(last) = pages_meta.last() {
-            (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
-        } else {
-            0
-        }
-    }
-
    #[inline]
    pub fn iter(&self) -> CompressedVecIterator<'_, I, T> {
        self.into_iter()
@@ -180,22 +181,13 @@ where
        self.inner.mut_header()
    }

-    #[inline]
-    fn mmap(&self) -> &ArcSwap<Mmap> {
-        self.inner.mmap()
-    }
-
    fn parent(&self) -> &Path {
        self.inner.parent()
    }

    #[inline]
    fn stored_len(&self) -> usize {
-        Self::stored_len__(&self.pages_meta.load())
-    }
-    #[inline]
-    fn stored_len_(&self, _: &Mmap) -> usize {
-        self.stored_len()
+        self.inner.stored_len()
    }

    #[inline]
@@ -206,12 +198,44 @@ where
    fn mut_pushed(&mut self) -> &mut Vec<T> {
        self.inner.mut_pushed()
    }
+    #[inline]
+    fn holes(&self) -> &BTreeSet<usize> {
+        self.inner.holes()
+    }
+    #[inline]
+    fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
+        self.inner.mut_holes()
+    }
+    #[inline]
+    fn updated(&self) -> &BTreeMap<usize, T> {
+        self.inner.updated()
+    }
+    #[inline]
+    fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
+        self.inner.mut_updated()
+    }

    #[inline]
    fn path(&self) -> PathBuf {
        self.inner.path()
    }

+    fn delete(&mut self, _: I) {
+        panic!("unsupported")
+    }
+
+    fn unchecked_delete(&mut self, _: I) {
+        panic!("unsupported")
+    }
+
+    fn fill_first_hole_or_push(&mut self, _: T) -> Result<I> {
+        panic!("unsupported")
+    }
+
+    fn update(&mut self, _: I, _: T) -> Result<()> {
+        panic!("unsupported")
+    }
+
    fn flush(&mut self) -> Result<()> {
        let file_opt = self.inner.write_header_if_needed()?;

@@ -223,6 +247,8 @@ where

        let stored_len = self.stored_len();

+        let mut file = file_opt.unwrap_or(self.open_file()?);
+
        let mut pages_meta = (**self.pages_meta.load()).clone();

        let mut starting_page_index = pages_meta.len();
@@ -236,16 +262,13 @@ where

            let last_page_index = pages_meta.len() - 1;

-            values = Self::decode_page_(
-                stored_len,
-                last_page_index,
-                &self.mmap().load(),
-                &pages_meta,
-            )
-            .inspect_err(|_| {
-                dbg!(last_page_index, &pages_meta);
-            })
-            .unwrap();
+            let mmap = unsafe { Mmap::map(&file)? };
+
+            values = Self::decode_page_(stored_len, last_page_index, &mmap, &pages_meta)
+                .inspect_err(|_| {
+                    dbg!(last_page_index, &pages_meta);
+                })
+                .unwrap();

            truncate_at.replace(pages_meta.pop().unwrap().start);
            starting_page_index = last_page_index;
@@ -287,8 +310,6 @@ where

        pages_meta.write()?;

-        let mut file = file_opt.unwrap_or(self.open_file()?);
-
        if let Some(truncate_at) = truncate_at {
            self.file_set_len(&mut file, truncate_at)?;
        }
@@ -324,7 +345,11 @@ where

        let page_index = Self::index_to_page_index(index);

-        let values = self.decode_page(page_index, &self.mmap().load())?;
+        let mut file = self.open_file()?;
+
+        let mmap = unsafe { Mmap::map(&file)? };
+
+        let values = self.decode_page(page_index, &mmap)?;
        let mut buf = vec![];

        let mut page = pages_meta.truncate(page_index).unwrap();
@@ -348,8 +373,6 @@ where

        self.pages_meta.store(Arc::new(pages_meta));

-        let mut file = self.open_file()?;
-
        self.file_truncate_and_write_all(&mut file, len, &buf)?;

        Ok(())
@@ -399,7 +422,7 @@ impl<I, T> Clone for CompressedVec<I, T> {
 #[derive(Debug)]
 pub struct CompressedVecIterator<'a, I, T> {
    vec: &'a CompressedVec<I, T>,
-    guard: Guard<Arc<Mmap>>,
+    mmap: Mmap,
    decoded_page: Option<(usize, Vec<T>)>,
    // second_decoded_page?: Option<(usize, Vec<T>)>,
    pages_meta: Guard<Arc<CompressedPagesMetadata>>,
@@ -445,7 +468,7 @@ where
    type Item = (I, Cow<'a, T>);

    fn next(&mut self) -> Option<Self::Item> {
-        let mmap = &self.guard;
+        let mmap = &self.mmap;
        let i = self.index;
        let stored_len = self.stored_len;

@@ -496,14 +519,15 @@ where

    fn into_iter(self) -> Self::IntoIter {
        let pages_meta = self.pages_meta.load();
-        let stored_len = CompressedVec::<I, T>::stored_len__(&pages_meta);
+        let stored_len = self.stored_len();
+
        CompressedVecIterator {
            vec: self,
-            guard: self.mmap().load(),
+            mmap: self.create_mmap().unwrap(),
            decoded_page: None,
            pages_meta,
-            stored_len,
            index: 0,
+            stored_len,
        }
    }
 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
nym21	537d98b41b	release: v0.0.81	2025-07-17 23:45:01 +02:00
nym21	9c4cadfc04	vec: fix holes export	2025-07-17 17:29:53 +02:00
nym21	2001370441	mcp: use rust-rmcp instead of brk-rmcp	2025-07-17 16:34:29 +02:00
nym21	cc87b22757	computer: perf improvements	2025-07-17 16:17:21 +02:00
nym21	c0a65b30ad	indexer: update example	2025-07-17 11:39:25 +02:00
nym21	c07e66c086	computer: fix stateful	2025-07-17 11:35:40 +02:00
nym21	a0cfc1be2b	computer: convert stores to vecs part 2	2025-07-16 16:23:40 +02:00
nym21	1505454793	computer: convert stores to vecs part 1	2025-07-15 22:47:46 +02:00
nym21	e1dff66283	pr: merge #21 from deadmanoz/dockerize Add Docker support	2025-07-15 15:51:32 +00:00
deadmanoz	5be801a086	Merge branch 'main' into dockerize	2025-07-15 08:50:22 -07:00
deadmanoz	94d4b05c29	Address review feedback	2025-07-15 08:48:39 -07:00
nym21	cebb889f7e	cargo: update	2025-07-14 16:00:31 +02:00
nym21	c4ed6ed034	store: remove rotate_memtable as could be the root cause of the issue	2025-07-14 15:48:19 +02:00
nym21	ec960bfefa	release: v0.0.80	2025-07-13 21:20:40 +02:00
nym21	79f689dde1	mcp: remove claude results examples due to dead links	2025-07-13 21:20:02 +02:00
nym21	3b3654df56	vec: add local and shared stored_len to raw variant	2025-07-13 19:30:50 +02:00
nym21	c66f008f07	release: v0.0.79	2025-07-13 17:18:14 +02:00
nym21	37d9498d90	crates: upgrade	2025-07-13 17:18:02 +02:00
nym21	1ff67093db	website: apply datasets changes to charts	2025-07-13 17:14:34 +02:00
nym21	daed37ccb8	stores: forgot some files	2025-07-13 16:52:19 +02:00
nym21	d41d807b4f	stores: bloom filters back to default due to slow reads, v3 will bring down the needed RAM	2025-07-13 16:49:45 +02:00
nym21	d6fa5c8a55	vec: fix header reading of existing file	2025-07-13 16:31:22 +02:00
nym21	2dd608dfed	vec: don't store mmap in struct anymore	2025-07-13 11:50:34 +02:00
deadmanoz	5de9757d46	Remove services from docker	2025-07-04 16:37:39 +08:00
deadmanoz	f89276d7b8	Remove redundant services	2025-07-04 15:51:28 +08:00
deadmanoz	30ba034206	Move docker artefacts into /docker directory	2025-07-04 13:00:12 +08:00
deadmanoz	fa1e5aaa7f	Make Parser::new the only entrypoint	2025-07-04 12:15:32 +08:00
deadmanoz	870c70180f	Back to a single image/container setup	2025-07-04 11:40:37 +08:00
deadmanoz	d83a833b4d	Switch to multiple container setup	2025-06-27 12:56:25 +08:00
deadmanoz	ec3a2f29f0	Docker functionality, change location of 'blk_index_to_blk_recap.json'	2025-06-27 12:56:03 +08:00