global: snapshot

Cargo.lock

@@ -324,51 +324,63 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
 [[package]]
 name = "brk"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "brk_computer",
  "brk_core",
+ "brk_exit",
  "brk_fetcher",
  "brk_indexer",
  "brk_logger",
  "brk_parser",
  "brk_server",
+ "brk_vec",
 ]
 
 [[package]]
 name = "brk_cli"
-version = "0.0.1"
+version = "0.0.2"
 
 [[package]]
 name = "brk_computer"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "brk_core",
+ "brk_exit",
  "brk_indexer",
  "brk_parser",
+ "brk_vec",
  "color-eyre",
- "hodor",
- "storable_vec",
 ]
 
 [[package]]
 name = "brk_core"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
  "byteview",
  "derive_deref",
  "jiff",
+ "log",
  "rapidhash",
+ "rlimit",
  "serde",
  "serde_bytes",
  "zerocopy 0.8.20",
 ]
 
+[[package]]
+name = "brk_exit"
+version = "0.0.2"
+dependencies = [
+ "ctrlc",
+ "log",
+]
+
 [[package]]
 name = "brk_fetcher"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -381,27 +393,26 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
  "brk_core",
+ "brk_exit",
  "brk_logger",
  "brk_parser",
+ "brk_vec",
  "byteview",
  "color-eyre",
  "fjall",
- "hodor",
  "log",
  "rayon",
- "rlimit",
- "storable_vec",
  "zerocopy 0.8.20",
 ]
 
 [[package]]
 name = "brk_logger"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -410,7 +421,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -425,12 +436,13 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.1"
+version = "0.0.2"
 dependencies = [
  "axum",
  "brk_computer",
  "brk_indexer",
  "brk_logger",
+ "brk_vec",
  "color-eyre",
  "derive_deref",
  "jiff",
@@ -438,11 +450,21 @@ dependencies = [
  "oxc",
  "serde",
  "serde_json",
- "storable_vec",
  "tokio",
  "tower-http",
 ]
 
+[[package]]
+name = "brk_vec"
+version = "0.0.2"
+dependencies = [
+ "memmap2",
+ "rayon",
+ "serde",
+ "serde_json",
+ "zerocopy 0.8.20",
+]
+
 [[package]]
 name = "brotli"
 version = "7.0.0"
@@ -923,14 +945,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
 
-[[package]]
-name = "hodor"
-version = "0.1.1"
-dependencies = [
- "ctrlc",
- "log",
-]
-
 [[package]]
 name = "http"
 version = "1.2.0"
@@ -2201,17 +2215,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
 
-[[package]]
-name = "storable_vec"
-version = "0.1.3"
-dependencies = [
- "memmap2",
- "rayon",
- "serde",
- "serde_json",
- "zerocopy 0.8.20",
-]
-
 [[package]]
 name = "str_indices"
 version = "0.4.4"

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.1"
+package.version = "0.0.2"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 
 [workspace.dependencies]
@@ -12,23 +12,21 @@ bitcoin = { version = "0.32.5", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
 brk_computer = { version = "0", path = "crates/brk_computer" }
 brk_core = { version = "0", path = "crates/brk_core" }
+brk_exit = { version = "0", path = "crates/brk_exit" }
 brk_fetcher = { version = "0", path = "crates/brk_fetcher" }
 brk_indexer = { version = "0", path = "crates/brk_indexer" }
-brk_parser = { version = "0", path = "crates/brk_parser" }
 brk_logger = { version = "0", path = "crates/brk_logger" }
+brk_parser = { version = "0", path = "crates/brk_parser" }
 brk_server = { version = "0", path = "crates/brk_server" }
+brk_vec = { version = "0", path = "crates/brk_vec", features = ["json"] }
 byteview = "0.5.4"
 color-eyre = "0.6.3"
 derive_deref = "1.1.1"
 fjall = "2.6.7"
-hodor = { version = "0", path = "crates/hodor" }
 jiff = "0.2.1"
 log = { version = "0.4.26" }
 minreq = { version = "2.13.2", features = ["https", "serde_json"] }
 rayon = "1.10.0"
 serde = { version = "1.0.218", features = ["derive"] }
 serde_json = { version = "1.0.139", features = ["float_roundtrip"] }
-storable_vec = { version = "0", path = "crates/storable_vec", features = [
-    "json",
-] }
 zerocopy = { version = "0.8.20", features = ["derive"] }

README.md

@@ -33,6 +33,12 @@ The Core (Structs and Errors) of the Bitcoin Research Kit.
 
 > Status: ✅
 
+### `brk_exit`
+
+An exit blocker built on top of ctrlc.
+
+> Status: ✅
+
 ### `brk_fetcher`
 
 A Bitcoin price fetcher.
@@ -63,13 +69,7 @@ A server that serves Bitcoin data and swappable front-ends, built on top of brk_
 
 > Status: ⚠️
 
-### `hodor`
-
-Hold the door, an exit blocker built on top of ctrlc.
-
-> Status: ✅
-
-### `storable_vec`
+### `brk_vec`
 
 A very small, fast, efficient and simple storable Vec.
 

crates/brk/Cargo.toml

@@ -8,23 +8,37 @@ edition.workspace = true
 version.workspace = true
 
 [features]
-full = ["core", "computer", "fetcher", "indexer", "logger", "parser", "server"]
+full = [
+    "core",
+    "computer",
+    "exit",
+    "fetcher",
+    "indexer",
+    "logger",
+    "parser",
+    "server",
+    "vec",
+]
 core = ["brk_core"]
 computer = ["brk_computer"]
+exit = ["brk_exit"]
 fetcher = ["brk_fetcher"]
 indexer = ["brk_indexer"]
 logger = ["brk_logger"]
 parser = ["brk_parser"]
 server = ["brk_server"]
+vec = ["brk_vec"]
 
 [dependencies]
 brk_core = { workspace = true, optional = true }
 brk_computer = { workspace = true, optional = true }
+brk_exit = { workspace = true, optional = true }
 brk_fetcher = { workspace = true, optional = true }
 brk_indexer = { workspace = true, optional = true }
 brk_logger = { workspace = true, optional = true }
 brk_parser = { workspace = true, optional = true }
 brk_server = { workspace = true, optional = true }
+brk_vec = { workspace = true, optional = true }
 
 [package.metadata.docs.rs]
 all-features = true

crates/brk_computer/Cargo.toml

@@ -8,8 +8,8 @@ repository.workspace = true
 
 [dependencies]
 brk_core = { workspace = true }
+brk_exit = { workspace = true }
 brk_indexer = { workspace = true }
 brk_parser = { workspace = true }
+brk_vec = { workspace = true }
 color-eyre = { workspace = true }
-hodor = { workspace = true }
-storable_vec = { workspace = true }

crates/brk_computer/src/lib.rs

@@ -1,35 +1,35 @@
 use std::path::{Path, PathBuf};
 
+use brk_exit::Exit;
 use brk_indexer::Indexer;
 pub use brk_parser::rpc;
-use hodor::Hodor;
 
 mod storage;
 
 use brk_core::Date;
-use storable_vec::SINGLE_THREAD;
-use storage::{Fjalls, StorableVecs};
+use brk_vec::SINGLE_THREAD;
+use storage::{Stores, Vecs};
 
 pub struct Computer<const MODE: u8> {
     path: PathBuf,
-    pub vecs: StorableVecs<MODE>,
-    pub trees: Fjalls,
+    pub vecs: Vecs<MODE>,
+    pub stores: Stores,
 }
 
 impl<const MODE: u8> Computer<MODE> {
     pub fn import(computed_dir: &Path) -> color_eyre::Result<Self> {
-        let vecs = StorableVecs::import(&computed_dir.join("vecs"))?;
-        let trees = Fjalls::import(&computed_dir.join("fjall"))?;
+        let vecs = Vecs::import(&computed_dir.join("vecs"))?;
+        let stores = Stores::import(&computed_dir.join("fjall"))?;
         Ok(Self {
             path: computed_dir.to_owned(),
             vecs,
-            trees,
+            stores,
         })
     }
 }
 
 impl Computer<SINGLE_THREAD> {
-    pub fn compute(&mut self, mut indexer: Indexer<SINGLE_THREAD>, hodor: &Hodor) -> color_eyre::Result<()> {
+    pub fn compute(&mut self, mut indexer: Indexer<SINGLE_THREAD>, exit: &Exit) -> color_eyre::Result<()> {
         let height_count = indexer.vecs.height_to_size.len();
         let txindexes_count = indexer.vecs.txindex_to_txid.len();
         let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len();

crates/brk_computer/src/main.rs

@@ -1,13 +1,13 @@
 use std::path::Path;
 
 use brk_computer::Computer;
+use brk_exit::Exit;
 use brk_indexer::Indexer;
-use hodor::Hodor;
 
 pub fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
-    let hodor = Hodor::new();
+    let exit = Exit::new();
 
     let i = std::time::Instant::now();
 
@@ -17,7 +17,7 @@ pub fn main() -> color_eyre::Result<()> {
 
     let mut computer = Computer::import(&outputs_dir.join("computed"))?;
 
-    computer.compute(indexer, &hodor)?;
+    computer.compute(indexer, &exit)?;
 
     dbg!(i.elapsed());
 

crates/brk_computer/src/storage/mod.rs

@@ -1,5 +1,5 @@
-mod fjalls;
-mod storable_vecs;
+mod stores;
+mod vecs;
 
-pub use fjalls::*;
-pub use storable_vecs::*;
+pub use stores::*;
+pub use vecs::*;

crates/brk_computer/src/storage/stores.rs

@@ -2,14 +2,14 @@ use std::path::Path;
 
 use brk_core::{AddressindexTxoutindex, Unit};
 use brk_indexer::Store;
-use storable_vec::Version;
+use brk_vec::Version;
 
-pub struct Fjalls {
+pub struct Stores {
     pub address_to_utxos_received: Store<AddressindexTxoutindex, Unit>,
     pub address_to_utxos_spent: Store<AddressindexTxoutindex, Unit>,
 }
 
-impl Fjalls {
+impl Stores {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?;
         let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?;

crates/brk_computer/src/storage/vecs.rs

@@ -4,13 +4,13 @@ use brk_core::{
     Addressindex, Cents, Close, Date, Dateindex, Dollars, Feerate, Height, High, Low, Open, Sats, Timestamp, Txindex,
     Txinindex, Txoutindex,
 };
-use storable_vec::{StorableVec, Version};
+use brk_vec::{StorableVec, Version};
 
 // mod base;
 
 // use base::*;
 
-pub struct StorableVecs<const MODE: u8> {
+pub struct Vecs<const MODE: u8> {
     pub dateindex_to_first_height: StorableVec<Dateindex, Height, MODE>,
     // pub dateindex_to_last_height: StorableVec<Dateindex, Height, MODE>,
     // pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
@@ -56,7 +56,7 @@ pub struct StorableVecs<const MODE: u8> {
     pub txindex_to_outputs_sum: StorableVec<Txindex, Sats, MODE>,
 }
 
-impl<const MODE: u8> StorableVecs<MODE> {
+impl<const MODE: u8> Vecs<MODE> {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 

crates/brk_core/Cargo.toml

@@ -12,7 +12,9 @@ bitcoincore-rpc = { workspace = true }
 byteview = { workspace = true }
 derive_deref = { workspace = true }
 jiff = { workspace = true }
+log = { workspace = true }
 rapidhash = "1.4.0"
+rlimit = "0.10.2"
 serde = { workspace = true }
 serde_bytes = "0.11.15"
 zerocopy = { workspace = true }

crates/brk_core/src/lib.rs

@@ -1,5 +1,7 @@
 mod error;
 mod structs;
+mod utils;
 
 pub use error::*;
 pub use structs::*;
+pub use utils::*;

crates/brk_core/src/structs/height.rs

@@ -177,7 +177,7 @@ impl From<Height> for u64 {
 impl TryFrom<&Client> for Height {
     type Error = bitcoincore_rpc::Error;
     fn try_from(value: &Client) -> Result<Self, Self::Error> {
-        Ok((value.get_blockchain_info()?.blocks as usize - 1).into())
+        Ok((value.get_block_count()? as usize).into())
     }
 }
 

crates/brk_core/src/utils/mod.rs

@@ -0,0 +1,5 @@
+mod pause;
+mod rlimit;
+
+pub use pause::*;
+pub use rlimit::*;

crates/brk_core/src/utils/pause.rs

@@ -0,0 +1,7 @@
+use log::info;
+
+pub fn pause() {
+    info!("Press enter to continue...");
+    let mut buffer = String::new();
+    std::io::stdin().read_line(&mut buffer).expect("Failed to read line");
+}

crates/brk_core/src/utils/rlimit.rs

@@ -0,0 +1,10 @@
+use std::io;
+
+use rlimit::{Resource, getrlimit};
+
+pub fn setrlimit() -> io::Result<()> {
+    let no_file_limit = getrlimit(Resource::NOFILE)?;
+    rlimit::setrlimit(Resource::NOFILE, no_file_limit.0.max(210_000), no_file_limit.1)?;
+
+    Ok(())
+}

crates/brk_exit/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
-name = "hodor"
-description = "Hold the door, an exit blocker built on top of ctrlc"
-version = "0.1.1"
+name = "brk_exit"
+description = "An exit blocker built on top of ctrlc"
+version.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true

crates/brk_exit/src/lib.rs

@@ -11,32 +11,32 @@ use std::{
 use log::info;
 
 #[derive(Default, Clone)]
-pub struct Hodor {
-    holding: Arc<AtomicBool>,
+pub struct Exit {
+    blocking: Arc<AtomicBool>,
     triggered: Arc<AtomicBool>,
 }
 
-impl Hodor {
+impl Exit {
     pub fn new() -> Self {
         let s = Self {
             triggered: Arc::new(AtomicBool::new(false)),
-            holding: Arc::new(AtomicBool::new(false)),
+            blocking: Arc::new(AtomicBool::new(false)),
         };
 
         let triggered = s.triggered.clone();
 
-        let holding = s.holding.clone();
-        let is_holding = move || holding.load(Ordering::SeqCst);
+        let blocking = s.blocking.clone();
+        let is_blocking = move || blocking.load(Ordering::SeqCst);
 
         ctrlc::set_handler(move || {
             info!("Exitting...");
 
             triggered.store(true, Ordering::SeqCst);
 
-            if is_holding() {
+            if is_blocking() {
                 info!("Waiting to exit safely...");
 
-                while is_holding() {
+                while is_blocking() {
                     sleep(Duration::from_millis(50));
                 }
             }
@@ -48,12 +48,12 @@ impl Hodor {
         s
     }
 
-    pub fn hold(&self) {
-        self.holding.store(true, Ordering::SeqCst);
+    pub fn block(&self) {
+        self.blocking.store(true, Ordering::SeqCst);
     }
 
     pub fn release(&self) {
-        self.holding.store(false, Ordering::SeqCst);
+        self.blocking.store(false, Ordering::SeqCst);
     }
 
     pub fn triggered(&self) -> bool {

crates/brk_indexer/Cargo.toml

@@ -10,14 +10,13 @@ repository.workspace = true
 bitcoin = { workspace = true }
 bitcoincore-rpc = { workspace = true }
 brk_core = { workspace = true }
+brk_exit = { workspace = true }
 brk_parser = { workspace = true }
 brk_logger = { workspace = true }
+brk_vec = { workspace = true }
 byteview = { workspace = true }
 color-eyre = { workspace = true }
 fjall = { workspace = true }
-hodor = { workspace = true }
 log = { workspace = true }
 rayon = { workspace = true }
-rlimit = { version = "0.10.2" }
-storable_vec = { workspace = true }
 zerocopy = { workspace = true }

crates/brk_indexer/README.md

@@ -1,6 +1,6 @@
 # BRK Indexer
 
-A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code.
+A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../brk_parser`) and creates a database of the vecs (`../brk_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code.
 
 The crate only stores the bare minimum to be self sufficient and not have to use an RPC client (except for scripts which are not stored). If you need more data, checkout `../computer` which uses the outputs from the indexer to compute a whole range of datasets.
 
@@ -18,9 +18,9 @@ Peaks at 11-13 GB of RAM
 
 ## Outputs
 
-Vecs: `src/storage/storable_vecs/mod.rs`
+Vecs: `src/storage/vecs/mod.rs`
 
-Stores: `src/storage/fjalls/mod.rs`
+Stores: `src/storage/stores/mod.rs`
 
 ## Examples
 

crates/brk_indexer/src/lib.rs

@@ -2,55 +2,52 @@ use std::{
     collections::BTreeMap,
     path::Path,
     str::FromStr,
-    thread::{self, sleep},
-    time::Duration,
+    thread::{self},
 };
 
 use brk_core::{
     AddressHash, Addressbytes, Addressindex, Addresstype, BlockHash, BlockHashPrefix, Height, Sats, Timestamp, Txid,
-    TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout,
+    TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, setrlimit,
 };
 pub use brk_parser::*;
 
 use bitcoin::{Transaction, TxIn, TxOut};
+use brk_exit::Exit;
+use brk_vec::CACHED_GETS;
 use color_eyre::eyre::{ContextCompat, eyre};
-use hodor::Hodor;
 use log::info;
 use rayon::prelude::*;
-use storable_vec::CACHED_GETS;
 
 mod storage;
 mod structs;
 
 pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
-use storage::{Fjalls, StorableVecs};
+use storage::{Stores, Vecs};
 pub use structs::*;
 
 const SNAPSHOT_BLOCK_RANGE: usize = 1000;
 
 pub struct Indexer<const MODE: u8> {
-    pub vecs: StorableVecs<MODE>,
-    pub stores: Fjalls,
+    pub vecs: Vecs<MODE>,
+    pub stores: Stores,
 }
 
 impl<const MODE: u8> Indexer<MODE> {
     pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
-        rlimit::setrlimit(
-            rlimit::Resource::NOFILE,
-            rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0.max(210_000),
-            rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
-        )?;
+        setrlimit()?;
 
         info!("Importing indexes...");
-        let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
-        let stores = Fjalls::import(&indexes_dir.join("fjall"))?;
+
+        let vecs = Vecs::import(&indexes_dir.join("vecs"))?;
+
+        let stores = Stores::import(&indexes_dir.join("fjall"))?;
 
         Ok(Self { vecs, stores })
     }
 }
 
 impl Indexer<CACHED_GETS> {
-    pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, hodor: &Hodor) -> color_eyre::Result<()> {
+    pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
         let check_collisions = true;
 
         let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| {
@@ -59,27 +56,26 @@ impl Indexer<CACHED_GETS> {
             indexes
         });
 
-        hodor.hold();
+        exit.block();
         self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?;
         self.vecs.rollback_if_needed(&starting_indexes)?;
-        hodor.release();
+        exit.release();
 
-        let export_if_needed = |stores: &mut Fjalls,
-                                vecs: &mut StorableVecs<CACHED_GETS>,
+        let export_if_needed = |stores: &mut Stores,
+                                vecs: &mut Vecs<CACHED_GETS>,
                                 height: Height,
-                                hodor: &Hodor|
+                                rem: bool,
+                                exit: &Exit|
          -> color_eyre::Result<()> {
-            if height == 0 || height % SNAPSHOT_BLOCK_RANGE != 0 || hodor.triggered() {
+            if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() {
                 return Ok(());
             }
 
             info!("Exporting...");
-            hodor.hold();
+            exit.block();
             stores.commit(height)?;
-            info!("Exported stores");
             vecs.flush(height)?;
-            info!("Exported vecs");
-            hodor.release();
+            exit.release();
             Ok(())
         };
 
@@ -88,7 +84,7 @@ impl Indexer<CACHED_GETS> {
 
         let mut idxs = starting_indexes;
 
-        if idxs.height >= Height::try_from(rpc)? {
+        if idxs.height > Height::try_from(rpc)? {
             return Ok(());
         }
 
@@ -629,16 +625,13 @@ impl Indexer<CACHED_GETS> {
 
                 idxs.push_future_if_needed(vecs)?;
 
-                export_if_needed(stores, vecs, height, hodor)?;
+                export_if_needed(stores, vecs, height, false, exit)?;
 
                 Ok(())
             },
         )?;
 
-        export_if_needed(stores, vecs, idxs.height, hodor)?;
-
-        // To make sure that Fjall had the time to flush everything properly
-        sleep(Duration::from_millis(100));
+        export_if_needed(stores, vecs, idxs.height, true, exit)?;
 
         Ok(())
     }

crates/brk_indexer/src/main.rs

@@ -1,13 +1,13 @@
 use std::{path::Path, thread::sleep, time::Duration};
 
+use brk_exit::Exit;
 use brk_indexer::{Indexer, rpc::RpcApi};
 use brk_parser::{
     Parser,
     rpc::{self},
 };
-use hodor::Hodor;
+use brk_vec::CACHED_GETS;
 use log::info;
-use storable_vec::CACHED_GETS;
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -19,27 +19,27 @@ fn main() -> color_eyre::Result<()> {
         "http://localhost:8332",
         rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
     )?));
-    let hodor = Hodor::new();
+    let exit = Exit::new();
 
     let parser = Parser::new(data_dir, rpc);
 
+    let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../../_outputs/indexes"))?;
+
     loop {
-        let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
+        let block_count = rpc.get_block_count()?;
 
         info!("{block_count} blocks found.");
 
-        let i = std::time::Instant::now();
+        indexer.index(&parser, rpc, &exit)?;
 
-        let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../../_outputs/indexes"))?;
+        info!("Waiting for new blocks...");
 
-        indexer.index(&parser, rpc, &hodor)?;
-
-        info!("Took: {:?}", i.elapsed());
-
-        info!("Waiting for a new block...");
-
-        while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
+        while block_count == rpc.get_block_count()? {
             sleep(Duration::from_secs(1))
         }
     }
+
+    #[allow(unreachable_code)]
+    // To make sure that Fjall had the time to flush everything properly
+    sleep(Duration::from_millis(100));
 }

crates/brk_indexer/src/storage/mod.rs

@@ -1,5 +1,5 @@
-mod fjalls;
-mod storable_vecs;
+mod stores;
+mod vecs;
 
-pub use fjalls::*;
-pub use storable_vecs::*;
+pub use stores::*;
+pub use vecs::*;

crates/brk_indexer/src/storage/stores/base.rs

@@ -7,11 +7,11 @@ use std::{
 };
 
 use brk_core::Height;
+use brk_vec::{Value, Version};
 use byteview::ByteView;
 use fjall::{
     PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace, TransactionalPartitionHandle,
 };
-use storable_vec::{Value, Version};
 use zerocopy::{Immutable, IntoBytes};
 
 use super::StoreMeta;

crates/brk_indexer/src/storage/stores/meta.rs

@@ -3,7 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use storable_vec::Version;
+use brk_vec::Version;
 use zerocopy::{FromBytes, IntoBytes};
 
 use super::Height;

crates/brk_indexer/src/storage/stores/mod.rs

@@ -1,7 +1,7 @@
 use std::{path::Path, thread};
 
 use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex};
-use storable_vec::{CACHED_GETS, Value, Version};
+use brk_vec::{CACHED_GETS, Value, Version};
 
 use crate::Indexes;
 
@@ -11,15 +11,15 @@ mod meta;
 pub use base::*;
 pub use meta::*;
 
-use super::StorableVecs;
+use super::Vecs;
 
-pub struct Fjalls {
+pub struct Stores {
     pub addresshash_to_addressindex: Store<AddressHash, Addressindex>,
     pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,
     pub txid_prefix_to_txindex: Store<TxidPrefix, Txindex>,
 }
 
-impl Fjalls {
+impl Stores {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         thread::scope(|scope| {
             let addresshash_to_addressindex =
@@ -39,7 +39,7 @@ impl Fjalls {
 
     pub fn rollback_if_needed(
         &mut self,
-        vecs: &StorableVecs<CACHED_GETS>,
+        vecs: &Vecs<CACHED_GETS>,
         starting_indexes: &Indexes,
     ) -> color_eyre::Result<()> {
         vecs.height_to_blockhash

crates/brk_indexer/src/storage/vecs/base.rs

@@ -5,14 +5,14 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use storable_vec::{StoredIndex, StoredType, Version};
+use brk_vec::{StoredIndex, StoredType, Version};
 
 use super::Height;
 
 #[derive(Debug)]
 pub struct StorableVec<I, T, const MODE: u8> {
     height: Option<Height>,
-    vec: storable_vec::StorableVec<I, T, MODE>,
+    vec: brk_vec::StorableVec<I, T, MODE>,
 }
 
 impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
@@ -20,10 +20,10 @@ where
     I: StoredIndex,
     T: StoredType,
 {
-    pub fn import(path: &Path, version: Version) -> storable_vec::Result<Self> {
+    pub fn import(path: &Path, version: Version) -> brk_vec::Result<Self> {
         Ok(Self {
             height: Height::try_from(Self::path_height_(path).as_path()).ok(),
-            vec: storable_vec::StorableVec::forced_import(path, version)?,
+            vec: brk_vec::StorableVec::forced_import(path, version)?,
         })
     }
 
@@ -32,7 +32,7 @@ where
         self.vec.flush()
     }
 
-    pub fn truncate_if_needed(&mut self, index: I, height: Height) -> storable_vec::Result<Option<T>> {
+    pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<Option<T>> {
         if self.height.is_none_or(|self_height| self_height != height) {
             height.write(&self.path_height())?;
         }
@@ -51,7 +51,7 @@ where
 }
 
 impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> {
-    type Target = storable_vec::StorableVec<I, T, MODE>;
+    type Target = brk_vec::StorableVec<I, T, MODE>;
     fn deref(&self) -> &Self::Target {
         &self.vec
     }

crates/brk_indexer/src/storage/vecs/mod.rs

@@ -6,8 +6,8 @@ use brk_core::{
     P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
     P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
 };
+use brk_vec::{AnyJsonStorableVec, CACHED_GETS, Version};
 use rayon::prelude::*;
-use storable_vec::{AnyJsonStorableVec, CACHED_GETS, Version};
 
 use crate::Indexes;
 
@@ -15,7 +15,7 @@ mod base;
 
 pub use base::*;
 
-pub struct StorableVecs<const MODE: u8> {
+pub struct Vecs<const MODE: u8> {
     pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype, MODE>,
     pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex, MODE>,
     pub addressindex_to_height: StorableVec<Addressindex, Height, MODE>,
@@ -61,7 +61,7 @@ pub struct StorableVecs<const MODE: u8> {
     pub txoutindex_to_value: StorableVec<Txoutindex, Sats, MODE>,
 }
 
-impl<const MODE: u8> StorableVecs<MODE> {
+impl<const MODE: u8> Vecs<MODE> {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
@@ -187,7 +187,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
         })
     }
 
-    pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> {
+    pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
         let saved_height = starting_indexes.height.decremented();
 
         // We don't want to override the starting indexes so we cut from n + 1
@@ -404,12 +404,12 @@ impl<const MODE: u8> StorableVecs<MODE> {
     }
 }
 
-impl StorableVecs<CACHED_GETS> {
+impl Vecs<CACHED_GETS> {
     pub fn get_addressbytes(
         &self,
         addresstype: Addresstype,
         addresstypeindex: Addresstypeindex,
-    ) -> storable_vec::Result<Option<Addressbytes>> {
+    ) -> brk_vec::Result<Option<Addressbytes>> {
         Ok(match addresstype {
             Addresstype::P2PK65 => self
                 .p2pk65index_to_p2pk65addressbytes
@@ -454,7 +454,7 @@ impl StorableVecs<CACHED_GETS> {
         &mut self,
         index: Addresstypeindex,
         addressbytes: Addressbytes,
-    ) -> storable_vec::Result<()> {
+    ) -> brk_vec::Result<()> {
         match addressbytes {
             Addressbytes::P2PK65(bytes) => self
                 .p2pk65index_to_p2pk65addressbytes

crates/brk_indexer/src/structs/indexes.rs

@@ -4,10 +4,10 @@ use brk_core::{
     P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
 };
 use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
+use brk_vec::CACHED_GETS;
 use color_eyre::eyre::ContextCompat;
-use storable_vec::CACHED_GETS;
 
-use crate::storage::{Fjalls, StorableVecs};
+use crate::storage::{Stores, Vecs};
 
 #[derive(Debug, Default)]
 pub struct Indexes {
@@ -31,7 +31,7 @@ pub struct Indexes {
 }
 
 impl Indexes {
-    pub fn push_if_needed(&self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
+    pub fn push_if_needed(&self, vecs: &mut Vecs<CACHED_GETS>) -> brk_vec::Result<()> {
         let height = self.height;
         vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?;
         vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?;
@@ -64,7 +64,7 @@ impl Indexes {
         Ok(())
     }
 
-    pub fn push_future_if_needed(&mut self, vecs: &mut StorableVecs<CACHED_GETS>) -> storable_vec::Result<()> {
+    pub fn push_future_if_needed(&mut self, vecs: &mut Vecs<CACHED_GETS>) -> brk_vec::Result<()> {
         self.height.increment();
         self.push_if_needed(vecs)?;
         self.height.decrement();
@@ -72,11 +72,11 @@ impl Indexes {
     }
 }
 
-impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
+impl TryFrom<(&mut Vecs<CACHED_GETS>, &Stores, &Client)> for Indexes {
     type Error = color_eyre::Report;
-    fn try_from((vecs, trees, rpc): (&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)) -> color_eyre::Result<Self> {
+    fn try_from((vecs, stores, rpc): (&mut Vecs<CACHED_GETS>, &Stores, &Client)) -> color_eyre::Result<Self> {
         // Height at which we wanna start: min last saved + 1 or 0
-        let starting_height = vecs.starting_height().min(trees.starting_height());
+        let starting_height = vecs.starting_height().min(stores.starting_height());
 
         let range = starting_height
             .checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32)
@@ -93,7 +93,13 @@ impl TryFrom<(&mut StorableVecs<CACHED_GETS>, &Fjalls, &Client)> for Indexes {
                     .unwrap();
 
                 vecs.height_to_blockhash.get(*height).map_or(true, |opt| {
-                    opt.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref())
+                    opt.is_none_or(|saved_blockhash| {
+                        let b = &rpc_blockhash != saved_blockhash.as_ref();
+                        if b {
+                            dbg!(rpc_blockhash, saved_blockhash.as_ref());
+                        }
+                        b
+                    })
                 })
             })
             .unwrap_or(starting_height);

crates/brk_server/Cargo.toml

@@ -11,6 +11,7 @@ axum = "0.8.1"
 brk_computer = { workspace = true }
 brk_indexer = { workspace = true }
 brk_logger = { workspace = true }
+brk_vec = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
 jiff = { workspace = true }
@@ -18,6 +19,5 @@ log = { workspace = true }
 oxc = { version = "0.53.0", features = ["codegen", "minifier"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
-storable_vec = { workspace = true }
 tokio = { version = "1.43.0", features = ["full"] }
 tower-http = { version = "0.6.2", features = ["compression-full"] }

crates/brk_server/src/api/vecs/mod.rs

@@ -1,10 +1,10 @@
 use std::time::Instant;
 
 use axum::{
+    Json,
     extract::{Query, State},
     http::{HeaderMap, StatusCode, Uri},
     response::{IntoResponse, Response},
-    Json,
 };
 use color_eyre::eyre::eyre;
 use serde_json::Value;
@@ -81,8 +81,8 @@ fn req_to_response_res(
     let values = ids
         .iter()
         .flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap()))
-        .map(|vec| -> storable_vec::Result<Vec<Value>> { vec.collect_range_values(from, to) })
-        .collect::<storable_vec::Result<Vec<_>>>()?;
+        .map(|vec| -> brk_vec::Result<Vec<Value>> { vec.collect_range_values(from, to) })
+        .collect::<brk_vec::Result<Vec<_>>>()?;
 
     if ids.is_empty() {
         return Ok(Json(()).into_response());

crates/brk_server/src/api/vecs/tree.rs

@@ -1,7 +1,7 @@
 use std::{collections::BTreeMap, fs, io};
 
+use brk_vec::AnyJsonStorableVec;
 use derive_deref::{Deref, DerefMut};
-use storable_vec::AnyJsonStorableVec;
 
 use crate::WEBSITE_DEV_PATH;
 

crates/brk_server/src/lib.rs

@@ -4,10 +4,10 @@ use api::{ApiRoutes, VecIdToIndexToVec};
 use axum::{Json, Router, http::StatusCode, routing::get, serve};
 use brk_computer::Computer;
 use brk_indexer::Indexer;
+use brk_vec::STATELESS;
 use color_eyre::owo_colors::OwoColorize;
 use files::FilesRoutes;
 use log::{error, info};
-use storable_vec::STATELESS;
 use tokio::net::TcpListener;
 use tower_http::compression::CompressionLayer;
 

crates/brk_server/src/main.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use brk_computer::Computer;
 use brk_indexer::Indexer;
-use storable_vec::STATELESS;
+use brk_vec::STATELESS;
 
 #[tokio::main]
 pub async fn main() -> color_eyre::Result<()> {

crates/brk_vec/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-name = "storable_vec"
+name = "brk_vec"
 description = "A very small, fast, efficient and simple storable Vec"
-version = "0.1.3"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
+version.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true

crates/brk_vec/README.md

@@ -1,4 +1,4 @@
-# storable_vec
+# BRK Vec
 
 A very small, fast, efficient and simple storable `vec` which uses `mmap2` for speed.
 
@@ -15,7 +15,7 @@ A very small, fast, efficient and simple storable `vec` which uses `mmap2` for s
 ```rust
 use std::path::Path;
 
-use storable_vec::{AnyStorableVec, StorableVec};
+use brk_vec::{AnyStorableVec, StorableVec};
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;

crates/brk_vec/src/main.rs

@@ -1,6 +1,6 @@
 use std::path::Path;
 
-use storable_vec::{StorableVec, Version, CACHED_GETS, SINGLE_THREAD};
+use brk_vec::{CACHED_GETS, SINGLE_THREAD, StorableVec, Version};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     {

crates/storable_vec/LICENSE.md

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 storable_vec
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

publish.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+cd crates/brk
+
+cd ../brk_core
+cargo publish --allow-dirty
+
+cd ../brk_exit
+cargo publish --allow-dirty
+
+cd ../brk_vec
+cargo publish --allow-dirty
+
+cd ../brk_logger
+cargo publish --allow-dirty
+
+cd ../brk_indexer
+cargo publish --allow-dirty
+
+cd ../brk_parser
+cargo publish --allow-dirty
+
+cd ../brk_fetcher
+cargo publish --allow-dirty
+
+cd ../brk_computer
+cargo publish --allow-dirty
+
+cd ../brk_server
+cargo publish --allow-dirty
+
+cd ../brk_cli
+cargo publish --allow-dirty
+
+cd ../brk
+cargo publish --allow-dirty
+
+cd ../..