indexer: update readme

Cargo.lock

@@ -941,9 +941,9 @@ checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
 
 [[package]]
 name = "h2"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
+checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -2976,9 +2976,9 @@ checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
+checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
 
 [[package]]
 name = "unicode-linebreak"

Cargo.toml

@@ -24,7 +24,6 @@ jiff = "0.2.1"
 logger = { path = "logger" }
 rayon = "1.10.0"
 pricer = { path = "pricer", package = "brice" }
-rlimit = { version = "0.10.2" }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = { version = "1.0.138", features = ["float_roundtrip"] }
 server = { path = "server", package = "berver" }

computer/Cargo.toml

@@ -1,5 +1,6 @@
 [package]
 name = "bomputer"
+description = "A Bitcoin dataset computer built on top of brice"
 version = "0.1.0"
 edition = "2021"
 

indexer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bindex"
-description = "A bitcoin-core indexer"
+description = "A bitcoin-core indexer built on top of biter"
 version = "0.1.0"
 edition = "2021"
 
@@ -15,7 +15,7 @@ jiff = { workspace = true }
 logger = { workspace = true }
 rapidhash = "1.3.0"
 rayon = { workspace = true }
-rlimit = { workspace = true }
+rlimit = { version = "0.10.2" }
 serde = { workspace = true }
 serde_bytes = "0.11.15"
 storable_vec = { workspace = true }

indexer/README.md

@@ -1,19 +1,27 @@
 # Indexer
 
-A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can used in your Rust code.
+A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code.
 
 The crate only stores the bare minimum to be self sufficient and not have to use an RPC client (except for scripts which are not stored). If you need more data, checkout `../computer` which uses the outputs from the indexer to compute a whole range of datasets.
 
-The neat thing about using simple vecs to store data is that you can parse the outputed files with another programming language such as Python very simply, you'll find an example below.
+Vecs are used sparingly instead of stores for multiple reasons:
+
+- Only stores the relevant data since the key is an index
+- Saved as uncompressed bytes and thus can be parsed manually (with any programming language) without relying on a server or library
+- Easy to work with and predictable
 
 ## Usage
 
-Rust: `src/main.rs`
-
-Python: `../python/parse.py`
+Peaks at 11-12 GB of RAM
 
 ## Outputs
 
 Vecs: `src/storage/storable_vecs/mod.rs`
 
 Stores: `src/storage/fjalls/mod.rs`
+
+## Examples
+
+Rust: `src/main.rs`
+
+Python: `../python/parse.py`

indexer/src/lib.rs

@@ -31,6 +31,14 @@ pub struct Indexer<const MODE: u8> {
 
 impl<const MODE: u8> Indexer<MODE> {
     pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
+        info!("Importing indexes...");
+
+        rlimit::setrlimit(
+            rlimit::Resource::NOFILE,
+            210_000,
+            rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
+        )?;
+
         let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
         let trees = Fjalls::import(&indexes_dir.join("fjall"))?;
 
@@ -40,6 +48,8 @@ impl<const MODE: u8> Indexer<MODE> {
 
 impl Indexer<CACHED_GETS> {
     pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
+        info!("Started indexing...");
+
         let check_collisions = true;
 
         let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, &rpc)).unwrap_or_else(|_| {
@@ -70,10 +80,10 @@ impl Indexer<CACHED_GETS> {
 
         iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc)
             .iter()
-            .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
-                info!("Indexing block {_height}...");
+            .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
+                info!("Indexing block {height}...");
 
-                let height = Height::from(_height);
+                let height = Height::from(height);
                 idxs.height = height;
 
                 let blockhash = BlockHash::from(blockhash);
@@ -300,7 +310,7 @@ impl Indexer<CACHED_GETS> {
                                         {
                                             let txid = tx.compute_txid();
                                             dbg!(
-                                                _height,
+                                                height,
                                                 txid,
                                                 vout,
                                                 block_txindex,
@@ -603,7 +613,7 @@ impl Indexer<CACHED_GETS> {
 
                 idxs.push_future_if_needed(vecs)?;
 
-                let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked();
+                let should_snapshot = height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked();
                 if should_snapshot {
                     export(trees, vecs, height)?;
                 }

indexer/src/main.rs

@@ -10,12 +10,6 @@ fn main() -> color_eyre::Result<()> {
 
     logger::init_log(None);
 
-    rlimit::setrlimit(
-        rlimit::Resource::NOFILE,
-        21_000,
-        rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
-    )?;
-
     let data_dir = Path::new("../../bitcoin");
     let rpc = rpc::Client::new(
         "http://localhost:8332",

iterator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "biter"
-description = "A very fast Bitcoin block iterator"
+description = "A very fast Bitcoin block iterator built on top of bitcoin-rust"
 version = "0.2.2"
 license = "MIT"
 repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"

pricer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brice"
-description = "A bitcoin price fetcher"
+description = "A bitcoin price fetcher built on top of bindex"
 version = "0.1.0"
 edition = "2021"