diff --git a/Cargo.lock b/Cargo.lock index 8d46b9174..afe2b6896 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -941,9 +941,9 @@ checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" [[package]] name = "h2" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" dependencies = [ "atomic-waker", "bytes", @@ -2976,9 +2976,9 @@ checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b" [[package]] name = "unicode-ident" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" +checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" [[package]] name = "unicode-linebreak" diff --git a/Cargo.toml b/Cargo.toml index 1356c50c9..09b788e62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ jiff = "0.2.1" logger = { path = "logger" } rayon = "1.10.0" pricer = { path = "pricer", package = "brice" } -rlimit = { version = "0.10.2" } serde = { version = "1.0.217", features = ["derive"] } serde_json = { version = "1.0.138", features = ["float_roundtrip"] } server = { path = "server", package = "berver" } diff --git a/computer/Cargo.toml b/computer/Cargo.toml index 35d646a2b..2dc009c9d 100644 --- a/computer/Cargo.toml +++ b/computer/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "bomputer" +description = "A Bitcoin dataset computer built on top of brice" version = "0.1.0" edition = "2021" diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index ea8f92c2a..c3e3e2391 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bindex" -description = "A bitcoin-core indexer" +description = "A bitcoin-core indexer built on top of biter" version = "0.1.0" edition = "2021" @@ -15,7 +15,7 @@ jiff = { workspace = true } logger = { workspace = true } rapidhash = "1.3.0" rayon = { workspace = true } -rlimit = { workspace = true } +rlimit = { version = "0.10.2" } serde = { workspace = true } serde_bytes = "0.11.15" storable_vec = { workspace = true } diff --git a/indexer/README.md b/indexer/README.md index 4d5c31bf9..78aff8cf9 100644 --- a/indexer/README.md +++ b/indexer/README.md @@ -1,19 +1,27 @@ # Indexer -A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can used in your Rust code. +A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code. The crate only stores the bare minimum to be self sufficient and not have to use an RPC client (except for scripts which are not stored). If you need more data, checkout `../computer` which uses the outputs from the indexer to compute a whole range of datasets. -The neat thing about using simple vecs to store data is that you can parse the outputed files with another programming language such as Python very simply, you'll find an example below. +Vecs are used sparingly instead of stores for multiple reasons: + +- Only stores the relevant data since the key is an index +- Saved as uncompressed bytes and thus can be parsed manually (with any programming language) without relying on a server or library +- Easy to work with and predictable ## Usage -Rust: `src/main.rs` - -Python: `../python/parse.py` +Peaks at 11-12 GB of RAM ## Outputs Vecs: `src/storage/storable_vecs/mod.rs` Stores: `src/storage/fjalls/mod.rs` + +## Examples + +Rust: `src/main.rs` + +Python: `../python/parse.py` diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 2ecd7dd9d..8418b57ed 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -31,6 +31,14 @@ pub struct Indexer { impl Indexer { pub fn import(indexes_dir: &Path) -> color_eyre::Result { + info!("Importing indexes..."); + + rlimit::setrlimit( + rlimit::Resource::NOFILE, + 210_000, + rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1, + )?; + let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?; let trees = Fjalls::import(&indexes_dir.join("fjall"))?; @@ -40,6 +48,8 @@ impl Indexer { impl Indexer { pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> { + info!("Started indexing..."); + let check_collisions = true; let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, &rpc)).unwrap_or_else(|_| { @@ -70,10 +80,10 @@ impl Indexer { iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc) .iter() - .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { - info!("Indexing block {_height}..."); + .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { + info!("Indexing block {height}..."); - let height = Height::from(_height); + let height = Height::from(height); idxs.height = height; let blockhash = BlockHash::from(blockhash); @@ -300,7 +310,7 @@ impl Indexer { { let txid = tx.compute_txid(); dbg!( - _height, + height, txid, vout, block_txindex, @@ -603,7 +613,7 @@ impl Indexer { idxs.push_future_if_needed(vecs)?; - let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked(); + let should_snapshot = height != 0 && height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked(); if should_snapshot { export(trees, vecs, height)?; } diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 3b7b6e7db..562711a7a 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -10,12 +10,6 @@ fn main() -> color_eyre::Result<()> { logger::init_log(None); - rlimit::setrlimit( - rlimit::Resource::NOFILE, - 21_000, - rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1, - )?; - let data_dir = Path::new("../../bitcoin"); let rpc = rpc::Client::new( "http://localhost:8332", diff --git a/iterator/Cargo.toml b/iterator/Cargo.toml index 943555b16..ef207a643 100644 --- a/iterator/Cargo.toml +++ b/iterator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "biter" -description = "A very fast Bitcoin block iterator" +description = "A very fast Bitcoin block iterator built on top of bitcoin-rust" version = "0.2.2" license = "MIT" repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter" diff --git a/pricer/Cargo.toml b/pricer/Cargo.toml index 55e4c724f..a031fa4a2 100644 --- a/pricer/Cargo.toml +++ b/pricer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brice" -description = "A bitcoin price fetcher" +description = "A bitcoin price fetcher built on top of bindex" version = "0.1.0" edition = "2021"