diff --git a/Cargo.lock b/Cargo.lock index 8ffa62669..5b828bbea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,51 +324,63 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "brk" -version = "0.0.1" +version = "0.0.2" dependencies = [ "brk_computer", "brk_core", + "brk_exit", "brk_fetcher", "brk_indexer", "brk_logger", "brk_parser", "brk_server", + "brk_vec", ] [[package]] name = "brk_cli" -version = "0.0.1" +version = "0.0.2" [[package]] name = "brk_computer" -version = "0.0.1" +version = "0.0.2" dependencies = [ "brk_core", + "brk_exit", "brk_indexer", "brk_parser", + "brk_vec", "color-eyre", - "hodor", - "storable_vec", ] [[package]] name = "brk_core" -version = "0.0.1" +version = "0.0.2" dependencies = [ "bitcoin", "bitcoincore-rpc", "byteview", "derive_deref", "jiff", + "log", "rapidhash", + "rlimit", "serde", "serde_bytes", "zerocopy 0.8.20", ] +[[package]] +name = "brk_exit" +version = "0.0.2" +dependencies = [ + "ctrlc", + "log", +] + [[package]] name = "brk_fetcher" -version = "0.0.1" +version = "0.0.2" dependencies = [ "brk_core", "brk_logger", @@ -381,27 +393,26 @@ dependencies = [ [[package]] name = "brk_indexer" -version = "0.0.1" +version = "0.0.2" dependencies = [ "bitcoin", "bitcoincore-rpc", "brk_core", + "brk_exit", "brk_logger", "brk_parser", + "brk_vec", "byteview", "color-eyre", "fjall", - "hodor", "log", "rayon", - "rlimit", - "storable_vec", "zerocopy 0.8.20", ] [[package]] name = "brk_logger" -version = "0.0.1" +version = "0.0.2" dependencies = [ "color-eyre", "env_logger", @@ -410,7 +421,7 @@ dependencies = [ [[package]] name = "brk_parser" -version = "0.0.1" +version = "0.0.2" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -425,12 +436,13 @@ dependencies = [ [[package]] name = "brk_server" -version = "0.0.1" +version = "0.0.2" dependencies = [ "axum", "brk_computer", "brk_indexer", "brk_logger", + "brk_vec", "color-eyre", "derive_deref", "jiff", @@ -438,11 +450,21 @@ dependencies = [ "oxc", "serde", "serde_json", - "storable_vec", "tokio", "tower-http", ] +[[package]] +name = "brk_vec" +version = "0.0.2" +dependencies = [ + "memmap2", + "rayon", + "serde", + "serde_json", + "zerocopy 0.8.20", +] + [[package]] name = "brotli" version = "7.0.0" @@ -923,14 +945,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" -[[package]] -name = "hodor" -version = "0.1.1" -dependencies = [ - "ctrlc", - "log", -] - [[package]] name = "http" version = "1.2.0" @@ -2201,17 +2215,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" -[[package]] -name = "storable_vec" -version = "0.1.3" -dependencies = [ - "memmap2", - "rayon", - "serde", - "serde_json", - "zerocopy 0.8.20", -] - [[package]] name = "str_indices" version = "0.4.4" diff --git a/Cargo.toml b/Cargo.toml index 40b74541d..a502821a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" -package.version = "0.0.1" +package.version = "0.0.2" package.repository = "https://github.com/bitcoinresearchkit/brk" [workspace.dependencies] @@ -12,23 +12,21 @@ bitcoin = { version = "0.32.5", features = ["serde"] } bitcoincore-rpc = "0.19.0" brk_computer = { version = "0", path = "crates/brk_computer" } brk_core = { version = "0", path = "crates/brk_core" } +brk_exit = { version = "0", path = "crates/brk_exit" } brk_fetcher = { version = "0", path = "crates/brk_fetcher" } brk_indexer = { version = "0", path = "crates/brk_indexer" } -brk_parser = { version = "0", path = "crates/brk_parser" } brk_logger = { version = "0", path = "crates/brk_logger" } +brk_parser = { version = "0", path = "crates/brk_parser" } brk_server = { version = "0", path = "crates/brk_server" } +brk_vec = { version = "0", path = "crates/brk_vec", features = ["json"] } byteview = "0.5.4" color-eyre = "0.6.3" derive_deref = "1.1.1" fjall = "2.6.7" -hodor = { version = "0", path = "crates/hodor" } jiff = "0.2.1" log = { version = "0.4.26" } minreq = { version = "2.13.2", features = ["https", "serde_json"] } rayon = "1.10.0" serde = { version = "1.0.218", features = ["derive"] } serde_json = { version = "1.0.139", features = ["float_roundtrip"] } -storable_vec = { version = "0", path = "crates/storable_vec", features = [ - "json", -] } zerocopy = { version = "0.8.20", features = ["derive"] } diff --git a/README.md b/README.md index b4c50cd50..d22b78084 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ The Core (Structs and Errors) of the Bitcoin Research Kit. > Status: ✅ +### `brk_exit` + +An exit blocker built on top of ctrlc. + +> Status: ✅ + ### `brk_fetcher` A Bitcoin price fetcher. @@ -63,13 +69,7 @@ A server that serves Bitcoin data and swappable front-ends, built on top of brk_ > Status: ⚠️ -### `hodor` - -Hold the door, an exit blocker built on top of ctrlc. - -> Status: ✅ - -### `storable_vec` +### `brk_vec` A very small, fast, efficient and simple storable Vec. diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index b49427f7b..64f9018ad 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -8,23 +8,37 @@ edition.workspace = true version.workspace = true [features] -full = ["core", "computer", "fetcher", "indexer", "logger", "parser", "server"] +full = [ + "core", + "computer", + "exit", + "fetcher", + "indexer", + "logger", + "parser", + "server", + "vec", +] core = ["brk_core"] computer = ["brk_computer"] +exit = ["brk_exit"] fetcher = ["brk_fetcher"] indexer = ["brk_indexer"] logger = ["brk_logger"] parser = ["brk_parser"] server = ["brk_server"] +vec = ["brk_vec"] [dependencies] brk_core = { workspace = true, optional = true } brk_computer = { workspace = true, optional = true } +brk_exit = { workspace = true, optional = true } brk_fetcher = { workspace = true, optional = true } brk_indexer = { workspace = true, optional = true } brk_logger = { workspace = true, optional = true } brk_parser = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } +brk_vec = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index cc51d0c3c..450b43ce2 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -8,8 +8,8 @@ repository.workspace = true [dependencies] brk_core = { workspace = true } +brk_exit = { workspace = true } brk_indexer = { workspace = true } brk_parser = { workspace = true } +brk_vec = { workspace = true } color-eyre = { workspace = true } -hodor = { workspace = true } -storable_vec = { workspace = true } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index cddbc6bde..e0c000775 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -1,35 +1,35 @@ use std::path::{Path, PathBuf}; +use brk_exit::Exit; use brk_indexer::Indexer; pub use brk_parser::rpc; -use hodor::Hodor; mod storage; use brk_core::Date; -use storable_vec::SINGLE_THREAD; -use storage::{Fjalls, StorableVecs}; +use brk_vec::SINGLE_THREAD; +use storage::{Stores, Vecs}; pub struct Computer { path: PathBuf, - pub vecs: StorableVecs, - pub trees: Fjalls, + pub vecs: Vecs, + pub stores: Stores, } impl Computer { pub fn import(computed_dir: &Path) -> color_eyre::Result { - let vecs = StorableVecs::import(&computed_dir.join("vecs"))?; - let trees = Fjalls::import(&computed_dir.join("fjall"))?; + let vecs = Vecs::import(&computed_dir.join("vecs"))?; + let stores = Stores::import(&computed_dir.join("fjall"))?; Ok(Self { path: computed_dir.to_owned(), vecs, - trees, + stores, }) } } impl Computer { - pub fn compute(&mut self, mut indexer: Indexer, hodor: &Hodor) -> color_eyre::Result<()> { + pub fn compute(&mut self, mut indexer: Indexer, exit: &Exit) -> color_eyre::Result<()> { let height_count = indexer.vecs.height_to_size.len(); let txindexes_count = indexer.vecs.txindex_to_txid.len(); let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len(); diff --git a/crates/brk_computer/src/main.rs b/crates/brk_computer/src/main.rs index 2250a8540..863c96ac4 100644 --- a/crates/brk_computer/src/main.rs +++ b/crates/brk_computer/src/main.rs @@ -1,13 +1,13 @@ use std::path::Path; use brk_computer::Computer; +use brk_exit::Exit; use brk_indexer::Indexer; -use hodor::Hodor; pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; - let hodor = Hodor::new(); + let exit = Exit::new(); let i = std::time::Instant::now(); @@ -17,7 +17,7 @@ pub fn main() -> color_eyre::Result<()> { let mut computer = Computer::import(&outputs_dir.join("computed"))?; - computer.compute(indexer, &hodor)?; + computer.compute(indexer, &exit)?; dbg!(i.elapsed()); diff --git a/crates/brk_computer/src/storage/mod.rs b/crates/brk_computer/src/storage/mod.rs index 15d757c6a..96983bd40 100644 --- a/crates/brk_computer/src/storage/mod.rs +++ b/crates/brk_computer/src/storage/mod.rs @@ -1,5 +1,5 @@ -mod fjalls; -mod storable_vecs; +mod stores; +mod vecs; -pub use fjalls::*; -pub use storable_vecs::*; +pub use stores::*; +pub use vecs::*; diff --git a/crates/brk_computer/src/storage/fjalls.rs b/crates/brk_computer/src/storage/stores.rs similarity index 91% rename from crates/brk_computer/src/storage/fjalls.rs rename to crates/brk_computer/src/storage/stores.rs index f5af2c92a..93d0fd977 100644 --- a/crates/brk_computer/src/storage/fjalls.rs +++ b/crates/brk_computer/src/storage/stores.rs @@ -2,14 +2,14 @@ use std::path::Path; use brk_core::{AddressindexTxoutindex, Unit}; use brk_indexer::Store; -use storable_vec::Version; +use brk_vec::Version; -pub struct Fjalls { +pub struct Stores { pub address_to_utxos_received: Store, pub address_to_utxos_spent: Store, } -impl Fjalls { +impl Stores { pub fn import(path: &Path) -> color_eyre::Result { let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?; let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?; diff --git a/crates/brk_computer/src/storage/storable_vecs.rs b/crates/brk_computer/src/storage/vecs.rs similarity index 98% rename from crates/brk_computer/src/storage/storable_vecs.rs rename to crates/brk_computer/src/storage/vecs.rs index da879a04a..e8d330e87 100644 --- a/crates/brk_computer/src/storage/storable_vecs.rs +++ b/crates/brk_computer/src/storage/vecs.rs @@ -4,13 +4,13 @@ use brk_core::{ Addressindex, Cents, Close, Date, Dateindex, Dollars, Feerate, Height, High, Low, Open, Sats, Timestamp, Txindex, Txinindex, Txoutindex, }; -use storable_vec::{StorableVec, Version}; +use brk_vec::{StorableVec, Version}; // mod base; // use base::*; -pub struct StorableVecs { +pub struct Vecs { pub dateindex_to_first_height: StorableVec, // pub dateindex_to_last_height: StorableVec, // pub height_to_block_interval: StorableVec, @@ -56,7 +56,7 @@ pub struct StorableVecs { pub txindex_to_outputs_sum: StorableVec, } -impl StorableVecs { +impl Vecs { pub fn import(path: &Path) -> color_eyre::Result { fs::create_dir_all(path)?; diff --git a/crates/brk_core/Cargo.toml b/crates/brk_core/Cargo.toml index 5cda329b7..89eb7b4f8 100644 --- a/crates/brk_core/Cargo.toml +++ b/crates/brk_core/Cargo.toml @@ -12,7 +12,9 @@ bitcoincore-rpc = { workspace = true } byteview = { workspace = true } derive_deref = { workspace = true } jiff = { workspace = true } +log = { workspace = true } rapidhash = "1.4.0" +rlimit = "0.10.2" serde = { workspace = true } serde_bytes = "0.11.15" zerocopy = { workspace = true } diff --git a/crates/brk_core/src/lib.rs b/crates/brk_core/src/lib.rs index 1bbcf827b..9ff95d89d 100644 --- a/crates/brk_core/src/lib.rs +++ b/crates/brk_core/src/lib.rs @@ -1,5 +1,7 @@ mod error; mod structs; +mod utils; pub use error::*; pub use structs::*; +pub use utils::*; diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 25040c983..4a4d11ae9 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -177,7 +177,7 @@ impl From for u64 { impl TryFrom<&Client> for Height { type Error = bitcoincore_rpc::Error; fn try_from(value: &Client) -> Result { - Ok((value.get_blockchain_info()?.blocks as usize - 1).into()) + Ok((value.get_block_count()? as usize).into()) } } diff --git a/crates/brk_core/src/utils/mod.rs b/crates/brk_core/src/utils/mod.rs new file mode 100644 index 000000000..cddefd60c --- /dev/null +++ b/crates/brk_core/src/utils/mod.rs @@ -0,0 +1,5 @@ +mod pause; +mod rlimit; + +pub use pause::*; +pub use rlimit::*; diff --git a/crates/brk_core/src/utils/pause.rs b/crates/brk_core/src/utils/pause.rs new file mode 100644 index 000000000..c46095dce --- /dev/null +++ b/crates/brk_core/src/utils/pause.rs @@ -0,0 +1,7 @@ +use log::info; + +pub fn pause() { + info!("Press enter to continue..."); + let mut buffer = String::new(); + std::io::stdin().read_line(&mut buffer).expect("Failed to read line"); +} diff --git a/crates/brk_core/src/utils/rlimit.rs b/crates/brk_core/src/utils/rlimit.rs new file mode 100644 index 000000000..4b5622fa1 --- /dev/null +++ b/crates/brk_core/src/utils/rlimit.rs @@ -0,0 +1,10 @@ +use std::io; + +use rlimit::{Resource, getrlimit}; + +pub fn setrlimit() -> io::Result<()> { + let no_file_limit = getrlimit(Resource::NOFILE)?; + rlimit::setrlimit(Resource::NOFILE, no_file_limit.0.max(210_000), no_file_limit.1)?; + + Ok(()) +} diff --git a/crates/hodor/Cargo.toml b/crates/brk_exit/Cargo.toml similarity index 59% rename from crates/hodor/Cargo.toml rename to crates/brk_exit/Cargo.toml index a069ca0ea..713a7a79e 100644 --- a/crates/hodor/Cargo.toml +++ b/crates/brk_exit/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "hodor" -description = "Hold the door, an exit blocker built on top of ctrlc" -version = "0.1.1" +name = "brk_exit" +description = "An exit blocker built on top of ctrlc" +version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true diff --git a/crates/hodor/src/lib.rs b/crates/brk_exit/src/lib.rs similarity index 67% rename from crates/hodor/src/lib.rs rename to crates/brk_exit/src/lib.rs index 9f23494ae..ae207df15 100644 --- a/crates/hodor/src/lib.rs +++ b/crates/brk_exit/src/lib.rs @@ -11,32 +11,32 @@ use std::{ use log::info; #[derive(Default, Clone)] -pub struct Hodor { - holding: Arc, +pub struct Exit { + blocking: Arc, triggered: Arc, } -impl Hodor { +impl Exit { pub fn new() -> Self { let s = Self { triggered: Arc::new(AtomicBool::new(false)), - holding: Arc::new(AtomicBool::new(false)), + blocking: Arc::new(AtomicBool::new(false)), }; let triggered = s.triggered.clone(); - let holding = s.holding.clone(); - let is_holding = move || holding.load(Ordering::SeqCst); + let blocking = s.blocking.clone(); + let is_blocking = move || blocking.load(Ordering::SeqCst); ctrlc::set_handler(move || { info!("Exitting..."); triggered.store(true, Ordering::SeqCst); - if is_holding() { + if is_blocking() { info!("Waiting to exit safely..."); - while is_holding() { + while is_blocking() { sleep(Duration::from_millis(50)); } } @@ -48,12 +48,12 @@ impl Hodor { s } - pub fn hold(&self) { - self.holding.store(true, Ordering::SeqCst); + pub fn block(&self) { + self.blocking.store(true, Ordering::SeqCst); } pub fn release(&self) { - self.holding.store(false, Ordering::SeqCst); + self.blocking.store(false, Ordering::SeqCst); } pub fn triggered(&self) -> bool { diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 09bfda837..b8c9ed309 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -10,14 +10,13 @@ repository.workspace = true bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_core = { workspace = true } +brk_exit = { workspace = true } brk_parser = { workspace = true } brk_logger = { workspace = true } +brk_vec = { workspace = true } byteview = { workspace = true } color-eyre = { workspace = true } fjall = { workspace = true } -hodor = { workspace = true } log = { workspace = true } rayon = { workspace = true } -rlimit = { version = "0.10.2" } -storable_vec = { workspace = true } zerocopy = { workspace = true } diff --git a/crates/brk_indexer/README.md b/crates/brk_indexer/README.md index 4021af02e..8274828cb 100644 --- a/crates/brk_indexer/README.md +++ b/crates/brk_indexer/README.md @@ -1,6 +1,6 @@ # BRK Indexer -A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../iterator`) and creates a database of the vecs (`../storable_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code. +A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../brk_parser`) and creates a database of the vecs (`../brk_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code. The crate only stores the bare minimum to be self sufficient and not have to use an RPC client (except for scripts which are not stored). If you need more data, checkout `../computer` which uses the outputs from the indexer to compute a whole range of datasets. @@ -18,9 +18,9 @@ Peaks at 11-13 GB of RAM ## Outputs -Vecs: `src/storage/storable_vecs/mod.rs` +Vecs: `src/storage/vecs/mod.rs` -Stores: `src/storage/fjalls/mod.rs` +Stores: `src/storage/stores/mod.rs` ## Examples diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index b89557dd0..7a5c6bedf 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -2,55 +2,52 @@ use std::{ collections::BTreeMap, path::Path, str::FromStr, - thread::{self, sleep}, - time::Duration, + thread::{self}, }; use brk_core::{ AddressHash, Addressbytes, Addressindex, Addresstype, BlockHash, BlockHashPrefix, Height, Sats, Timestamp, Txid, - TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, + TxidPrefix, Txindex, Txinindex, Txoutindex, Vin, Vout, setrlimit, }; pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; +use brk_exit::Exit; +use brk_vec::CACHED_GETS; use color_eyre::eyre::{ContextCompat, eyre}; -use hodor::Hodor; use log::info; use rayon::prelude::*; -use storable_vec::CACHED_GETS; mod storage; mod structs; pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta}; -use storage::{Fjalls, StorableVecs}; +use storage::{Stores, Vecs}; pub use structs::*; const SNAPSHOT_BLOCK_RANGE: usize = 1000; pub struct Indexer { - pub vecs: StorableVecs, - pub stores: Fjalls, + pub vecs: Vecs, + pub stores: Stores, } impl Indexer { pub fn import(indexes_dir: &Path) -> color_eyre::Result { - rlimit::setrlimit( - rlimit::Resource::NOFILE, - rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0.max(210_000), - rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1, - )?; + setrlimit()?; info!("Importing indexes..."); - let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?; - let stores = Fjalls::import(&indexes_dir.join("fjall"))?; + + let vecs = Vecs::import(&indexes_dir.join("vecs"))?; + + let stores = Stores::import(&indexes_dir.join("fjall"))?; Ok(Self { vecs, stores }) } } impl Indexer { - pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, hodor: &Hodor) -> color_eyre::Result<()> { + pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> { let check_collisions = true; let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| { @@ -59,27 +56,26 @@ impl Indexer { indexes }); - hodor.hold(); + exit.block(); self.stores.rollback_if_needed(&self.vecs, &starting_indexes)?; self.vecs.rollback_if_needed(&starting_indexes)?; - hodor.release(); + exit.release(); - let export_if_needed = |stores: &mut Fjalls, - vecs: &mut StorableVecs, + let export_if_needed = |stores: &mut Stores, + vecs: &mut Vecs, height: Height, - hodor: &Hodor| + rem: bool, + exit: &Exit| -> color_eyre::Result<()> { - if height == 0 || height % SNAPSHOT_BLOCK_RANGE != 0 || hodor.triggered() { + if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() { return Ok(()); } info!("Exporting..."); - hodor.hold(); + exit.block(); stores.commit(height)?; - info!("Exported stores"); vecs.flush(height)?; - info!("Exported vecs"); - hodor.release(); + exit.release(); Ok(()) }; @@ -88,7 +84,7 @@ impl Indexer { let mut idxs = starting_indexes; - if idxs.height >= Height::try_from(rpc)? { + if idxs.height > Height::try_from(rpc)? { return Ok(()); } @@ -629,16 +625,13 @@ impl Indexer { idxs.push_future_if_needed(vecs)?; - export_if_needed(stores, vecs, height, hodor)?; + export_if_needed(stores, vecs, height, false, exit)?; Ok(()) }, )?; - export_if_needed(stores, vecs, idxs.height, hodor)?; - - // To make sure that Fjall had the time to flush everything properly - sleep(Duration::from_millis(100)); + export_if_needed(stores, vecs, idxs.height, true, exit)?; Ok(()) } diff --git a/crates/brk_indexer/src/main.rs b/crates/brk_indexer/src/main.rs index a65bfc310..92f5fe2b0 100644 --- a/crates/brk_indexer/src/main.rs +++ b/crates/brk_indexer/src/main.rs @@ -1,13 +1,13 @@ use std::{path::Path, thread::sleep, time::Duration}; +use brk_exit::Exit; use brk_indexer::{Indexer, rpc::RpcApi}; use brk_parser::{ Parser, rpc::{self}, }; -use hodor::Hodor; +use brk_vec::CACHED_GETS; use log::info; -use storable_vec::CACHED_GETS; fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -19,27 +19,27 @@ fn main() -> color_eyre::Result<()> { "http://localhost:8332", rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), )?)); - let hodor = Hodor::new(); + let exit = Exit::new(); let parser = Parser::new(data_dir, rpc); + let mut indexer: Indexer = Indexer::import(Path::new("../../_outputs/indexes"))?; + loop { - let block_count = rpc.get_blockchain_info().unwrap().blocks as usize; + let block_count = rpc.get_block_count()?; info!("{block_count} blocks found."); - let i = std::time::Instant::now(); + indexer.index(&parser, rpc, &exit)?; - let mut indexer: Indexer = Indexer::import(Path::new("../../_outputs/indexes"))?; + info!("Waiting for new blocks..."); - indexer.index(&parser, rpc, &hodor)?; - - info!("Took: {:?}", i.elapsed()); - - info!("Waiting for a new block..."); - - while block_count == rpc.get_blockchain_info().unwrap().blocks as usize { + while block_count == rpc.get_block_count()? { sleep(Duration::from_secs(1)) } } + + #[allow(unreachable_code)] + // To make sure that Fjall had the time to flush everything properly + sleep(Duration::from_millis(100)); } diff --git a/crates/brk_indexer/src/storage/mod.rs b/crates/brk_indexer/src/storage/mod.rs index 15d757c6a..96983bd40 100644 --- a/crates/brk_indexer/src/storage/mod.rs +++ b/crates/brk_indexer/src/storage/mod.rs @@ -1,5 +1,5 @@ -mod fjalls; -mod storable_vecs; +mod stores; +mod vecs; -pub use fjalls::*; -pub use storable_vecs::*; +pub use stores::*; +pub use vecs::*; diff --git a/crates/brk_indexer/src/storage/fjalls/base.rs b/crates/brk_indexer/src/storage/stores/base.rs similarity index 99% rename from crates/brk_indexer/src/storage/fjalls/base.rs rename to crates/brk_indexer/src/storage/stores/base.rs index 968ef0b03..b10e20727 100644 --- a/crates/brk_indexer/src/storage/fjalls/base.rs +++ b/crates/brk_indexer/src/storage/stores/base.rs @@ -7,11 +7,11 @@ use std::{ }; use brk_core::Height; +use brk_vec::{Value, Version}; use byteview::ByteView; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace, TransactionalPartitionHandle, }; -use storable_vec::{Value, Version}; use zerocopy::{Immutable, IntoBytes}; use super::StoreMeta; diff --git a/crates/brk_indexer/src/storage/fjalls/meta.rs b/crates/brk_indexer/src/storage/stores/meta.rs similarity index 98% rename from crates/brk_indexer/src/storage/fjalls/meta.rs rename to crates/brk_indexer/src/storage/stores/meta.rs index 85f45e2af..a87885253 100644 --- a/crates/brk_indexer/src/storage/fjalls/meta.rs +++ b/crates/brk_indexer/src/storage/stores/meta.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use storable_vec::Version; +use brk_vec::Version; use zerocopy::{FromBytes, IntoBytes}; use super::Height; diff --git a/crates/brk_indexer/src/storage/fjalls/mod.rs b/crates/brk_indexer/src/storage/stores/mod.rs similarity index 97% rename from crates/brk_indexer/src/storage/fjalls/mod.rs rename to crates/brk_indexer/src/storage/stores/mod.rs index 2c7b601cd..cb712b57e 100644 --- a/crates/brk_indexer/src/storage/fjalls/mod.rs +++ b/crates/brk_indexer/src/storage/stores/mod.rs @@ -1,7 +1,7 @@ use std::{path::Path, thread}; use brk_core::{AddressHash, Addressbytes, Addressindex, Addresstype, BlockHashPrefix, Height, TxidPrefix, Txindex}; -use storable_vec::{CACHED_GETS, Value, Version}; +use brk_vec::{CACHED_GETS, Value, Version}; use crate::Indexes; @@ -11,15 +11,15 @@ mod meta; pub use base::*; pub use meta::*; -use super::StorableVecs; +use super::Vecs; -pub struct Fjalls { +pub struct Stores { pub addresshash_to_addressindex: Store, pub blockhash_prefix_to_height: Store, pub txid_prefix_to_txindex: Store, } -impl Fjalls { +impl Stores { pub fn import(path: &Path) -> color_eyre::Result { thread::scope(|scope| { let addresshash_to_addressindex = @@ -39,7 +39,7 @@ impl Fjalls { pub fn rollback_if_needed( &mut self, - vecs: &StorableVecs, + vecs: &Vecs, starting_indexes: &Indexes, ) -> color_eyre::Result<()> { vecs.height_to_blockhash diff --git a/crates/brk_indexer/src/storage/storable_vecs/base.rs b/crates/brk_indexer/src/storage/vecs/base.rs similarity index 83% rename from crates/brk_indexer/src/storage/storable_vecs/base.rs rename to crates/brk_indexer/src/storage/vecs/base.rs index 05e1a53b8..53280f761 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/base.rs +++ b/crates/brk_indexer/src/storage/vecs/base.rs @@ -5,14 +5,14 @@ use std::{ path::{Path, PathBuf}, }; -use storable_vec::{StoredIndex, StoredType, Version}; +use brk_vec::{StoredIndex, StoredType, Version}; use super::Height; #[derive(Debug)] pub struct StorableVec { height: Option, - vec: storable_vec::StorableVec, + vec: brk_vec::StorableVec, } impl StorableVec @@ -20,10 +20,10 @@ where I: StoredIndex, T: StoredType, { - pub fn import(path: &Path, version: Version) -> storable_vec::Result { + pub fn import(path: &Path, version: Version) -> brk_vec::Result { Ok(Self { height: Height::try_from(Self::path_height_(path).as_path()).ok(), - vec: storable_vec::StorableVec::forced_import(path, version)?, + vec: brk_vec::StorableVec::forced_import(path, version)?, }) } @@ -32,7 +32,7 @@ where self.vec.flush() } - pub fn truncate_if_needed(&mut self, index: I, height: Height) -> storable_vec::Result> { + pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result> { if self.height.is_none_or(|self_height| self_height != height) { height.write(&self.path_height())?; } @@ -51,7 +51,7 @@ where } impl Deref for StorableVec { - type Target = storable_vec::StorableVec; + type Target = brk_vec::StorableVec; fn deref(&self) -> &Self::Target { &self.vec } diff --git a/crates/brk_indexer/src/storage/storable_vecs/mod.rs b/crates/brk_indexer/src/storage/vecs/mod.rs similarity index 98% rename from crates/brk_indexer/src/storage/storable_vecs/mod.rs rename to crates/brk_indexer/src/storage/vecs/mod.rs index 0ea9ebaf5..0eedf60e0 100644 --- a/crates/brk_indexer/src/storage/storable_vecs/mod.rs +++ b/crates/brk_indexer/src/storage/vecs/mod.rs @@ -6,8 +6,8 @@ use brk_core::{ P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, }; +use brk_vec::{AnyJsonStorableVec, CACHED_GETS, Version}; use rayon::prelude::*; -use storable_vec::{AnyJsonStorableVec, CACHED_GETS, Version}; use crate::Indexes; @@ -15,7 +15,7 @@ mod base; pub use base::*; -pub struct StorableVecs { +pub struct Vecs { pub addressindex_to_addresstype: StorableVec, pub addressindex_to_addresstypeindex: StorableVec, pub addressindex_to_height: StorableVec, @@ -61,7 +61,7 @@ pub struct StorableVecs { pub txoutindex_to_value: StorableVec, } -impl StorableVecs { +impl Vecs { pub fn import(path: &Path) -> color_eyre::Result { fs::create_dir_all(path)?; @@ -187,7 +187,7 @@ impl StorableVecs { }) } - pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> storable_vec::Result<()> { + pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> { let saved_height = starting_indexes.height.decremented(); // We don't want to override the starting indexes so we cut from n + 1 @@ -404,12 +404,12 @@ impl StorableVecs { } } -impl StorableVecs { +impl Vecs { pub fn get_addressbytes( &self, addresstype: Addresstype, addresstypeindex: Addresstypeindex, - ) -> storable_vec::Result> { + ) -> brk_vec::Result> { Ok(match addresstype { Addresstype::P2PK65 => self .p2pk65index_to_p2pk65addressbytes @@ -454,7 +454,7 @@ impl StorableVecs { &mut self, index: Addresstypeindex, addressbytes: Addressbytes, - ) -> storable_vec::Result<()> { + ) -> brk_vec::Result<()> { match addressbytes { Addressbytes::P2PK65(bytes) => self .p2pk65index_to_p2pk65addressbytes diff --git a/crates/brk_indexer/src/structs/indexes.rs b/crates/brk_indexer/src/structs/indexes.rs index a35fcfdab..93c1e8285 100644 --- a/crates/brk_indexer/src/structs/indexes.rs +++ b/crates/brk_indexer/src/structs/indexes.rs @@ -4,10 +4,10 @@ use brk_core::{ P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, }; use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; +use brk_vec::CACHED_GETS; use color_eyre::eyre::ContextCompat; -use storable_vec::CACHED_GETS; -use crate::storage::{Fjalls, StorableVecs}; +use crate::storage::{Stores, Vecs}; #[derive(Debug, Default)] pub struct Indexes { @@ -31,7 +31,7 @@ pub struct Indexes { } impl Indexes { - pub fn push_if_needed(&self, vecs: &mut StorableVecs) -> storable_vec::Result<()> { + pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> { let height = self.height; vecs.height_to_first_txindex.push_if_needed(height, self.txindex)?; vecs.height_to_first_txinindex.push_if_needed(height, self.txinindex)?; @@ -64,7 +64,7 @@ impl Indexes { Ok(()) } - pub fn push_future_if_needed(&mut self, vecs: &mut StorableVecs) -> storable_vec::Result<()> { + pub fn push_future_if_needed(&mut self, vecs: &mut Vecs) -> brk_vec::Result<()> { self.height.increment(); self.push_if_needed(vecs)?; self.height.decrement(); @@ -72,11 +72,11 @@ impl Indexes { } } -impl TryFrom<(&mut StorableVecs, &Fjalls, &Client)> for Indexes { +impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes { type Error = color_eyre::Report; - fn try_from((vecs, trees, rpc): (&mut StorableVecs, &Fjalls, &Client)) -> color_eyre::Result { + fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result { // Height at which we wanna start: min last saved + 1 or 0 - let starting_height = vecs.starting_height().min(trees.starting_height()); + let starting_height = vecs.starting_height().min(stores.starting_height()); let range = starting_height .checked_sub(NUMBER_OF_UNSAFE_BLOCKS as u32) @@ -93,7 +93,13 @@ impl TryFrom<(&mut StorableVecs, &Fjalls, &Client)> for Indexes { .unwrap(); vecs.height_to_blockhash.get(*height).map_or(true, |opt| { - opt.is_none_or(|saved_blockhash| &rpc_blockhash != saved_blockhash.as_ref()) + opt.is_none_or(|saved_blockhash| { + let b = &rpc_blockhash != saved_blockhash.as_ref(); + if b { + dbg!(rpc_blockhash, saved_blockhash.as_ref()); + } + b + }) }) }) .unwrap_or(starting_height); diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index e7021f5c2..a6c09ba3d 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -11,6 +11,7 @@ axum = "0.8.1" brk_computer = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } +brk_vec = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } jiff = { workspace = true } @@ -18,6 +19,5 @@ log = { workspace = true } oxc = { version = "0.53.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } -storable_vec = { workspace = true } tokio = { version = "1.43.0", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } diff --git a/crates/brk_server/src/api/vecs/mod.rs b/crates/brk_server/src/api/vecs/mod.rs index baa3200db..7f07a6bbe 100644 --- a/crates/brk_server/src/api/vecs/mod.rs +++ b/crates/brk_server/src/api/vecs/mod.rs @@ -1,10 +1,10 @@ use std::time::Instant; use axum::{ + Json, extract::{Query, State}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, - Json, }; use color_eyre::eyre::eyre; use serde_json::Value; @@ -81,8 +81,8 @@ fn req_to_response_res( let values = ids .iter() .flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap())) - .map(|vec| -> storable_vec::Result> { vec.collect_range_values(from, to) }) - .collect::>>()?; + .map(|vec| -> brk_vec::Result> { vec.collect_range_values(from, to) }) + .collect::>>()?; if ids.is_empty() { return Ok(Json(()).into_response()); diff --git a/crates/brk_server/src/api/vecs/tree.rs b/crates/brk_server/src/api/vecs/tree.rs index d3bc4e7e2..3e08950bc 100644 --- a/crates/brk_server/src/api/vecs/tree.rs +++ b/crates/brk_server/src/api/vecs/tree.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, fs, io}; +use brk_vec::AnyJsonStorableVec; use derive_deref::{Deref, DerefMut}; -use storable_vec::AnyJsonStorableVec; use crate::WEBSITE_DEV_PATH; diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index 8cdc6423f..4acd76f7f 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -4,10 +4,10 @@ use api::{ApiRoutes, VecIdToIndexToVec}; use axum::{Json, Router, http::StatusCode, routing::get, serve}; use brk_computer::Computer; use brk_indexer::Indexer; +use brk_vec::STATELESS; use color_eyre::owo_colors::OwoColorize; use files::FilesRoutes; use log::{error, info}; -use storable_vec::STATELESS; use tokio::net::TcpListener; use tower_http::compression::CompressionLayer; diff --git a/crates/brk_server/src/main.rs b/crates/brk_server/src/main.rs index 7d3e07b7d..e21b3a6bf 100644 --- a/crates/brk_server/src/main.rs +++ b/crates/brk_server/src/main.rs @@ -2,7 +2,7 @@ use std::path::Path; use brk_computer::Computer; use brk_indexer::Indexer; -use storable_vec::STATELESS; +use brk_vec::STATELESS; #[tokio::main] pub async fn main() -> color_eyre::Result<()> { diff --git a/crates/storable_vec/.gitignore b/crates/brk_vec/.gitignore similarity index 100% rename from crates/storable_vec/.gitignore rename to crates/brk_vec/.gitignore diff --git a/crates/storable_vec/Cargo.lock b/crates/brk_vec/Cargo.lock similarity index 100% rename from crates/storable_vec/Cargo.lock rename to crates/brk_vec/Cargo.lock diff --git a/crates/storable_vec/Cargo.toml b/crates/brk_vec/Cargo.toml similarity index 91% rename from crates/storable_vec/Cargo.toml rename to crates/brk_vec/Cargo.toml index ebee0f823..82b897d9b 100644 --- a/crates/storable_vec/Cargo.toml +++ b/crates/brk_vec/Cargo.toml @@ -1,9 +1,9 @@ [package] -name = "storable_vec" +name = "brk_vec" description = "A very small, fast, efficient and simple storable Vec" -version = "0.1.3" keywords = ["vec", "disk", "data"] categories = ["database"] +version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true diff --git a/crates/storable_vec/README.md b/crates/brk_vec/README.md similarity index 94% rename from crates/storable_vec/README.md rename to crates/brk_vec/README.md index e098e2cd4..f0d6d3854 100644 --- a/crates/storable_vec/README.md +++ b/crates/brk_vec/README.md @@ -1,4 +1,4 @@ -# storable_vec +# BRK Vec A very small, fast, efficient and simple storable `vec` which uses `mmap2` for speed. @@ -15,7 +15,7 @@ A very small, fast, efficient and simple storable `vec` which uses `mmap2` for s ```rust use std::path::Path; -use storable_vec::{AnyStorableVec, StorableVec}; +use brk_vec::{AnyStorableVec, StorableVec}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; diff --git a/crates/storable_vec/src/enums/error.rs b/crates/brk_vec/src/enums/error.rs similarity index 100% rename from crates/storable_vec/src/enums/error.rs rename to crates/brk_vec/src/enums/error.rs diff --git a/crates/storable_vec/src/enums/mod.rs b/crates/brk_vec/src/enums/mod.rs similarity index 100% rename from crates/storable_vec/src/enums/mod.rs rename to crates/brk_vec/src/enums/mod.rs diff --git a/crates/storable_vec/src/enums/value.rs b/crates/brk_vec/src/enums/value.rs similarity index 100% rename from crates/storable_vec/src/enums/value.rs rename to crates/brk_vec/src/enums/value.rs diff --git a/crates/storable_vec/src/lib.rs b/crates/brk_vec/src/lib.rs similarity index 100% rename from crates/storable_vec/src/lib.rs rename to crates/brk_vec/src/lib.rs diff --git a/crates/storable_vec/src/main.rs b/crates/brk_vec/src/main.rs similarity index 90% rename from crates/storable_vec/src/main.rs rename to crates/brk_vec/src/main.rs index 15c162146..2807bb612 100644 --- a/crates/storable_vec/src/main.rs +++ b/crates/brk_vec/src/main.rs @@ -1,6 +1,6 @@ use std::path::Path; -use storable_vec::{StorableVec, Version, CACHED_GETS, SINGLE_THREAD}; +use brk_vec::{CACHED_GETS, SINGLE_THREAD, StorableVec, Version}; fn main() -> Result<(), Box> { { diff --git a/crates/storable_vec/src/structs/mod.rs b/crates/brk_vec/src/structs/mod.rs similarity index 100% rename from crates/storable_vec/src/structs/mod.rs rename to crates/brk_vec/src/structs/mod.rs diff --git a/crates/storable_vec/src/structs/unsafe_slice.rs b/crates/brk_vec/src/structs/unsafe_slice.rs similarity index 100% rename from crates/storable_vec/src/structs/unsafe_slice.rs rename to crates/brk_vec/src/structs/unsafe_slice.rs diff --git a/crates/storable_vec/src/structs/version.rs b/crates/brk_vec/src/structs/version.rs similarity index 100% rename from crates/storable_vec/src/structs/version.rs rename to crates/brk_vec/src/structs/version.rs diff --git a/crates/storable_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs similarity index 100% rename from crates/storable_vec/src/traits/any.rs rename to crates/brk_vec/src/traits/any.rs diff --git a/crates/storable_vec/src/traits/bytes.rs b/crates/brk_vec/src/traits/bytes.rs similarity index 100% rename from crates/storable_vec/src/traits/bytes.rs rename to crates/brk_vec/src/traits/bytes.rs diff --git a/crates/storable_vec/src/traits/mod.rs b/crates/brk_vec/src/traits/mod.rs similarity index 100% rename from crates/storable_vec/src/traits/mod.rs rename to crates/brk_vec/src/traits/mod.rs diff --git a/crates/storable_vec/src/traits/stored_index.rs b/crates/brk_vec/src/traits/stored_index.rs similarity index 100% rename from crates/storable_vec/src/traits/stored_index.rs rename to crates/brk_vec/src/traits/stored_index.rs diff --git a/crates/storable_vec/src/traits/stored_type.rs b/crates/brk_vec/src/traits/stored_type.rs similarity index 100% rename from crates/storable_vec/src/traits/stored_type.rs rename to crates/brk_vec/src/traits/stored_type.rs diff --git a/crates/storable_vec/LICENSE.md b/crates/storable_vec/LICENSE.md deleted file mode 100644 index bd3da5717..000000000 --- a/crates/storable_vec/LICENSE.md +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 storable_vec - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/publish.sh b/publish.sh new file mode 100755 index 000000000..49cfbefd7 --- /dev/null +++ b/publish.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +cd crates/brk + +cd ../brk_core +cargo publish --allow-dirty + +cd ../brk_exit +cargo publish --allow-dirty + +cd ../brk_vec +cargo publish --allow-dirty + +cd ../brk_logger +cargo publish --allow-dirty + +cd ../brk_indexer +cargo publish --allow-dirty + +cd ../brk_parser +cargo publish --allow-dirty + +cd ../brk_fetcher +cargo publish --allow-dirty + +cd ../brk_computer +cargo publish --allow-dirty + +cd ../brk_server +cargo publish --allow-dirty + +cd ../brk_cli +cargo publish --allow-dirty + +cd ../brk +cargo publish --allow-dirty + +cd ../..