diff --git a/Cargo.lock b/Cargo.lock index 27ea3f0a8..575ab6cd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,6 +327,7 @@ name = "brk" version = "0.0.0" dependencies = [ "brk_computer", + "brk_core", "brk_fetcher", "brk_indexer", "brk_logger", @@ -701,9 +702,9 @@ checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "enum_dispatch" @@ -797,9 +798,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide 0.8.5", @@ -1782,9 +1783,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "portable-atomic-util" diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 9dd70b5dc..4eb1c0886 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -6,7 +6,8 @@ edition.workspace = true version.workspace = true [features] -full = ["computer", "fetcher", "indexer", "logger", "parser", "server"] +full = ["core", "computer", "fetcher", "indexer", "logger", "parser", "server"] +core = ["brk_core"] computer = ["brk_computer"] fetcher = ["brk_fetcher"] indexer = ["brk_indexer"] @@ -15,6 +16,7 @@ parser = ["brk_parser"] server = ["brk_server"] [dependencies] +brk_core = { workspace = true, optional = true } brk_computer = { workspace = true, optional = true } brk_fetcher = { workspace = true, optional = true } brk_indexer = { workspace = true, optional = true } diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index 458340dca..924ef762b 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -1,3 +1,9 @@ +#[cfg(feature = "core")] +pub mod core { + #[doc(inline)] + pub use brk_core::*; +} + #[cfg(feature = "computer")] pub mod computer { #[doc(inline)] diff --git a/crates/brk_core/src/structs/height.rs b/crates/brk_core/src/structs/height.rs index 9d21ad5bc..25040c983 100644 --- a/crates/brk_core/src/structs/height.rs +++ b/crates/brk_core/src/structs/height.rs @@ -32,6 +32,10 @@ impl Height { pub const ZERO: Self = Height(0); pub const MAX: Self = Height(u32::MAX); + pub fn new(height: u32) -> Self { + Self(height) + } + pub fn write(&self, path: &std::path::Path) -> Result<(), std::io::Error> { std::fs::write(path, self.as_bytes()) } diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index fb932ec00..a9c06529a 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -60,6 +60,9 @@ impl Indexer { indexes }); + // dbg!(starting_indexes); + // panic!(); + exit.block(); self.stores.rollback(&self.vecs, &starting_indexes)?; self.vecs.rollback(&starting_indexes)?; @@ -88,7 +91,7 @@ impl Indexer { info!("Started indexing..."); - parser.parse(Some(idxs.height), Some(400_000_u32.into())) + parser.parse(Some(idxs.height), None) .iter() .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); @@ -628,7 +631,10 @@ impl Indexer { Ok(()) })?; - export(stores, vecs, idxs.height)?; + if idxs.height % SNAPSHOT_BLOCK_RANGE != 0 { + export(stores, vecs, idxs.height)?; + } + sleep(Duration::from_millis(100)); Ok(()) diff --git a/crates/brk_indexer/src/storage/fjalls/mod.rs b/crates/brk_indexer/src/storage/fjalls/mod.rs index 058a84e0a..64cc683bc 100644 --- a/crates/brk_indexer/src/storage/fjalls/mod.rs +++ b/crates/brk_indexer/src/storage/fjalls/mod.rs @@ -48,103 +48,87 @@ impl Fjalls { Ok(()) })?; - vecs.height_to_first_p2pk65index - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2pk65index_to_p2pk65addressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2PK65)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2pk65index.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs + .p2pk65index_to_p2pk65addressbytes + .get(index)? + .map(Value::into_inner) + { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2PK65)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2pk33index - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2pk33index_to_p2pk33addressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2PK33)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2pk33index.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs + .p2pk33index_to_p2pk33addressbytes + .get(index)? + .map(Value::into_inner) + { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2PK33)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2pkhindex - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2pkhindex_to_p2pkhaddressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2PKH)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2pkhindex.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs.p2pkhindex_to_p2pkhaddressbytes.get(index)?.map(Value::into_inner) { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2PKH)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2shindex - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2shindex_to_p2shaddressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2SH)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2shindex.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs.p2shindex_to_p2shaddressbytes.get(index)?.map(Value::into_inner) { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2SH)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2trindex - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2trindex_to_p2traddressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2TR)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2trindex.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs.p2trindex_to_p2traddressbytes.get(index)?.map(Value::into_inner) { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2TR)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2wpkhindex - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2wpkhindex_to_p2wpkhaddressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2WPKH)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2wpkhindex.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs + .p2wpkhindex_to_p2wpkhaddressbytes + .get(index)? + .map(Value::into_inner) + { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2WPKH)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } - vecs.height_to_first_p2wshindex - .iter_from(starting_indexes.height, |(_, index)| { - if let Some(typedbytes) = vecs - .p2wshindex_to_p2wshaddressbytes - .get(index.into_inner())? - .map(Value::into_inner) - { - let bytes = Addressbytes::from(typedbytes); - let hash = AddressHash::from((&bytes, Addresstype::P2WSH)); - self.addresshash_to_addressindex.remove(hash); - } - Ok(()) - })?; + if let Some(index) = vecs.height_to_first_p2wshindex.get(starting_indexes.height)? { + let mut index = index.into_inner(); + while let Some(typedbytes) = vecs.p2wshindex_to_p2wshaddressbytes.get(index)?.map(Value::into_inner) { + let bytes = Addressbytes::from(typedbytes); + let hash = AddressHash::from((&bytes, Addresstype::P2WSH)); + self.addresshash_to_addressindex.remove(hash); + index.increment(); + } + } self.commit(starting_indexes.height.decremented())?; diff --git a/crates/brk_indexer/src/structs/indexes.rs b/crates/brk_indexer/src/structs/indexes.rs index 91e485bfe..a35fcfdab 100644 --- a/crates/brk_indexer/src/structs/indexes.rs +++ b/crates/brk_indexer/src/structs/indexes.rs @@ -98,8 +98,6 @@ impl TryFrom<(&mut StorableVecs, &Fjalls, &Client)> for Indexes { }) .unwrap_or(starting_height); - // let height = 885000_u32.into(); - Ok(Self { addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?, emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?, diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 96224397c..ad9cda4ec 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -191,7 +191,6 @@ impl Parser { } let height = Height::from(header.height); - // println!("{height}"); let len = blk_index_to_blk_recap.tree.len(); if blk_metadata.index == len as u16 || blk_metadata.index + 1 == len as u16 { @@ -235,9 +234,13 @@ impl Parser { None } }) { + if end.is_some_and(|end| end < current_height) { + return ControlFlow::Break(()); + } + send_height_block_hash.send((current_height, block, hash)).unwrap(); - if end == Some(current_height) { + if end.is_some_and(|end| end == current_height) { return ControlFlow::Break(()); }