diff --git a/bindex/src/lib.rs b/bindex/src/lib.rs index 8509fbdf5..17584650a 100644 --- a/bindex/src/lib.rs +++ b/bindex/src/lib.rs @@ -24,8 +24,7 @@ use structs::{ }; const UNSAFE_BLOCKS: u32 = 100; -const DAILY_BLOCK_TARGET: usize = 144; -const SNAPSHOT_BLOCK_RANGE: usize = DAILY_BLOCK_TARGET * 10; +const SNAPSHOT_BLOCK_RANGE: usize = 1000; #[derive(Debug)] pub struct Indexer; @@ -53,29 +52,27 @@ impl Indexer { let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?; let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?; let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; - let mut multisigindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; - let mut opreturnindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; - let mut pushonlyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; - let mut unknownindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; - let mut p2pk33index_global = vecs.height_to_p2pk33index.get_or_default(height)?; - let mut p2pk65index_global = vecs.height_to_p2pk65index.get_or_default(height)?; - let mut p2pkhindex_global = vecs.height_to_p2pkhindex.get_or_default(height)?; - let mut p2shindex_global = vecs.height_to_p2shindex.get_or_default(height)?; - let mut p2trindex_global = vecs.height_to_p2trindex.get_or_default(height)?; - let mut p2wpkhindex_global = vecs.height_to_p2wpkhindex.get_or_default(height)?; - let mut p2wshindex_global = vecs.height_to_p2wshindex.get_or_default(height)?; + let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?; + let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?; + let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?; + let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?; + let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?; + let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?; + let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?; + let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?; + let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?; + let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?; + let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?; let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> { println!("Exporting..."); + if height >= Height::from(400_000_u32) { + pause(); + // println!("Flushing vecs..."); + } + exit.block(); - // At 401760 - // Memory: 1.87 GB - // Real Memory: 13.46 GB - // if height >= Height::from(400_000_u32) { - // pause(); - // } - println!("Flushing vecs..."); thread::scope(|scope| -> color_eyre::Result<()> { let vecs_handle = scope.spawn(|| vecs.flush(height)); @@ -85,27 +82,13 @@ impl Indexer { Ok(()) })?; - // At 401760 - // Memory: 1.83 GB - // Real Memory: 9.45 GB - // if height >= Height::from(400_000_u32) { - // pause(); - // } - - // At: 401760 - // Memory: 1.34 GB - // Real Memory: 1.52 GB - println!("All done..."); - // if height >= Height::from(400_000_u32) { - // pause(); - // } exit.unblock(); Ok(()) }; let mut stores_opt = Some(stores); - biter::new(bitcoin_dir, Some(height.into()), None, rpc) + biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { println!("Processing block {_height}..."); @@ -158,13 +141,13 @@ impl Indexer { .push_if_needed(height, pushonlyindex_global)?; vecs.height_to_first_unknownindex .push_if_needed(height, unknownindex_global)?; - vecs.height_to_p2pk33index.push_if_needed(height, p2pk33index_global)?; - vecs.height_to_p2pk65index.push_if_needed(height, p2pk65index_global)?; - vecs.height_to_p2pkhindex.push_if_needed(height, p2pkhindex_global)?; - vecs.height_to_p2shindex.push_if_needed(height, p2shindex_global)?; - vecs.height_to_p2trindex.push_if_needed(height, p2trindex_global)?; - vecs.height_to_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?; - vecs.height_to_p2wshindex.push_if_needed(height, p2wshindex_global)?; + vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?; + vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?; + vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?; + vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?; + vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?; + vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?; + vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?; let inputs = block .txdata @@ -276,21 +259,10 @@ impl Indexer { })? + vout; - let addressindex = *vecs - .txoutindex_to_addressindex - .get(txoutindex)? - .context("Expect addressindex to not be none") - .inspect_err(|_| { - // let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail") - // .expect("Expect height for txindex"); - dbg!(outpoint.txid, prev_txindex, vout, txoutindex); - })?; - Ok((txinindex, InputSource::PreviousBlock(( vin, txindex, txoutindex, - addressindex, )))) }) .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { @@ -540,10 +512,6 @@ impl Indexer { vecs.txoutindex_to_addressindex .push_if_needed(txoutindex, addressindex)?; - // stores - // .addressindex_to_txoutindex_in - // .insert_if_needed(addressindex, txoutindex, height); - Ok(()) }, )?; @@ -555,13 +523,13 @@ impl Indexer { .map( #[allow(clippy::type_complexity)] |(txinindex, input_source)| -> color_eyre::Result<( - Txinindex, Vin, Txindex, Option<(Addressindex, Txoutindex)> + Txinindex, Vin, Txindex, Txoutindex )> { match input_source { - InputSource::PreviousBlock((vin, txindex, txoutindex, addressindex)) => Ok((txinindex, vin, txindex, Some((addressindex, txoutindex)))), + InputSource::PreviousBlock((vin, txindex, txoutindex)) => Ok((txinindex, vin, txindex, txoutindex)), InputSource::SameBlock((tx, txindex, txin, vin)) => { if tx.is_coinbase() { - return Ok((txinindex, vin, txindex, None)); + return Ok((txinindex, vin, txindex, Txoutindex::COINBASE)); } let outpoint = txin.previous_output; @@ -581,28 +549,20 @@ impl Indexer { dbg!(&new_txindexvout_to_txoutindex, txin, prev_txindex, vout, txid); })?; - Ok((txinindex, vin, txindex, Some((Addressindex::from(0_u32), prev_txoutindex)))) + Ok((txinindex, vin, txindex, prev_txoutindex)) } } }, ) .try_for_each(|res| -> color_eyre::Result<()> { - let (txinindex, vin, txindex, addressindex_and_txoutindex_opt) = res?; + let (txinindex, vin, txindex, txoutindex) = res?; if vin.is_zero() { vecs.txindex_to_first_txinindex.push_if_needed(txindex, txinindex)?; } - let txoutindex = addressindex_and_txoutindex_opt.map_or(Txoutindex::MAX, |(_, txoutindex)| txoutindex); - vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?; - // if let Some(addressindex) = addressindex_and_txoutindex_opt.map(|(addressindex, _)| addressindex) { - // stores - // .addressindex_to_txoutindex_out - // .insert_if_needed(addressindex, txoutindex, height); - // } - Ok(()) })?; @@ -703,7 +663,7 @@ impl Indexer { #[derive(Debug)] enum InputSource<'a> { - PreviousBlock((Vin, Txindex, Txoutindex, Addressindex)), + PreviousBlock((Vin, Txindex, Txoutindex)), SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)), } diff --git a/bindex/src/storage/partitions/mod.rs b/bindex/src/storage/partitions/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/bindex/src/storage/stores/mod.rs b/bindex/src/storage/stores/mod.rs index 778943ad8..8a00fb17a 100644 --- a/bindex/src/storage/stores/mod.rs +++ b/bindex/src/storage/stores/mod.rs @@ -9,13 +9,10 @@ mod multi; mod unique; use meta::*; -use multi::*; use unique::*; pub struct Stores { pub addressbytes_prefix_to_addressindex: StoreUnique, - // pub addressindex_to_txoutindex_in: StoreMulti, // Received - // pub addressindex_to_txoutindex_out: StoreMulti, // Spent pub blockhash_prefix_to_height: StoreUnique, pub txid_prefix_to_txindex: StoreUnique, } @@ -27,14 +24,6 @@ impl Stores { &path.join("addressbytes_prefix_to_addressindex"), Version::from(1), )?, - // addressindex_to_txoutindex_in: StoreMulti::open( - // &path.join("addressindex_to_txoutindex_in"), - // Version::from(1), - // )?, - // addressindex_to_txoutindex_out: StoreMulti::open( - // &path.join("addressindex_to_txoutindex_out"), - // Version::from(1), - // )?, blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?, txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?, }) @@ -162,8 +151,6 @@ impl Stores { pub fn min_height(&self) -> Option { [ self.addressbytes_prefix_to_addressindex.height(), - // self.addressindex_to_txoutindex_in.height(), - // self.addressindex_to_txoutindex_out.height(), self.blockhash_prefix_to_height.height(), self.txid_prefix_to_txindex.height(), ] @@ -177,8 +164,6 @@ impl Stores { thread::scope(|scope| { vec![ scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)), - // scope.spawn(|| self.addressindex_to_txoutindex_in.export(height)), - // scope.spawn(|| self.addressindex_to_txoutindex_out.export(height)), scope.spawn(|| self.blockhash_prefix_to_height.export(height)), scope.spawn(|| self.txid_prefix_to_txindex.export(height)), ] diff --git a/bindex/src/storage/stores/multi.rs b/bindex/src/storage/stores/multi.rs index 68250e8d7..4ac8f15b6 100644 --- a/bindex/src/storage/stores/multi.rs +++ b/bindex/src/storage/stores/multi.rs @@ -11,7 +11,7 @@ where V: DatabaseValue, { meta: StoreMeta, - pub parts: [OnceLock>>; 64], + pub parts: [OnceLock>>; 256], } impl StoreMulti @@ -33,7 +33,7 @@ where // } fn get_or_init_store(&self, key: &K) -> &DatabaseMulti { - self.get_or_init_store_(key.as_ne_six_bits() as usize) + self.get_or_init_store_(key.as_ne_byte() as usize) } fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti { @@ -45,7 +45,7 @@ where self.get_or_init_store(key); self.parts - .get_mut(key.as_ne_six_bits() as usize) + .get_mut(key.as_ne_byte() as usize) .unwrap() .get_mut() .unwrap() diff --git a/bindex/src/storage/stores/unique.rs b/bindex/src/storage/stores/unique.rs index ff4864181..6c958d6d1 100644 --- a/bindex/src/storage/stores/unique.rs +++ b/bindex/src/storage/stores/unique.rs @@ -11,7 +11,7 @@ where V: DatabaseValue, { meta: StoreMeta, - pub parts: [OnceLock>>; 64], + pub parts: [OnceLock>>; 256], } impl StoreUnique @@ -33,7 +33,7 @@ where // } fn get_or_init_store(&self, key: &K) -> &DatabaseUnique { - self.get_or_init_store_(key.as_ne_six_bits() as usize) + self.get_or_init_store_(key.as_ne_byte() as usize) } fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique { @@ -46,7 +46,7 @@ where self.get_or_init_store(key); self.parts - .get_mut(key.as_ne_six_bits() as usize) + .get_mut(key.as_ne_byte() as usize) .unwrap() .get_mut() .unwrap() diff --git a/bindex/src/storage/vecs/mod.rs b/bindex/src/storage/vecs/mod.rs index 471f37c1c..83487ebab 100644 --- a/bindex/src/storage/vecs/mod.rs +++ b/bindex/src/storage/vecs/mod.rs @@ -28,13 +28,13 @@ pub struct Vecs { pub height_to_first_txinindex: StorableVec, pub height_to_first_txoutindex: StorableVec, pub height_to_first_unknownindex: StorableVec, - pub height_to_p2pk33index: StorableVec, - pub height_to_p2pk65index: StorableVec, - pub height_to_p2pkhindex: StorableVec, - pub height_to_p2shindex: StorableVec, - pub height_to_p2trindex: StorableVec, - pub height_to_p2wpkhindex: StorableVec, - pub height_to_p2wshindex: StorableVec, + pub height_to_first_p2pk33index: StorableVec, + pub height_to_first_p2pk65index: StorableVec, + pub height_to_first_p2pkhindex: StorableVec, + pub height_to_first_p2shindex: StorableVec, + pub height_to_first_p2trindex: StorableVec, + pub height_to_first_p2wpkhindex: StorableVec, + pub height_to_first_p2wshindex: StorableVec, pub height_to_size: StorableVec, pub height_to_timestamp: StorableVec, pub height_to_weight: StorableVec, @@ -120,13 +120,28 @@ impl Vecs { &path.join("height_to_first_unkownindex"), Version::from(1), )?, - height_to_p2pk33index: StorableVec::import(&path.join("height_to_p2pk33index"), Version::from(1))?, - height_to_p2pk65index: StorableVec::import(&path.join("height_to_p2pk65index"), Version::from(1))?, - height_to_p2pkhindex: StorableVec::import(&path.join("height_to_p2pkhindex"), Version::from(1))?, - height_to_p2shindex: StorableVec::import(&path.join("height_to_p2shindex"), Version::from(1))?, - height_to_p2trindex: StorableVec::import(&path.join("height_to_p2trindex"), Version::from(1))?, - height_to_p2wpkhindex: StorableVec::import(&path.join("height_to_p2wpkhindex"), Version::from(1))?, - height_to_p2wshindex: StorableVec::import(&path.join("height_to_p2wshindex"), Version::from(1))?, + height_to_first_p2pk33index: StorableVec::import( + &path.join("height_to_first_p2pk33index"), + Version::from(1), + )?, + height_to_first_p2pk65index: StorableVec::import( + &path.join("height_to_first_p2pk65index"), + Version::from(1), + )?, + height_to_first_p2pkhindex: StorableVec::import( + &path.join("height_to_first_p2pkhindex"), + Version::from(1), + )?, + height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?, + height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?, + height_to_first_p2wpkhindex: StorableVec::import( + &path.join("height_to_first_p2wpkhindex"), + Version::from(1), + )?, + height_to_first_p2wshindex: StorableVec::import( + &path.join("height_to_first_p2wshindex"), + Version::from(1), + )?, height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?, height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?, height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?, @@ -383,13 +398,13 @@ impl Vecs { &self.height_to_first_txinindex, &self.height_to_first_txoutindex, &self.height_to_first_unknownindex, - &self.height_to_p2pk33index, - &self.height_to_p2pk65index, - &self.height_to_p2pkhindex, - &self.height_to_p2shindex, - &self.height_to_p2trindex, - &self.height_to_p2wpkhindex, - &self.height_to_p2wshindex, + &self.height_to_first_p2pk33index, + &self.height_to_first_p2pk65index, + &self.height_to_first_p2pkhindex, + &self.height_to_first_p2shindex, + &self.height_to_first_p2trindex, + &self.height_to_first_p2wpkhindex, + &self.height_to_first_p2wshindex, &self.height_to_size, &self.height_to_timestamp, &self.height_to_weight, @@ -427,13 +442,13 @@ impl Vecs { &mut self.height_to_first_txinindex, &mut self.height_to_first_txoutindex, &mut self.height_to_first_unknownindex, - &mut self.height_to_p2pk33index, - &mut self.height_to_p2pk65index, - &mut self.height_to_p2pkhindex, - &mut self.height_to_p2shindex, - &mut self.height_to_p2trindex, - &mut self.height_to_p2wpkhindex, - &mut self.height_to_p2wshindex, + &mut self.height_to_first_p2pk33index, + &mut self.height_to_first_p2pk65index, + &mut self.height_to_first_p2pkhindex, + &mut self.height_to_first_p2shindex, + &mut self.height_to_first_p2trindex, + &mut self.height_to_first_p2wpkhindex, + &mut self.height_to_first_p2wshindex, &mut self.height_to_size, &mut self.height_to_timestamp, &mut self.height_to_weight, diff --git a/bindex/src/structs/txoutindex.rs b/bindex/src/structs/txoutindex.rs index d4083fb05..91740bcb1 100644 --- a/bindex/src/structs/txoutindex.rs +++ b/bindex/src/structs/txoutindex.rs @@ -1,7 +1,4 @@ -use std::{ - ops::{Add, AddAssign}, - u64, -}; +use std::ops::{Add, AddAssign}; use derive_deref::{Deref, DerefMut}; use snkrj::{direct_repr, Storable, UnsizedStorable}; @@ -13,7 +10,7 @@ pub struct Txoutindex(u64); direct_repr!(Txoutindex); impl Txoutindex { - pub const MAX: Self = Self(u64::MAX); + pub const COINBASE: Self = Self(u64::MAX); pub fn incremented(self) -> Self { Self(*self + 1) @@ -24,7 +21,7 @@ impl Txoutindex { } pub fn is_coinbase(self) -> bool { - self == Self::MAX + self == Self::COINBASE } }