bindex: removed addressindex to in/out puts as they're computable later

This commit is contained in:
nym21
2025-01-27 09:55:28 +01:00
parent 4923c2e204
commit 042be6e229
7 changed files with 83 additions and 126 deletions
+31 -71
View File
@@ -24,8 +24,7 @@ use structs::{
}; };
const UNSAFE_BLOCKS: u32 = 100; const UNSAFE_BLOCKS: u32 = 100;
const DAILY_BLOCK_TARGET: usize = 144; const SNAPSHOT_BLOCK_RANGE: usize = 1000;
const SNAPSHOT_BLOCK_RANGE: usize = DAILY_BLOCK_TARGET * 10;
#[derive(Debug)] #[derive(Debug)]
pub struct Indexer; pub struct Indexer;
@@ -53,29 +52,27 @@ impl Indexer {
let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?; let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?;
let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?; let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?;
let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?; let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_p2pk33index.get_or_default(height)?; let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_p2pk65index.get_or_default(height)?; let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_p2pkhindex.get_or_default(height)?; let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_p2shindex.get_or_default(height)?; let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_p2trindex.get_or_default(height)?; let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_p2wpkhindex.get_or_default(height)?; let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_p2wshindex.get_or_default(height)?; let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> { let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
println!("Exporting..."); println!("Exporting...");
if height >= Height::from(400_000_u32) {
pause();
// println!("Flushing vecs...");
}
exit.block(); exit.block();
// At 401760
// Memory: 1.87 GB
// Real Memory: 13.46 GB
// if height >= Height::from(400_000_u32) {
// pause();
// }
println!("Flushing vecs...");
thread::scope(|scope| -> color_eyre::Result<()> { thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height)); let vecs_handle = scope.spawn(|| vecs.flush(height));
@@ -85,27 +82,13 @@ impl Indexer {
Ok(()) Ok(())
})?; })?;
// At 401760
// Memory: 1.83 GB
// Real Memory: 9.45 GB
// if height >= Height::from(400_000_u32) {
// pause();
// }
// At: 401760
// Memory: 1.34 GB
// Real Memory: 1.52 GB
println!("All done...");
// if height >= Height::from(400_000_u32) {
// pause();
// }
exit.unblock(); exit.unblock();
Ok(()) Ok(())
}; };
let mut stores_opt = Some(stores); let mut stores_opt = Some(stores);
biter::new(bitcoin_dir, Some(height.into()), None, rpc) biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc)
.iter() .iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}..."); println!("Processing block {_height}...");
@@ -158,13 +141,13 @@ impl Indexer {
.push_if_needed(height, pushonlyindex_global)?; .push_if_needed(height, pushonlyindex_global)?;
vecs.height_to_first_unknownindex vecs.height_to_first_unknownindex
.push_if_needed(height, unknownindex_global)?; .push_if_needed(height, unknownindex_global)?;
vecs.height_to_p2pk33index.push_if_needed(height, p2pk33index_global)?; vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_p2pk65index.push_if_needed(height, p2pk65index_global)?; vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_p2pkhindex.push_if_needed(height, p2pkhindex_global)?; vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_p2shindex.push_if_needed(height, p2shindex_global)?; vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_p2trindex.push_if_needed(height, p2trindex_global)?; vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?; vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_p2wshindex.push_if_needed(height, p2wshindex_global)?; vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?;
let inputs = block let inputs = block
.txdata .txdata
@@ -276,21 +259,10 @@ impl Indexer {
})? })?
+ vout; + vout;
let addressindex = *vecs
.txoutindex_to_addressindex
.get(txoutindex)?
.context("Expect addressindex to not be none")
.inspect_err(|_| {
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
// .expect("Expect height for txindex");
dbg!(outpoint.txid, prev_txindex, vout, txoutindex);
})?;
Ok((txinindex, InputSource::PreviousBlock(( Ok((txinindex, InputSource::PreviousBlock((
vin, vin,
txindex, txindex,
txoutindex, txoutindex,
addressindex,
)))) ))))
}) })
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> { .try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
@@ -540,10 +512,6 @@ impl Indexer {
vecs.txoutindex_to_addressindex vecs.txoutindex_to_addressindex
.push_if_needed(txoutindex, addressindex)?; .push_if_needed(txoutindex, addressindex)?;
// stores
// .addressindex_to_txoutindex_in
// .insert_if_needed(addressindex, txoutindex, height);
Ok(()) Ok(())
}, },
)?; )?;
@@ -555,13 +523,13 @@ impl Indexer {
.map( .map(
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
|(txinindex, input_source)| -> color_eyre::Result<( |(txinindex, input_source)| -> color_eyre::Result<(
Txinindex, Vin, Txindex, Option<(Addressindex, Txoutindex)> Txinindex, Vin, Txindex, Txoutindex
)> { )> {
match input_source { match input_source {
InputSource::PreviousBlock((vin, txindex, txoutindex, addressindex)) => Ok((txinindex, vin, txindex, Some((addressindex, txoutindex)))), InputSource::PreviousBlock((vin, txindex, txoutindex)) => Ok((txinindex, vin, txindex, txoutindex)),
InputSource::SameBlock((tx, txindex, txin, vin)) => { InputSource::SameBlock((tx, txindex, txin, vin)) => {
if tx.is_coinbase() { if tx.is_coinbase() {
return Ok((txinindex, vin, txindex, None)); return Ok((txinindex, vin, txindex, Txoutindex::COINBASE));
} }
let outpoint = txin.previous_output; let outpoint = txin.previous_output;
@@ -581,28 +549,20 @@ impl Indexer {
dbg!(&new_txindexvout_to_txoutindex, txin, prev_txindex, vout, txid); dbg!(&new_txindexvout_to_txoutindex, txin, prev_txindex, vout, txid);
})?; })?;
Ok((txinindex, vin, txindex, Some((Addressindex::from(0_u32), prev_txoutindex)))) Ok((txinindex, vin, txindex, prev_txoutindex))
} }
} }
}, },
) )
.try_for_each(|res| -> color_eyre::Result<()> { .try_for_each(|res| -> color_eyre::Result<()> {
let (txinindex, vin, txindex, addressindex_and_txoutindex_opt) = res?; let (txinindex, vin, txindex, txoutindex) = res?;
if vin.is_zero() { if vin.is_zero() {
vecs.txindex_to_first_txinindex.push_if_needed(txindex, txinindex)?; vecs.txindex_to_first_txinindex.push_if_needed(txindex, txinindex)?;
} }
let txoutindex = addressindex_and_txoutindex_opt.map_or(Txoutindex::MAX, |(_, txoutindex)| txoutindex);
vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?; vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?;
// if let Some(addressindex) = addressindex_and_txoutindex_opt.map(|(addressindex, _)| addressindex) {
// stores
// .addressindex_to_txoutindex_out
// .insert_if_needed(addressindex, txoutindex, height);
// }
Ok(()) Ok(())
})?; })?;
@@ -703,7 +663,7 @@ impl Indexer {
#[derive(Debug)] #[derive(Debug)]
enum InputSource<'a> { enum InputSource<'a> {
PreviousBlock((Vin, Txindex, Txoutindex, Addressindex)), PreviousBlock((Vin, Txindex, Txoutindex)),
SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)), SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)),
} }
-15
View File
@@ -9,13 +9,10 @@ mod multi;
mod unique; mod unique;
use meta::*; use meta::*;
use multi::*;
use unique::*; use unique::*;
pub struct Stores { pub struct Stores {
pub addressbytes_prefix_to_addressindex: StoreUnique<AddressbytesPrefix, Addressindex>, pub addressbytes_prefix_to_addressindex: StoreUnique<AddressbytesPrefix, Addressindex>,
// pub addressindex_to_txoutindex_in: StoreMulti<Addressindex, Txoutindex>, // Received
// pub addressindex_to_txoutindex_out: StoreMulti<Addressindex, Txoutindex>, // Spent
pub blockhash_prefix_to_height: StoreUnique<BlockHashPrefix, Height>, pub blockhash_prefix_to_height: StoreUnique<BlockHashPrefix, Height>,
pub txid_prefix_to_txindex: StoreUnique<TxidPrefix, Txindex>, pub txid_prefix_to_txindex: StoreUnique<TxidPrefix, Txindex>,
} }
@@ -27,14 +24,6 @@ impl Stores {
&path.join("addressbytes_prefix_to_addressindex"), &path.join("addressbytes_prefix_to_addressindex"),
Version::from(1), Version::from(1),
)?, )?,
// addressindex_to_txoutindex_in: StoreMulti::open(
// &path.join("addressindex_to_txoutindex_in"),
// Version::from(1),
// )?,
// addressindex_to_txoutindex_out: StoreMulti::open(
// &path.join("addressindex_to_txoutindex_out"),
// Version::from(1),
// )?,
blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?, blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?, txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
}) })
@@ -162,8 +151,6 @@ impl Stores {
pub fn min_height(&self) -> Option<Height> { pub fn min_height(&self) -> Option<Height> {
[ [
self.addressbytes_prefix_to_addressindex.height(), self.addressbytes_prefix_to_addressindex.height(),
// self.addressindex_to_txoutindex_in.height(),
// self.addressindex_to_txoutindex_out.height(),
self.blockhash_prefix_to_height.height(), self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(), self.txid_prefix_to_txindex.height(),
] ]
@@ -177,8 +164,6 @@ impl Stores {
thread::scope(|scope| { thread::scope(|scope| {
vec![ vec![
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)), scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
// scope.spawn(|| self.addressindex_to_txoutindex_in.export(height)),
// scope.spawn(|| self.addressindex_to_txoutindex_out.export(height)),
scope.spawn(|| self.blockhash_prefix_to_height.export(height)), scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
scope.spawn(|| self.txid_prefix_to_txindex.export(height)), scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
] ]
+3 -3
View File
@@ -11,7 +11,7 @@ where
V: DatabaseValue, V: DatabaseValue,
{ {
meta: StoreMeta, meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 64], pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 256],
} }
impl<K, V> StoreMulti<K, V> impl<K, V> StoreMulti<K, V>
@@ -33,7 +33,7 @@ where
// } // }
fn get_or_init_store(&self, key: &K) -> &DatabaseMulti<K, V> { fn get_or_init_store(&self, key: &K) -> &DatabaseMulti<K, V> {
self.get_or_init_store_(key.as_ne_six_bits() as usize) self.get_or_init_store_(key.as_ne_byte() as usize)
} }
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti<K, V> { fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti<K, V> {
@@ -45,7 +45,7 @@ where
self.get_or_init_store(key); self.get_or_init_store(key);
self.parts self.parts
.get_mut(key.as_ne_six_bits() as usize) .get_mut(key.as_ne_byte() as usize)
.unwrap() .unwrap()
.get_mut() .get_mut()
.unwrap() .unwrap()
+3 -3
View File
@@ -11,7 +11,7 @@ where
V: DatabaseValue, V: DatabaseValue,
{ {
meta: StoreMeta, meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 64], pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 256],
} }
impl<K, V> StoreUnique<K, V> impl<K, V> StoreUnique<K, V>
@@ -33,7 +33,7 @@ where
// } // }
fn get_or_init_store(&self, key: &K) -> &DatabaseUnique<K, V> { fn get_or_init_store(&self, key: &K) -> &DatabaseUnique<K, V> {
self.get_or_init_store_(key.as_ne_six_bits() as usize) self.get_or_init_store_(key.as_ne_byte() as usize)
} }
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique<K, V> { fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique<K, V> {
@@ -46,7 +46,7 @@ where
self.get_or_init_store(key); self.get_or_init_store(key);
self.parts self.parts
.get_mut(key.as_ne_six_bits() as usize) .get_mut(key.as_ne_byte() as usize)
.unwrap() .unwrap()
.get_mut() .get_mut()
.unwrap() .unwrap()
+43 -28
View File
@@ -28,13 +28,13 @@ pub struct Vecs {
pub height_to_first_txinindex: StorableVec<Height, Txinindex>, pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>, pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pk33index: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pk65index: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pkhindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2shindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2trindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2wpkhindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2wshindex: StorableVec<Height, Addresstypeindex>, pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex>,
pub height_to_size: StorableVec<Height, usize>, pub height_to_size: StorableVec<Height, usize>,
pub height_to_timestamp: StorableVec<Height, Timestamp>, pub height_to_timestamp: StorableVec<Height, Timestamp>,
pub height_to_weight: StorableVec<Height, Weight>, pub height_to_weight: StorableVec<Height, Weight>,
@@ -120,13 +120,28 @@ impl Vecs {
&path.join("height_to_first_unkownindex"), &path.join("height_to_first_unkownindex"),
Version::from(1), Version::from(1),
)?, )?,
height_to_p2pk33index: StorableVec::import(&path.join("height_to_p2pk33index"), Version::from(1))?, height_to_first_p2pk33index: StorableVec::import(
height_to_p2pk65index: StorableVec::import(&path.join("height_to_p2pk65index"), Version::from(1))?, &path.join("height_to_first_p2pk33index"),
height_to_p2pkhindex: StorableVec::import(&path.join("height_to_p2pkhindex"), Version::from(1))?, Version::from(1),
height_to_p2shindex: StorableVec::import(&path.join("height_to_p2shindex"), Version::from(1))?, )?,
height_to_p2trindex: StorableVec::import(&path.join("height_to_p2trindex"), Version::from(1))?, height_to_first_p2pk65index: StorableVec::import(
height_to_p2wpkhindex: StorableVec::import(&path.join("height_to_p2wpkhindex"), Version::from(1))?, &path.join("height_to_first_p2pk65index"),
height_to_p2wshindex: StorableVec::import(&path.join("height_to_p2wshindex"), Version::from(1))?, Version::from(1),
)?,
height_to_first_p2pkhindex: StorableVec::import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
)?,
height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
)?,
height_to_first_p2wshindex: StorableVec::import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
)?,
height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?, height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?, height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?, height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
@@ -383,13 +398,13 @@ impl Vecs {
&self.height_to_first_txinindex, &self.height_to_first_txinindex,
&self.height_to_first_txoutindex, &self.height_to_first_txoutindex,
&self.height_to_first_unknownindex, &self.height_to_first_unknownindex,
&self.height_to_p2pk33index, &self.height_to_first_p2pk33index,
&self.height_to_p2pk65index, &self.height_to_first_p2pk65index,
&self.height_to_p2pkhindex, &self.height_to_first_p2pkhindex,
&self.height_to_p2shindex, &self.height_to_first_p2shindex,
&self.height_to_p2trindex, &self.height_to_first_p2trindex,
&self.height_to_p2wpkhindex, &self.height_to_first_p2wpkhindex,
&self.height_to_p2wshindex, &self.height_to_first_p2wshindex,
&self.height_to_size, &self.height_to_size,
&self.height_to_timestamp, &self.height_to_timestamp,
&self.height_to_weight, &self.height_to_weight,
@@ -427,13 +442,13 @@ impl Vecs {
&mut self.height_to_first_txinindex, &mut self.height_to_first_txinindex,
&mut self.height_to_first_txoutindex, &mut self.height_to_first_txoutindex,
&mut self.height_to_first_unknownindex, &mut self.height_to_first_unknownindex,
&mut self.height_to_p2pk33index, &mut self.height_to_first_p2pk33index,
&mut self.height_to_p2pk65index, &mut self.height_to_first_p2pk65index,
&mut self.height_to_p2pkhindex, &mut self.height_to_first_p2pkhindex,
&mut self.height_to_p2shindex, &mut self.height_to_first_p2shindex,
&mut self.height_to_p2trindex, &mut self.height_to_first_p2trindex,
&mut self.height_to_p2wpkhindex, &mut self.height_to_first_p2wpkhindex,
&mut self.height_to_p2wshindex, &mut self.height_to_first_p2wshindex,
&mut self.height_to_size, &mut self.height_to_size,
&mut self.height_to_timestamp, &mut self.height_to_timestamp,
&mut self.height_to_weight, &mut self.height_to_weight,
+3 -6
View File
@@ -1,7 +1,4 @@
use std::{ use std::ops::{Add, AddAssign};
ops::{Add, AddAssign},
u64,
};
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable}; use snkrj::{direct_repr, Storable, UnsizedStorable};
@@ -13,7 +10,7 @@ pub struct Txoutindex(u64);
direct_repr!(Txoutindex); direct_repr!(Txoutindex);
impl Txoutindex { impl Txoutindex {
pub const MAX: Self = Self(u64::MAX); pub const COINBASE: Self = Self(u64::MAX);
pub fn incremented(self) -> Self { pub fn incremented(self) -> Self {
Self(*self + 1) Self(*self + 1)
@@ -24,7 +21,7 @@ impl Txoutindex {
} }
pub fn is_coinbase(self) -> bool { pub fn is_coinbase(self) -> bool {
self == Self::MAX self == Self::COINBASE
} }
} }