Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-09 14:41:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

bindex: removed addressindex to in/out puts as they're computable later

Browse Source
This commit is contained in:
nym21
2025-01-27 09:55:28 +01:00
parent 4923c2e204
commit 042be6e229
7 changed files with 83 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bindex/src/lib.rs
+31 -71
View File
@@ -24,8 +24,7 @@ use structs::{
};
const UNSAFE_BLOCKS: u32 = 100;
const DAILY_BLOCK_TARGET: usize = 144;
const SNAPSHOT_BLOCK_RANGE: usize = DAILY_BLOCK_TARGET * 10;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
#[derive(Debug)]
pub struct Indexer;
@@ -53,29 +52,27 @@ impl Indexer {
let mut txoutindex_global = vecs.height_to_first_txoutindex.get_or_default(height)?;
let mut addressindex_global = vecs.height_to_first_addressindex.get_or_default(height)?;
let mut emptyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_emptyindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_p2wshindex.get_or_default(height)?;
let mut multisigindex_global = vecs.height_to_first_multisigindex.get_or_default(height)?;
let mut opreturnindex_global = vecs.height_to_first_opreturnindex.get_or_default(height)?;
let mut pushonlyindex_global = vecs.height_to_first_pushonlyindex.get_or_default(height)?;
let mut unknownindex_global = vecs.height_to_first_unknownindex.get_or_default(height)?;
let mut p2pk33index_global = vecs.height_to_first_p2pk33index.get_or_default(height)?;
let mut p2pk65index_global = vecs.height_to_first_p2pk65index.get_or_default(height)?;
let mut p2pkhindex_global = vecs.height_to_first_p2pkhindex.get_or_default(height)?;
let mut p2shindex_global = vecs.height_to_first_p2shindex.get_or_default(height)?;
let mut p2trindex_global = vecs.height_to_first_p2trindex.get_or_default(height)?;
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
println!("Exporting...");
if height >= Height::from(400_000_u32) {
pause();
// println!("Flushing vecs...");
}
exit.block();
// At 401760
// Memory: 1.87 GB
// Real Memory: 13.46 GB
// if height >= Height::from(400_000_u32) {
// pause();
// }
println!("Flushing vecs...");
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
@@ -85,27 +82,13 @@ impl Indexer {
Ok(())
})?;
// At 401760
// Memory: 1.83 GB
// Real Memory: 9.45 GB
// if height >= Height::from(400_000_u32) {
// pause();
// }
// At: 401760
// Memory: 1.34 GB
// Real Memory: 1.52 GB
println!("All done...");
// if height >= Height::from(400_000_u32) {
// pause();
// }
exit.unblock();
Ok(())
};
let mut stores_opt = Some(stores);
biter::new(bitcoin_dir, Some(height.into()), None, rpc)
biter::new(bitcoin_dir, Some(height.into()), Some(500_000), rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
@@ -158,13 +141,13 @@ impl Indexer {
.push_if_needed(height, pushonlyindex_global)?;
vecs.height_to_first_unknownindex
.push_if_needed(height, unknownindex_global)?;
vecs.height_to_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_p2wshindex.push_if_needed(height, p2wshindex_global)?;
vecs.height_to_first_p2pk33index.push_if_needed(height, p2pk33index_global)?;
vecs.height_to_first_p2pk65index.push_if_needed(height, p2pk65index_global)?;
vecs.height_to_first_p2pkhindex.push_if_needed(height, p2pkhindex_global)?;
vecs.height_to_first_p2shindex.push_if_needed(height, p2shindex_global)?;
vecs.height_to_first_p2trindex.push_if_needed(height, p2trindex_global)?;
vecs.height_to_first_p2wpkhindex.push_if_needed(height, p2wpkhindex_global)?;
vecs.height_to_first_p2wshindex.push_if_needed(height, p2wshindex_global)?;
let inputs = block
.txdata
@@ -276,21 +259,10 @@ impl Indexer {
})?
+ vout;
let addressindex = *vecs
.txoutindex_to_addressindex
.get(txoutindex)?
.context("Expect addressindex to not be none")
.inspect_err(|_| {
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
// .expect("Expect height for txindex");
dbg!(outpoint.txid, prev_txindex, vout, txoutindex);
})?;
Ok((txinindex, InputSource::PreviousBlock((
vin,
txindex,
txoutindex,
addressindex,
))))
})
.try_fold(BTreeMap::new, |mut map, tuple| -> color_eyre::Result<_> {
@@ -540,10 +512,6 @@ impl Indexer {
vecs.txoutindex_to_addressindex
.push_if_needed(txoutindex, addressindex)?;
// stores
// .addressindex_to_txoutindex_in
// .insert_if_needed(addressindex, txoutindex, height);
Ok(())
},
)?;
@@ -555,13 +523,13 @@ impl Indexer {
.map(
#[allow(clippy::type_complexity)]
|(txinindex, input_source)| -> color_eyre::Result<(
Txinindex, Vin, Txindex, Option<(Addressindex, Txoutindex)>
Txinindex, Vin, Txindex, Txoutindex
)> {
match input_source {
InputSource::PreviousBlock((vin, txindex, txoutindex, addressindex)) => Ok((txinindex, vin, txindex, Some((addressindex, txoutindex)))),
InputSource::PreviousBlock((vin, txindex, txoutindex)) => Ok((txinindex, vin, txindex, txoutindex)),
InputSource::SameBlock((tx, txindex, txin, vin)) => {
if tx.is_coinbase() {
return Ok((txinindex, vin, txindex, None));
return Ok((txinindex, vin, txindex, Txoutindex::COINBASE));
}
let outpoint = txin.previous_output;
@@ -581,28 +549,20 @@ impl Indexer {
dbg!(&new_txindexvout_to_txoutindex, txin, prev_txindex, vout, txid);
})?;
Ok((txinindex, vin, txindex, Some((Addressindex::from(0_u32), prev_txoutindex))))
Ok((txinindex, vin, txindex, prev_txoutindex))
}
}
},
)
.try_for_each(|res| -> color_eyre::Result<()> {
let (txinindex, vin, txindex, addressindex_and_txoutindex_opt) = res?;
let (txinindex, vin, txindex, txoutindex) = res?;
if vin.is_zero() {
vecs.txindex_to_first_txinindex.push_if_needed(txindex, txinindex)?;
}
let txoutindex = addressindex_and_txoutindex_opt.map_or(Txoutindex::MAX, |(_, txoutindex)| txoutindex);
vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?;
// if let Some(addressindex) = addressindex_and_txoutindex_opt.map(|(addressindex, _)| addressindex) {
// stores
// .addressindex_to_txoutindex_out
// .insert_if_needed(addressindex, txoutindex, height);
// }
Ok(())
})?;
@@ -703,7 +663,7 @@ impl Indexer {
#[derive(Debug)]
enum InputSource<'a> {
PreviousBlock((Vin, Txindex, Txoutindex, Addressindex)),
PreviousBlock((Vin, Txindex, Txoutindex)),
SameBlock((&'a Transaction, Txindex, &'a TxIn, Vin)),
}
bindex/src/storage/partitions/mod.rs
View File
bindex/src/storage/stores/mod.rs
-15
View File
@@ -9,13 +9,10 @@ mod multi;
mod unique;
use meta::*;
use multi::*;
use unique::*;
pub struct Stores {
pub addressbytes_prefix_to_addressindex: StoreUnique<AddressbytesPrefix, Addressindex>,
// pub addressindex_to_txoutindex_in: StoreMulti<Addressindex, Txoutindex>, // Received
// pub addressindex_to_txoutindex_out: StoreMulti<Addressindex, Txoutindex>, // Spent
pub blockhash_prefix_to_height: StoreUnique<BlockHashPrefix, Height>,
pub txid_prefix_to_txindex: StoreUnique<TxidPrefix, Txindex>,
}
@@ -27,14 +24,6 @@ impl Stores {
&path.join("addressbytes_prefix_to_addressindex"),
Version::from(1),
)?,
// addressindex_to_txoutindex_in: StoreMulti::open(
// &path.join("addressindex_to_txoutindex_in"),
// Version::from(1),
// )?,
// addressindex_to_txoutindex_out: StoreMulti::open(
// &path.join("addressindex_to_txoutindex_out"),
// Version::from(1),
// )?,
blockhash_prefix_to_height: StoreUnique::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
txid_prefix_to_txindex: StoreUnique::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
})
@@ -162,8 +151,6 @@ impl Stores {
pub fn min_height(&self) -> Option<Height> {
[
self.addressbytes_prefix_to_addressindex.height(),
// self.addressindex_to_txoutindex_in.height(),
// self.addressindex_to_txoutindex_out.height(),
self.blockhash_prefix_to_height.height(),
self.txid_prefix_to_txindex.height(),
]
@@ -177,8 +164,6 @@ impl Stores {
thread::scope(|scope| {
vec![
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height)),
// scope.spawn(|| self.addressindex_to_txoutindex_in.export(height)),
// scope.spawn(|| self.addressindex_to_txoutindex_out.export(height)),
scope.spawn(|| self.blockhash_prefix_to_height.export(height)),
scope.spawn(|| self.txid_prefix_to_txindex.export(height)),
]
bindex/src/storage/stores/multi.rs
+3 -3
View File
@@ -11,7 +11,7 @@ where
V: DatabaseValue,
{
meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 64],
pub parts: [OnceLock<Box<DatabaseMulti<K, V>>>; 256],
}
impl<K, V> StoreMulti<K, V>
@@ -33,7 +33,7 @@ where
// }
fn get_or_init_store(&self, key: &K) -> &DatabaseMulti<K, V> {
self.get_or_init_store_(key.as_ne_six_bits() as usize)
self.get_or_init_store_(key.as_ne_byte() as usize)
}
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseMulti<K, V> {
@@ -45,7 +45,7 @@ where
self.get_or_init_store(key);
self.parts
.get_mut(key.as_ne_six_bits() as usize)
.get_mut(key.as_ne_byte() as usize)
.unwrap()
.get_mut()
.unwrap()
bindex/src/storage/stores/unique.rs
+3 -3
View File
@@ -11,7 +11,7 @@ where
V: DatabaseValue,
{
meta: StoreMeta,
pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 64],
pub parts: [OnceLock<Box<DatabaseUnique<K, V>>>; 256],
}
impl<K, V> StoreUnique<K, V>
@@ -33,7 +33,7 @@ where
// }
fn get_or_init_store(&self, key: &K) -> &DatabaseUnique<K, V> {
self.get_or_init_store_(key.as_ne_six_bits() as usize)
self.get_or_init_store_(key.as_ne_byte() as usize)
}
fn get_or_init_store_(&self, storeindex: usize) -> &DatabaseUnique<K, V> {
@@ -46,7 +46,7 @@ where
self.get_or_init_store(key);
self.parts
.get_mut(key.as_ne_six_bits() as usize)
.get_mut(key.as_ne_byte() as usize)
.unwrap()
.get_mut()
.unwrap()
bindex/src/storage/vecs/mod.rs
+43 -28
View File
@@ -28,13 +28,13 @@ pub struct Vecs {
pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pk33index: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pk65index: StorableVec<Height, Addresstypeindex>,
pub height_to_p2pkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2shindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2trindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_p2wshindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex>,
pub height_to_size: StorableVec<Height, usize>,
pub height_to_timestamp: StorableVec<Height, Timestamp>,
pub height_to_weight: StorableVec<Height, Weight>,
@@ -120,13 +120,28 @@ impl Vecs {
&path.join("height_to_first_unkownindex"),
Version::from(1),
)?,
height_to_p2pk33index: StorableVec::import(&path.join("height_to_p2pk33index"), Version::from(1))?,
height_to_p2pk65index: StorableVec::import(&path.join("height_to_p2pk65index"), Version::from(1))?,
height_to_p2pkhindex: StorableVec::import(&path.join("height_to_p2pkhindex"), Version::from(1))?,
height_to_p2shindex: StorableVec::import(&path.join("height_to_p2shindex"), Version::from(1))?,
height_to_p2trindex: StorableVec::import(&path.join("height_to_p2trindex"), Version::from(1))?,
height_to_p2wpkhindex: StorableVec::import(&path.join("height_to_p2wpkhindex"), Version::from(1))?,
height_to_p2wshindex: StorableVec::import(&path.join("height_to_p2wshindex"), Version::from(1))?,
height_to_first_p2pk33index: StorableVec::import(
&path.join("height_to_first_p2pk33index"),
Version::from(1),
)?,
height_to_first_p2pk65index: StorableVec::import(
&path.join("height_to_first_p2pk65index"),
Version::from(1),
)?,
height_to_first_p2pkhindex: StorableVec::import(
&path.join("height_to_first_p2pkhindex"),
Version::from(1),
)?,
height_to_first_p2shindex: StorableVec::import(&path.join("height_to_first_p2shindex"), Version::from(1))?,
height_to_first_p2trindex: StorableVec::import(&path.join("height_to_first_p2trindex"), Version::from(1))?,
height_to_first_p2wpkhindex: StorableVec::import(
&path.join("height_to_first_p2wpkhindex"),
Version::from(1),
)?,
height_to_first_p2wshindex: StorableVec::import(
&path.join("height_to_first_p2wshindex"),
Version::from(1),
)?,
height_to_size: StorableVec::import(&path.join("height_to_size"), Version::from(1))?,
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
height_to_weight: StorableVec::import(&path.join("height_to_weight"), Version::from(1))?,
@@ -383,13 +398,13 @@ impl Vecs {
&self.height_to_first_txinindex,
&self.height_to_first_txoutindex,
&self.height_to_first_unknownindex,
&self.height_to_p2pk33index,
&self.height_to_p2pk65index,
&self.height_to_p2pkhindex,
&self.height_to_p2shindex,
&self.height_to_p2trindex,
&self.height_to_p2wpkhindex,
&self.height_to_p2wshindex,
&self.height_to_first_p2pk33index,
&self.height_to_first_p2pk65index,
&self.height_to_first_p2pkhindex,
&self.height_to_first_p2shindex,
&self.height_to_first_p2trindex,
&self.height_to_first_p2wpkhindex,
&self.height_to_first_p2wshindex,
&self.height_to_size,
&self.height_to_timestamp,
&self.height_to_weight,
@@ -427,13 +442,13 @@ impl Vecs {
&mut self.height_to_first_txinindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_first_unknownindex,
&mut self.height_to_p2pk33index,
&mut self.height_to_p2pk65index,
&mut self.height_to_p2pkhindex,
&mut self.height_to_p2shindex,
&mut self.height_to_p2trindex,
&mut self.height_to_p2wpkhindex,
&mut self.height_to_p2wshindex,
&mut self.height_to_first_p2pk33index,
&mut self.height_to_first_p2pk65index,
&mut self.height_to_first_p2pkhindex,
&mut self.height_to_first_p2shindex,
&mut self.height_to_first_p2trindex,
&mut self.height_to_first_p2wpkhindex,
&mut self.height_to_first_p2wshindex,
&mut self.height_to_size,
&mut self.height_to_timestamp,
&mut self.height_to_weight,
bindex/src/structs/txoutindex.rs
+3 -6
View File
@@ -1,7 +1,4 @@
use std::{
ops::{Add, AddAssign},
u64,
};
use std::ops::{Add, AddAssign};
use derive_deref::{Deref, DerefMut};
use snkrj::{direct_repr, Storable, UnsizedStorable};
@@ -13,7 +10,7 @@ pub struct Txoutindex(u64);
direct_repr!(Txoutindex);
impl Txoutindex {
pub const MAX: Self = Self(u64::MAX);
pub const COINBASE: Self = Self(u64::MAX);
pub fn incremented(self) -> Self {
Self(*self + 1)
@@ -24,7 +21,7 @@ impl Txoutindex {
}
pub fn is_coinbase(self) -> bool {
self == Self::MAX
self == Self::COINBASE
}
}