Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-24 09:28:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-13 10:52:00 +01:00
parent 3526a177fc
commit 158b0254ed
73 changed files with 2230 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/chain.rs
+4 -8
View File
@@ -7,7 +7,7 @@ use brk_types::{
Bitcoin, CheckedSub, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, FeeRate, HalvingEpoch,
Height, MonthIndex, ONE_DAY_IN_SEC_F64, QuarterIndex, Sats, SemesterIndex, StoredBool,
StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxInIndex, TxIndex, TxOutIndex,
TxVersion, Version, WeekIndex, Weight, YearIndex,
TxVersion, VSize, Version, WeekIndex, Weight, YearIndex,
};
use vecdb::{
Database, EagerVec, Exit, GenericStoredVec, ImportableVec, IterableCloneableVec, IterableVec,
@@ -95,14 +95,14 @@ pub struct Vecs {
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU64>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredU64>,
pub indexes_to_tx_vsize: ComputedVecsFromTxindex<VSize>,
pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
pub indexes_to_unknownoutput_count: ComputedVecsFromHeight<StoredU64>,
pub txinindex_to_value: EagerVec<PcoVec<TxInIndex, Sats>>,
pub indexes_to_input_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_is_coinbase: LazyVecFrom2<TxIndex, StoredBool, TxIndex, Height, Height, TxIndex>,
pub indexes_to_output_count: ComputedVecsFromTxindex<StoredU64>,
pub txindex_to_vsize: LazyVecFrom1<TxIndex, StoredU64, TxIndex, Weight>,
pub txindex_to_vsize: LazyVecFrom1<TxIndex, VSize, TxIndex, Weight>,
pub txindex_to_weight: LazyVecFrom2<TxIndex, Weight, TxIndex, StoredU32, TxIndex, StoredU32>,
pub txindex_to_fee: EagerVec<PcoVec<TxIndex, Sats>>,
pub txindex_to_fee_rate: EagerVec<PcoVec<TxIndex, FeeRate>>,
@@ -269,11 +269,7 @@ impl Vecs {
"vsize",
version + Version::ZERO,
txindex_to_weight.boxed_clone(),
|index: TxIndex, iter| {
iter.get(index).map(|weight| {
StoredU64::from(bitcoin::Weight::from(weight).to_vbytes_ceil() as usize)
})
},
|index: TxIndex, iter| iter.get(index).map(VSize::from),
);
let txindex_to_is_coinbase = LazyVecFrom2::init(
crates/brk_computer/src/grouped/value_from_txindex.rs
+1 -1
View File
@@ -56,7 +56,7 @@ impl ComputedValueVecsFromTxindex {
let bitcoin_txindex = LazyVecFrom1::init(
&name_btc,
version + VERSION,
source_vec.map_or_else(|| sats.txindex.u().boxed_clone(), |s| s),
source_vec.unwrap_or_else(|| sats.txindex.u().boxed_clone()),
|txindex: TxIndex, iter| iter.get_at(txindex.to_usize()).map(Bitcoin::from),
);
crates/brk_computer/src/lib.rs
+1 -1
View File
@@ -22,7 +22,7 @@ mod market;
mod pools;
mod price;
mod stateful;
mod stateful_new;
// mod stateful_new;
mod states;
mod traits;
mod utils;
crates/brk_computer/src/stateful/mod.rs
+95 -9
View File
@@ -64,7 +64,7 @@ mod address_cohorts;
mod address_indexes;
mod addresstype;
mod common;
mod flushable;
// mod flushable;
mod range_map;
mod readers;
mod r#trait;
@@ -102,6 +102,7 @@ pub struct Vecs {
// States
// ---
pub chain_state: BytesVec<Height, SupplyState>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: utxo_cohorts::Vecs,
@@ -179,6 +180,10 @@ impl Vecs {
ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
txoutindex_to_txinindex: BytesVec::forced_import_with(
ImportOptions::new(&db, "txinindex", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
@@ -382,13 +387,20 @@ impl Vecs {
.map(|price| price.timeindexes_to_price_close.dateindex.u());
let height_to_date_fixed = &indexes.height_to_date_fixed;
let height_to_first_p2aaddressindex = &indexer.vecs.address.height_to_first_p2aaddressindex;
let height_to_first_p2pk33addressindex = &indexer.vecs.address.height_to_first_p2pk33addressindex;
let height_to_first_p2pk65addressindex = &indexer.vecs.address.height_to_first_p2pk65addressindex;
let height_to_first_p2pkhaddressindex = &indexer.vecs.address.height_to_first_p2pkhaddressindex;
let height_to_first_p2shaddressindex = &indexer.vecs.address.height_to_first_p2shaddressindex;
let height_to_first_p2traddressindex = &indexer.vecs.address.height_to_first_p2traddressindex;
let height_to_first_p2wpkhaddressindex = &indexer.vecs.address.height_to_first_p2wpkhaddressindex;
let height_to_first_p2wshaddressindex = &indexer.vecs.address.height_to_first_p2wshaddressindex;
let height_to_first_p2pk33addressindex =
&indexer.vecs.address.height_to_first_p2pk33addressindex;
let height_to_first_p2pk65addressindex =
&indexer.vecs.address.height_to_first_p2pk65addressindex;
let height_to_first_p2pkhaddressindex =
&indexer.vecs.address.height_to_first_p2pkhaddressindex;
let height_to_first_p2shaddressindex =
&indexer.vecs.address.height_to_first_p2shaddressindex;
let height_to_first_p2traddressindex =
&indexer.vecs.address.height_to_first_p2traddressindex;
let height_to_first_p2wpkhaddressindex =
&indexer.vecs.address.height_to_first_p2wpkhaddressindex;
let height_to_first_p2wshaddressindex =
&indexer.vecs.address.height_to_first_p2wshaddressindex;
let height_to_first_txindex = &indexer.vecs.tx.height_to_first_txindex;
let height_to_txindex_count = chain.indexes_to_tx_count.height.u();
let height_to_first_txinindex = &indexer.vecs.txin.height_to_first_txinindex;
@@ -489,6 +501,7 @@ impl Vecs {
.unwrap_or_default(),
)
.min(chain_state_starting_height)
.min(Height::from(self.txoutindex_to_txinindex.stamp()).incremented())
.min(self.any_address_indexes.min_stamped_height())
.min(self.addresses_data.min_stamped_height())
.min(Height::from(self.height_to_unspendable_supply.len()))
@@ -507,6 +520,7 @@ impl Vecs {
let starting_height = if starting_height.is_not_zero() {
let mut set = [self.chain_state.rollback_before(stamp)?]
.into_iter()
.chain([self.txoutindex_to_txinindex.rollback_before(stamp)?])
.chain(self.any_address_indexes.rollback_before(stamp)?)
.chain(self.addresses_data.rollback_before(stamp)?)
.map(Height::from)
@@ -589,6 +603,7 @@ impl Vecs {
chain_state = vec![];
self.txoutindex_to_txinindex.reset()?;
self.any_address_indexes.reset()?;
self.addresses_data.reset()?;
@@ -760,6 +775,7 @@ impl Vecs {
addresstype_to_typedindex_to_sent_data,
mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource,
mut combined_txindex_vecs,
txoutindex_to_txinindex_updates,
) = thread::scope(|scope| {
scope.spawn(|| {
self.utxo_cohorts
@@ -860,6 +876,7 @@ impl Vecs {
addresstype_to_typedindex_to_sent_data,
sending_addresstype_to_typeindex_to_addressdatawithsource,
input_txindex_vecs,
txoutindex_to_txinindex_updates,
) = (first_txinindex + 1..first_txinindex + usize::from(input_count))
.into_par_iter()
.map(|i| {
@@ -884,7 +901,15 @@ impl Vecs {
let prev_height = *txoutindex_range_to_height.get(txoutindex).unwrap();
if input_type.is_not_address() {
return (txindex, prev_height, value, input_type, None);
return (
txinindex,
txoutindex,
txindex,
prev_height,
value,
input_type,
None,
);
}
let typeindex = txoutindex_to_typeindex
@@ -902,6 +927,8 @@ impl Vecs {
);
(
txinindex,
txoutindex,
txindex,
prev_height,
value,
@@ -916,6 +943,7 @@ impl Vecs {
HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(),
AddressTypeToTypeIndexMap::default(),
AddressTypeToTypeIndexMap::<TxIndexVec>::default(),
Vec::<(TxOutIndex, TxInIndex)>::new(),
)
},
|(
@@ -923,8 +951,11 @@ impl Vecs {
mut height_to_addresstype_to_typedindex_to_data,
mut addresstype_to_typeindex_to_addressdatawithsource,
mut txindex_vecs,
mut txoutindex_to_txinindex_updates,
),
(
txinindex,
txoutindex,
txindex,
prev_height,
value,
@@ -936,6 +967,8 @@ impl Vecs {
.or_default()
.iterate(value, output_type);
txoutindex_to_txinindex_updates.push((txoutindex, txinindex));
if let Some((typeindex, addressdata_opt)) =
typeindex_with_addressdata_opt
{
@@ -966,6 +999,7 @@ impl Vecs {
height_to_addresstype_to_typedindex_to_data,
addresstype_to_typeindex_to_addressdatawithsource,
txindex_vecs,
txoutindex_to_txinindex_updates,
)
},
)
@@ -976,6 +1010,7 @@ impl Vecs {
HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(),
AddressTypeToTypeIndexMap::default(),
AddressTypeToTypeIndexMap::<TxIndexVec>::default(),
Vec::<(TxOutIndex, TxInIndex)>::new(),
)
},
|(
@@ -983,12 +1018,14 @@ impl Vecs {
addresstype_to_typedindex_to_data,
addresstype_to_typeindex_to_addressdatawithsource,
txindex_vecs,
txoutindex_to_txinindex_updates,
),
(
height_to_transacted2,
addresstype_to_typedindex_to_data2,
addresstype_to_typeindex_to_addressdatawithsource2,
txindex_vecs2,
txoutindex_to_txinindex_updates2,
)| {
let (mut height_to_transacted, height_to_transacted_consumed) =
if height_to_transacted.len() > height_to_transacted2.len() {
@@ -1028,12 +1065,32 @@ impl Vecs {
.merge_mut(v);
});
let (
mut txoutindex_to_txinindex_updates,
txoutindex_to_txinindex_updates_consumed,
) = if txoutindex_to_txinindex_updates.len()
> txoutindex_to_txinindex_updates2.len()
{
(
txoutindex_to_txinindex_updates,
txoutindex_to_txinindex_updates2,
)
} else {
(
txoutindex_to_txinindex_updates2,
txoutindex_to_txinindex_updates,
)
};
txoutindex_to_txinindex_updates
.extend(txoutindex_to_txinindex_updates_consumed);
(
height_to_transacted,
addresstype_to_typedindex_to_data,
addresstype_to_typeindex_to_addressdatawithsource
.merge(addresstype_to_typeindex_to_addressdatawithsource2),
txindex_vecs.merge_vec(txindex_vecs2),
txoutindex_to_txinindex_updates,
)
},
);
@@ -1051,6 +1108,7 @@ impl Vecs {
addresstype_to_typedindex_to_sent_data,
addresstype_to_typeindex_to_addressdatawithsource,
combined_txindex_vecs,
txoutindex_to_txinindex_updates,
)
});
@@ -1159,6 +1217,12 @@ impl Vecs {
self.utxo_cohorts.send(height_to_sent, &mut chain_state);
});
// Update txoutindex_to_txinindex
self.update_txoutindex_to_txinindex(
usize::from(output_count),
txoutindex_to_txinindex_updates,
)?;
self.height_to_unspendable_supply
.truncate_push(height, unspendable_supply)?;
@@ -1599,6 +1663,8 @@ impl Vecs {
let stamp = Stamp::from(height);
self.txoutindex_to_txinindex
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.any_address_indexes
.stamped_flush_maybe_with_changes(stamp, with_changes)?;
self.addresses_data
@@ -1613,4 +1679,24 @@ impl Vecs {
Ok(())
}
/// Update txoutindex_to_txinindex for a block.
///
/// 1. Push UNSPENT for all new outputs in the block
/// 2. Update spent outputs with their spending txinindex
pub fn update_txoutindex_to_txinindex(
&mut self,
output_count: usize,
updates: Vec<(TxOutIndex, TxInIndex)>,
) -> Result<()> {
// Push UNSPENT for all new outputs in this block
for _ in 0..output_count {
self.txoutindex_to_txinindex.push(TxInIndex::UNSPENT);
}
// Update spent outputs with their spending txinindex
for (txoutindex, txinindex) in updates {
self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
}
Ok(())
}
}
crates/brk_computer/src/stateful_new/compute/block_loop.rs
+11 -2
View File
@@ -13,7 +13,7 @@ use std::thread;
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_indexer::Indexer;
use brk_types::{DateIndex, Dollars, Height, OutputType, Sats, Timestamp, TypeIndex};
use brk_types::{DateIndex, Dollars, Height, OutputType, Sats, Timestamp, TxInIndex, TxOutIndex, TypeIndex};
use log::info;
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex};
@@ -248,6 +248,7 @@ pub fn process_blocks(
sent_data: Default::default(),
address_data: Default::default(),
txindex_vecs: Default::default(),
txoutindex_to_txinindex_updates: Default::default(),
}
};
@@ -346,6 +347,12 @@ pub fn process_blocks(
vecs.utxo_cohorts.send(height_to_sent, chain_state);
});
// Update txoutindex_to_txinindex
vecs.update_txoutindex_to_txinindex(
output_count,
inputs_result.txoutindex_to_txinindex_updates,
)?;
// Push to height-indexed vectors
vecs.height_to_unspendable_supply
.truncate_push(height, unspendable_supply)?;
@@ -503,7 +510,9 @@ fn flush_checkpoint(
exit,
)?;
// Flush chain state with stamp
// Flush chain state and txoutindex_to_txinindex with stamp
vecs.txoutindex_to_txinindex
.stamped_flush_with_changes(height.into())?;
vecs.chain_state.stamped_flush_with_changes(height.into())?;
Ok(())
crates/brk_computer/src/stateful_new/process/inputs.rs
+24 -7
View File
@@ -33,6 +33,8 @@ pub struct InputsResult {
pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Transaction indexes per address for tx_count tracking.
pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
/// Updates to txoutindex_to_txinindex: (spent txoutindex, spending txinindex).
pub txoutindex_to_txinindex_updates: Vec<(TxOutIndex, TxInIndex)>,
}
/// Process inputs (spent UTXOs) for a block in parallel.
@@ -64,7 +66,7 @@ pub fn process_inputs(
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> InputsResult {
let (height_to_sent, sent_data, address_data, txindex_vecs) = (first_txinindex
let (height_to_sent, sent_data, address_data, txindex_vecs, txoutindex_to_txinindex_updates) = (first_txinindex
..first_txinindex + input_count)
.into_par_iter()
.map(|i| {
@@ -88,7 +90,7 @@ pub fn process_inputs(
// Non-address inputs don't need typeindex or address lookup
if input_type.is_not_address() {
return (prev_height, value, input_type, None);
return (txinindex, txoutindex, prev_height, value, input_type, None);
}
let typeindex =
@@ -107,6 +109,8 @@ pub fn process_inputs(
);
(
txinindex,
txoutindex,
prev_height,
value,
input_type,
@@ -120,15 +124,18 @@ pub fn process_inputs(
HeightToAddressTypeToVec::default(),
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::default(),
AddressTypeToTypeIndexMap::<TxIndexVec>::default(),
Vec::<(TxOutIndex, TxInIndex)>::new(),
)
},
|(mut height_to_sent, mut sent_data, mut address_data, mut txindex_vecs),
(prev_height, value, output_type, addr_info)| {
|(mut height_to_sent, mut sent_data, mut address_data, mut txindex_vecs, mut txoutindex_to_txinindex_updates),
(txinindex, txoutindex, prev_height, value, output_type, addr_info)| {
height_to_sent
.entry(prev_height)
.or_default()
.iterate(value, output_type);
txoutindex_to_txinindex_updates.push((txoutindex, txinindex));
if let Some((typeindex, txindex, value, addr_data_opt)) = addr_info {
sent_data
.entry(prev_height)
@@ -149,7 +156,7 @@ pub fn process_inputs(
.push(txindex);
}
(height_to_sent, sent_data, address_data, txindex_vecs)
(height_to_sent, sent_data, address_data, txindex_vecs, txoutindex_to_txinindex_updates)
},
)
.reduce(
@@ -159,9 +166,10 @@ pub fn process_inputs(
HeightToAddressTypeToVec::default(),
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::default(),
AddressTypeToTypeIndexMap::<TxIndexVec>::default(),
Vec::<(TxOutIndex, TxInIndex)>::new(),
)
},
|(mut h1, mut s1, a1, tx1), (h2, s2, a2, tx2)| {
|(mut h1, mut s1, a1, tx1, updates1), (h2, s2, a2, tx2, updates2)| {
// Merge height_to_sent maps
for (k, v) in h2 {
*h1.entry(k).or_default() += v;
@@ -170,7 +178,15 @@ pub fn process_inputs(
// Merge sent_data maps
s1.merge_mut(s2);
(h1, s1, a1.merge(a2), tx1.merge_vec(tx2))
// Merge txoutindex_to_txinindex updates (extend longest with shortest)
let (mut updates, updates_consumed) = if updates1.len() > updates2.len() {
(updates1, updates2)
} else {
(updates2, updates1)
};
updates.extend(updates_consumed);
(h1, s1, a1.merge(a2), tx1.merge_vec(tx2), updates)
},
);
@@ -179,6 +195,7 @@ pub fn process_inputs(
sent_data,
address_data,
txindex_vecs,
txoutindex_to_txinindex_updates,
}
}
crates/brk_computer/src/stateful_new/vecs.rs
+40 -7
View File
@@ -5,10 +5,10 @@ use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Dollars, Height, Sats, StoredU64, Version};
use brk_types::{Dollars, Height, Sats, StoredU64, TxInIndex, TxOutIndex, Version};
use vecdb::{
BytesVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, LazyVecFrom1,
PAGE_SIZE, PcoVec,
AnyStoredVec, BytesVec, Database, EagerVec, Exit, ImportableVec, IterableCloneableVec,
LazyVecFrom1, PAGE_SIZE, PcoVec,
};
use crate::{
@@ -35,6 +35,7 @@ pub struct Vecs {
// States
// ---
pub chain_state: BytesVec<Height, SupplyState>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: UTXOCohorts,
@@ -83,6 +84,10 @@ impl Vecs {
vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
txoutindex_to_txinindex: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "txinindex", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
@@ -185,9 +190,7 @@ impl Vecs {
starting_indexes: &mut Indexes,
exit: &Exit,
) -> Result<()> {
use super::compute::{
StartMode, determine_start_mode, process_blocks,
};
use super::compute::{StartMode, determine_start_mode, process_blocks};
use crate::states::BlockState;
use vecdb::{AnyVec, GenericStoredVec, Stamp, TypedVecIterator, VecIndex};
@@ -201,6 +204,7 @@ impl Vecs {
let stateful_min = utxo_min
.min(address_min)
.min(Height::from(self.chain_state.len()))
.min(Height::from(self.txoutindex_to_txinindex.stamp()).incremented())
.min(self.any_address_indexes.min_stamped_height())
.min(self.addresses_data.min_stamped_height())
.min(Height::from(self.height_to_unspendable_supply.len()))
@@ -215,6 +219,7 @@ impl Vecs {
// Rollback state vectors
let _ = self.chain_state.rollback_before(stamp);
let _ = self.txoutindex_to_txinindex.rollback_before(stamp);
let _ = self.any_address_indexes.rollback_before(stamp);
let _ = self.addresses_data.rollback_before(stamp);
@@ -252,6 +257,7 @@ impl Vecs {
}
StartMode::Fresh => {
// Reset all state
self.txoutindex_to_txinindex.reset()?;
self.any_address_indexes.reset()?;
self.addresses_data.reset()?;
@@ -271,7 +277,14 @@ impl Vecs {
};
// 3. Get last height from indexer
let last_height = Height::from(indexer.vecs.block.height_to_blockhash.len().saturating_sub(1));
let last_height = Height::from(
indexer
.vecs
.block
.height_to_blockhash
.len()
.saturating_sub(1),
);
// 4. Process blocks
if starting_height <= last_height {
@@ -401,4 +414,24 @@ impl Vecs {
self.db.compact()?;
Ok(())
}
/// Update txoutindex_to_txinindex for a block.
///
/// 1. Push UNSPENT for all new outputs in the block
/// 2. Update spent outputs with their spending txinindex
pub fn update_txoutindex_to_txinindex(
&mut self,
output_count: usize,
updates: Vec<(TxOutIndex, TxInIndex)>,
) -> Result<()> {
// Push UNSPENT for all new outputs in this block
for _ in 0..output_count {
self.txoutindex_to_txinindex.push(TxInIndex::UNSPENT);
}
// Update spent outputs with their spending txinindex
for (txoutindex, txinindex) in updates {
self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
}
Ok(())
}
}