Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-27 02:38:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: snapshot

Browse Source
This commit is contained in:
nym21
2026-02-18 19:38:19 +01:00
parent f04b548f8c
commit 4352868410
9 changed files with 413 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_indexer/src/processor/metadata.rs
+25 -10
View File
@@ -3,7 +3,7 @@ use brk_types::{BlockHashPrefix, Timestamp};
use tracing::error;
use vecdb::WritableVec;
use super::BlockProcessor;
use super::{BlockProcessor, ComputedTx};
use crate::IndexesExt;
impl BlockProcessor<'_> {
@@ -45,16 +45,31 @@ impl BlockProcessor<'_> {
.blocks
.timestamp
.checked_push(height, Timestamp::from(self.block.header.time))?;
let (block_total_size, block_weight) = self.block.total_size_and_weight();
self.vecs
.blocks
.total_size
.checked_push(height, block_total_size.into())?;
self.vecs
.blocks
.weight
.checked_push(height, block_weight.into())?;
Ok(())
}
/// Push block total_size and weight, reusing per-tx sizes already computed in ComputedTx.
/// This avoids redundant tx serialization (base_size + total_size were already computed).
pub fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> {
let overhead =
bitcoin::block::Header::SIZE + bitcoin::VarInt::from(txs.len()).size();
let mut total_size = overhead;
let mut weight_wu = overhead * 4;
for ct in txs {
let base = ct.base_size as usize;
let total = ct.total_size as usize;
total_size += total;
weight_wu += base * 3 + total;
}
self.vecs
.blocks
.total_size
.checked_push(self.height, total_size.into())?;
self.vecs
.blocks
.weight
.checked_push(self.height, weight_wu.into())?;
Ok(())
}
}
crates/brk_indexer/src/processor/mod.rs
+66 -1
View File
@@ -6,7 +6,10 @@ mod types;
pub use types::*;
use brk_types::{Block, Height, TxInIndex, TxIndex, TxOutIndex};
use brk_cohort::ByAddressType;
use brk_error::Result;
use brk_types::{AddressHash, Block, Height, OutPoint, TxInIndex, TxIndex, TxOutIndex, TypeIndex};
use rustc_hash::{FxHashMap, FxHashSet};
use crate::{Indexes, Readers, Stores, Vecs};
@@ -28,4 +31,66 @@ impl BlockProcessor<'_> {
self.indexes.txinindex += TxInIndex::from(input_count);
self.indexes.txoutindex += TxOutIndex::from(output_count);
}
/// Finalizes outputs/inputs in parallel with storing tx metadata.
#[allow(clippy::too_many_arguments)]
pub fn finalize_and_store_metadata(
&mut self,
txs: Vec<ComputedTx>,
txouts: Vec<ProcessedOutput>,
txins: Vec<(TxInIndex, InputSource)>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
already_added: &mut ByAddressType<FxHashMap<AddressHash, TypeIndex>>,
same_block_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
let height = self.height;
let indexes = &mut *self.indexes;
// Split transactions vecs: finalize needs first_txoutindex/first_txinindex, metadata needs the rest
let (first_txoutindex, first_txinindex, mut tx_metadata) =
self.vecs.transactions.split_for_finalize();
let outputs = &mut self.vecs.outputs;
let inputs = &mut self.vecs.inputs;
let addresses = &mut self.vecs.addresses;
let scripts = &mut self.vecs.scripts;
let addr_hash_stores = &mut self.stores.addresstype_to_addresshash_to_addressindex;
let addr_txindex_stores = &mut self.stores.addresstype_to_addressindex_and_txindex;
let addr_outpoint_stores =
&mut self.stores.addresstype_to_addressindex_and_unspentoutpoint;
let txidprefix_store = &mut self.stores.txidprefix_to_txindex;
let (finalize_result, metadata_result) = rayon::join(
|| -> Result<()> {
txout::finalize_outputs(
indexes,
first_txoutindex,
outputs,
addresses,
scripts,
addr_hash_stores,
addr_txindex_stores,
addr_outpoint_stores,
txouts,
same_block_spent_outpoints,
already_added,
same_block_info,
)?;
txin::finalize_inputs(
first_txinindex,
inputs,
addr_txindex_stores,
addr_outpoint_stores,
txins,
same_block_info,
)
},
|| tx::store_tx_metadata(height, txs, txidprefix_store, &mut tx_metadata),
);
finalize_result?;
metadata_result?;
Ok(())
}
}
crates/brk_indexer/src/processor/tx.rs
+29 -44
View File
@@ -1,8 +1,10 @@
use brk_error::{Error, Result};
use brk_types::{StoredBool, TxIndex, Txid, TxidPrefix};
use brk_store::Store;
use brk_types::{Height, StoredBool, TxIndex, Txid, TxidPrefix};
use rayon::prelude::*;
use vecdb::{AnyVec, WritableVec, likely};
use crate::TxMetadataVecs;
use crate::constants::DUPLICATE_TXIDS;
use super::{BlockProcessor, ComputedTx};
@@ -78,47 +80,30 @@ impl<'a> BlockProcessor<'a> {
Ok(())
}
pub fn store_transaction_metadata(&mut self, txs: Vec<ComputedTx>) -> Result<()> {
let height = self.height;
for ct in txs {
if ct.prev_txindex_opt.is_none() {
self.stores
.txidprefix_to_txindex
.insert(ct.txid_prefix, ct.txindex);
}
self.vecs
.transactions
.height
.checked_push(ct.txindex, height)?;
self.vecs
.transactions
.txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
self.vecs
.transactions
.txid
.checked_push(ct.txindex, ct.txid)?;
self.vecs
.transactions
.rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
self.vecs
.transactions
.base_size
.checked_push(ct.txindex, ct.base_size.into())?;
self.vecs
.transactions
.total_size
.checked_push(ct.txindex, ct.total_size.into())?;
self.vecs
.transactions
.is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
}
pub(super) fn store_tx_metadata(
height: Height,
txs: Vec<ComputedTx>,
store: &mut Store<TxidPrefix, TxIndex>,
md: &mut TxMetadataVecs<'_>,
) -> Result<()> {
for ct in txs {
if ct.prev_txindex_opt.is_none() {
store.insert(ct.txid_prefix, ct.txindex);
}
md.height.checked_push(ct.txindex, height)?;
md.txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
md.txid.checked_push(ct.txindex, ct.txid)?;
md.rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
md.base_size
.checked_push(ct.txindex, ct.base_size.into())?;
md.total_size
.checked_push(ct.txindex, ct.total_size.into())?;
md.is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
crates/brk_indexer/src/processor/txin.rs
+82 -97
View File
@@ -1,21 +1,25 @@
use brk_cohort::ByAddressType;
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{
AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, Txid,
TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::WritableVec;
use vecdb::{PcoVec, WritableVec};
use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo};
use crate::InputsVecs;
impl<'a> BlockProcessor<'a> {
pub fn process_inputs(
&self,
txs: &[ComputedTx],
txid_prefix_to_txindex: &mut FxHashMap<TxidPrefix, TxIndex>,
) -> Result<Vec<(TxInIndex, InputSource)>> {
let txid_prefix_to_txindex: FxHashMap<_, _> =
txs.iter().map(|ct| (ct.txid_prefix, ct.txindex)).collect();
txid_prefix_to_txindex.clear();
txid_prefix_to_txindex.extend(txs.iter().map(|ct| (ct.txid_prefix, ct.txindex)));
let base_txindex = self.indexes.txindex;
let base_txinindex = self.indexes.txinindex;
@@ -28,6 +32,8 @@ impl<'a> BlockProcessor<'a> {
}
}
let txid_prefix_to_txindex = &*txid_prefix_to_txindex;
let txins = items
.into_par_iter()
.enumerate()
@@ -125,99 +131,78 @@ impl<'a> BlockProcessor<'a> {
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
) -> FxHashSet<OutPoint> {
txins
.iter()
.filter_map(|(_, input_source)| {
let InputSource::SameBlock { outpoint, .. } = input_source else {
return None;
};
if !outpoint.is_coinbase() {
Some(*outpoint)
} else {
None
}
})
.collect()
}
pub fn finalize_inputs(
&mut self,
txins: Vec<(TxInIndex, InputSource)>,
mut same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
for (txinindex, input_source) in txins {
let (vin, txindex, outpoint, outputtype, typeindex) = match input_source {
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
outputtype,
typeindex,
} => (vin, txindex, outpoint, outputtype, typeindex),
InputSource::SameBlock {
txindex,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(vin, txindex, outpoint, OutputType::Unknown, TypeIndex::COINBASE)
} else {
let info = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block output not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, outpoint);
})?;
(vin, txindex, outpoint, info.outputtype, info.typeindex)
}
}
};
if vin.is_zero() {
self.vecs
.transactions
.first_txinindex
.checked_push(txindex, txinindex)?;
}
self.vecs
.inputs
.txindex
.checked_push(txinindex, txindex)?;
self.vecs
.inputs
.outpoint
.checked_push(txinindex, outpoint)?;
self.vecs
.inputs
.outputtype
.checked_push(txinindex, outputtype)?;
self.vecs
.inputs
.typeindex
.checked_push(txinindex, typeindex)?;
if !outputtype.is_address() {
continue;
}
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
);
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove(AddressIndexOutPoint::from((addressindex, outpoint)));
}
Ok(())
out: &mut FxHashSet<OutPoint>,
) {
out.clear();
out.extend(txins.iter().filter_map(|(_, input_source)| match input_source {
InputSource::SameBlock { outpoint, .. } if !outpoint.is_coinbase() => Some(*outpoint),
_ => None,
}));
}
}
pub(super) fn finalize_inputs(
first_txinindex: &mut PcoVec<TxIndex, TxInIndex>,
inputs: &mut InputsVecs,
addr_txindex_stores: &mut ByAddressType<Store<AddressIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddressType<Store<AddressIndexOutPoint, Unit>>,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
for (txinindex, input_source) in txins {
let (vin, txindex, outpoint, outputtype, typeindex) = match input_source {
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
outputtype,
typeindex,
} => (vin, txindex, outpoint, outputtype, typeindex),
InputSource::SameBlock {
txindex,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(vin, txindex, outpoint, OutputType::Unknown, TypeIndex::COINBASE)
} else {
let info = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block output not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, outpoint);
})?;
(vin, txindex, outpoint, info.outputtype, info.typeindex)
}
}
};
if vin.is_zero() {
first_txinindex.checked_push(txindex, txinindex)?;
}
inputs.txindex.checked_push(txinindex, txindex)?;
inputs.outpoint.checked_push(txinindex, outpoint)?;
inputs.outputtype.checked_push(txinindex, outputtype)?;
inputs.typeindex.checked_push(txinindex, typeindex)?;
if !outputtype.is_address() {
continue;
}
let addresstype = outputtype;
let addressindex = typeindex;
addr_txindex_stores
.get_mut_unwrap(addresstype)
.insert(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
);
addr_outpoint_stores
.get_mut_unwrap(addresstype)
.remove(AddressIndexOutPoint::from((addressindex, outpoint)));
}
Ok(())
}
crates/brk_indexer/src/processor/txout.rs
+131 -141
View File
@@ -1,14 +1,16 @@
use brk_cohort::ByAddressType;
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType,
Sats, TxIndex, TxOutIndex, TypeIndex, Unit, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::WritableVec;
use vecdb::{BytesVec, WritableVec};
use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo};
use crate::{AddressesVecs, Indexes, OutputsVecs, ScriptsVecs};
impl<'a> BlockProcessor<'a> {
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
@@ -104,150 +106,138 @@ impl<'a> BlockProcessor<'a> {
)
.collect()
}
}
pub fn finalize_outputs(
&mut self,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
) -> Result<FxHashMap<OutPoint, SameBlockOutputInfo>> {
let mut already_added_addresshash: ByAddressType<FxHashMap<AddressHash, TypeIndex>> =
ByAddressType::default();
let mut same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo> =
FxHashMap::with_capacity_and_hasher(
same_block_spent_outpoints.len(),
Default::default(),
);
#[allow(clippy::too_many_arguments)]
pub(super) fn finalize_outputs(
indexes: &mut Indexes,
first_txoutindex: &mut BytesVec<TxIndex, TxOutIndex>,
outputs: &mut OutputsVecs,
addresses: &mut AddressesVecs,
scripts: &mut ScriptsVecs,
addr_hash_stores: &mut ByAddressType<Store<AddressHash, TypeIndex>>,
addr_txindex_stores: &mut ByAddressType<Store<AddressIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddressType<Store<AddressIndexOutPoint, Unit>>,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
already_added_addresshash: &mut ByAddressType<FxHashMap<AddressHash, TypeIndex>>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
already_added_addresshash
.values_mut()
.for_each(|m| m.clear());
same_block_output_info.clear();
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
self.vecs
.transactions
.first_txoutindex
.checked_push(txindex, txoutindex)?;
}
self.vecs
.outputs
.txindex
.checked_push(txoutindex, txindex)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = self.indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.stores
.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.vecs.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
self.vecs
.scripts
.p2ms_to_txindex
.checked_push(self.indexes.p2msoutputindex, txindex)?;
self.indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
self.vecs
.scripts
.opreturn_to_txindex
.checked_push(self.indexes.opreturnindex, txindex)?;
self.indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
self.vecs
.scripts
.empty_to_txindex
.checked_push(self.indexes.emptyoutputindex, txindex)?;
self.indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
self.vecs
.scripts
.unknown_to_txindex
.checked_push(self.indexes.unknownoutputindex, txindex)?;
self.indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
self.vecs.outputs.value.checked_push(txoutindex, sats)?;
self.vecs
.outputs
.outputtype
.checked_push(txoutindex, outputtype)?;
self.vecs
.outputs
.typeindex
.checked_push(txoutindex, typeindex)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(
outpoint,
SameBlockOutputInfo {
outputtype,
typeindex,
},
);
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
);
}
if vout.is_zero() {
first_txoutindex.checked_push(txindex, txoutindex)?;
}
Ok(same_block_output_info)
outputs.txindex.checked_push(txoutindex, txindex)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
addr_hash_stores
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
addresses.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
scripts
.p2ms_to_txindex
.checked_push(indexes.p2msoutputindex, txindex)?;
indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
scripts
.opreturn_to_txindex
.checked_push(indexes.opreturnindex, txindex)?;
indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
scripts
.empty_to_txindex
.checked_push(indexes.emptyoutputindex, txindex)?;
indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
scripts
.unknown_to_txindex
.checked_push(indexes.unknownoutputindex, txindex)?;
indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
outputs.value.checked_push(txoutindex, sats)?;
outputs.outputtype.checked_push(txoutindex, outputtype)?;
outputs.typeindex.checked_push(txoutindex, typeindex)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
addr_txindex_stores
.get_mut_unwrap(addresstype)
.insert(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(
outpoint,
SameBlockOutputInfo {
outputtype,
typeindex,
},
);
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
addr_outpoint_stores
.get_mut_unwrap(addresstype)
.insert(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
);
}
}
Ok(())
}
crates/brk_indexer/src/processor/types.rs
+12
View File
@@ -1,8 +1,10 @@
use bitcoin::{Transaction, TxOut};
use brk_cohort::ByAddressType;
use brk_types::{
AddressBytes, AddressHash, OutPoint, OutputType, TxIndex, TxOutIndex, Txid, TxidPrefix,
TypeIndex, Vin, Vout,
};
use rustc_hash::{FxHashMap, FxHashSet};
#[derive(Debug)]
pub enum InputSource {
@@ -45,3 +47,13 @@ pub struct ComputedTx<'a> {
pub base_size: u32,
pub total_size: u32,
}
/// Reusable buffers cleared and refilled each block to avoid allocation churn.
#[derive(Default)]
pub struct BlockBuffers {
pub txid_prefix_map: FxHashMap<TxidPrefix, TxIndex>,
pub same_block_spent: FxHashSet<OutPoint>,
pub already_added_addresses: ByAddressType<FxHashMap<AddressHash, TypeIndex>>,
pub same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo>,
}