Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

indexer: fix bug

Browse Source
This commit is contained in:
nym21
2025-12-04 21:30:08 +01:00
parent a1f31a14be
commit dc2fa233ab
4 changed files with 213 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
crates/brk_computer/examples/debug_indexer.rs Normal file
View File

@@ -0,0 +1,96 @@
use std::{env, path::Path};
use brk_indexer::Indexer;
use brk_types::{Height, P2PKHAddressIndex, P2SHAddressIndex, TxOutIndex, TypeIndex};
use vecdb::GenericStoredVec;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
let indexer = Indexer::forced_import(&outputs_dir)?;
let reader_outputtype = indexer.vecs.txoutindex_to_outputtype.create_reader();
let reader_typeindex = indexer.vecs.txoutindex_to_typeindex.create_reader();
let reader_txindex = indexer.vecs.txoutindex_to_txindex.create_reader();
let reader_txid = indexer.vecs.txindex_to_txid.create_reader();
let reader_height_to_first_txoutindex = indexer.vecs.height_to_first_txoutindex.create_reader();
let reader_p2pkh = indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.create_reader();
let reader_p2sh = indexer.vecs.p2shaddressindex_to_p2shbytes.create_reader();
// Check what's stored at typeindex 254909199 in both P2PKH and P2SH vecs
let typeindex = TypeIndex::from(254909199_usize);
let p2pkh_bytes = indexer
.vecs
.p2pkhaddressindex_to_p2pkhbytes
.read(P2PKHAddressIndex::from(typeindex), &reader_p2pkh);
println!("P2PKH at typeindex 254909199: {:?}", p2pkh_bytes);
let p2sh_bytes = indexer
.vecs
.p2shaddressindex_to_p2shbytes
.read(P2SHAddressIndex::from(typeindex), &reader_p2sh);
println!("P2SH at typeindex 254909199: {:?}", p2sh_bytes);
// Check first P2SH index at height 476152
let reader_first_p2sh = indexer.vecs.height_to_first_p2shaddressindex.create_reader();
let reader_first_p2pkh = indexer.vecs.height_to_first_p2pkhaddressindex.create_reader();
let first_p2sh_at_476152 = indexer.vecs.height_to_first_p2shaddressindex.read(Height::from(476152_usize), &reader_first_p2sh);
let first_p2pkh_at_476152 = indexer.vecs.height_to_first_p2pkhaddressindex.read(Height::from(476152_usize), &reader_first_p2pkh);
println!("First P2SH index at height 476152: {:?}", first_p2sh_at_476152);
println!("First P2PKH index at height 476152: {:?}", first_p2pkh_at_476152);
// Check the problematic txoutindexes found during debugging
for txoutindex_usize in [653399433_usize, 653399443_usize] {
let txoutindex = TxOutIndex::from(txoutindex_usize);
let outputtype = indexer
.vecs
.txoutindex_to_outputtype
.read(txoutindex, &reader_outputtype)
.unwrap();
let typeindex = indexer
.vecs
.txoutindex_to_typeindex
.read(txoutindex, &reader_typeindex)
.unwrap();
let txindex = indexer
.vecs
.txoutindex_to_txindex
.read(txoutindex, &reader_txindex)
.unwrap();
let txid = indexer
.vecs
.txindex_to_txid
.read(txindex, &reader_txid)
.unwrap();
// Find height by binary search
let mut height = Height::from(0_usize);
for h in 0..900_000_usize {
let first_txoutindex = indexer
.vecs
.height_to_first_txoutindex
.read(Height::from(h), &reader_height_to_first_txoutindex);
if let Ok(first) = first_txoutindex {
if usize::from(first) > txoutindex_usize {
break;
}
height = Height::from(h);
}
}
println!(
"txoutindex={}, outputtype={:?}, typeindex={}, txindex={}, txid={}, height={}",
txoutindex_usize,
outputtype,
usize::from(typeindex),
usize::from(txindex),
txid,
usize::from(height)
);
}
Ok(())
}

20
crates/brk_computer/src/stateful/address_indexes.rs
View File

@@ -6,7 +6,7 @@ use brk_types::{
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TypeIndex,
};
use vecdb::{AnyStoredVec, AnyVec, BytesVec, GenericStoredVec, Reader, Stamp};
use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp};
#[derive(Clone, Traversable)]
pub struct AnyAddressIndexesVecs {
@@ -99,24 +99,6 @@ impl AnyAddressIndexesVecs {
typeindex: TypeIndex,
anyaddressindex: AnyAddressIndex,
) -> Result<()> {
let vec_len = match address_type {
OutputType::P2PK33 => self.p2pk33.len(),
OutputType::P2PK65 => self.p2pk65.len(),
OutputType::P2PKH => self.p2pkh.len(),
OutputType::P2SH => self.p2sh.len(),
OutputType::P2TR => self.p2tr.len(),
OutputType::P2WPKH => self.p2wpkh.len(),
OutputType::P2WSH => self.p2wsh.len(),
OutputType::P2A => self.p2a.len(),
_ => unreachable!(),
};
let typeindex_usize: usize = typeindex.into();
if typeindex_usize > vec_len {
eprintln!(
"DEBUG update_or_push: address_type={:?}, typeindex={}, vec_len={}, anyaddressindex={:?}",
address_type, typeindex_usize, vec_len, anyaddressindex
);
}
(match address_type {
OutputType::P2PK33 => self
.p2pk33

109
crates/brk_computer/src/stateful/mod.rs
View File

@@ -147,11 +147,7 @@ impl Vecs {
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&db,
"unspendable_supply",
v0,
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
&db,
"unspendable_supply",
@@ -161,11 +157,7 @@ impl Vecs {
compute_dollars,
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
&db,
"opreturn_supply",
v0,
)?,
height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
&db,
"opreturn_supply",
@@ -219,12 +211,20 @@ impl Vecs {
}),
addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::from(
ByAddressType::new_with_name(|name| {
Ok(EagerVec::forced_import(&db, &format!("{name}_addr_count"), v0)?)
Ok(EagerVec::forced_import(
&db,
&format!("{name}_addr_count"),
v0,
)?)
})?,
),
addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount::from(
ByAddressType::new_with_name(|name| {
Ok(EagerVec::forced_import(&db, &format!("{name}_empty_addr_count"), v0)?)
Ok(EagerVec::forced_import(
&db,
&format!("{name}_empty_addr_count"),
v0,
)?)
})?,
),
addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::from(
@@ -470,7 +470,6 @@ impl Vecs {
let starting_height = starting_indexes.height.min(stateful_starting_height);
let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp());
if starting_height <= last_height {
let stamp = starting_height.into();
let starting_height = if starting_height.is_not_zero() {
let mut set = [self.chain_state.rollback_before(stamp)?]
@@ -756,12 +755,12 @@ impl Vecs {
let typeindex = txoutindex_to_typeindex
.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
let typeindex_usize: usize = typeindex.into();
if output_type == OutputType::P2SH && typeindex_usize > 100_000_000 {
let txoutindex_usize: usize = txoutindex.into();
// Debug: track the exact bad typeindex
let ti: usize = typeindex.into();
if ti == 254909199 {
eprintln!(
"DEBUG P2SH bad typeindex at read: txoutindex={}, typeindex={}, txindex={}",
txoutindex_usize, typeindex_usize, txindex.to_usize()
"DEBUG outputs EXACT: output_type={:?}, txoutindex={}, typeindex={}",
output_type, txoutindex.to_usize(), ti
);
}
@@ -796,8 +795,15 @@ impl Vecs {
transacted.iterate(value, output_type);
if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt {
let ti: usize = typeindex.into();
if ti == 254909199 {
eprintln!("DEBUG fold outputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some());
}
if let Some(addressdata) = addressdata_opt
{
if ti == 254909199 {
eprintln!("DEBUG fold inserting to addressdatawithsource EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new());
}
addresstype_to_typeindex_to_addressdatawithsource
.insert_for_type(output_type, typeindex, addressdata);
}
@@ -868,6 +874,15 @@ impl Vecs {
let typeindex = txoutindex_to_typeindex
.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
// Debug: track the exact bad typeindex
let ti: usize = typeindex.into();
if ti == 254909199 {
eprintln!(
"DEBUG inputs EXACT: input_type={:?}, txoutindex={}, typeindex={}",
input_type, txoutindex.to_usize(), ti
);
}
let addressdata_opt = Self::get_addressdatawithsource(
input_type,
typeindex,
@@ -917,7 +932,14 @@ impl Vecs {
if let Some((typeindex, addressdata_opt)) =
typeindex_with_addressdata_opt
{
let ti: usize = typeindex.into();
if ti == 254909199 {
eprintln!("DEBUG fold inputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some());
}
if let Some(addressdata) = addressdata_opt {
if ti == 254909199 {
eprintln!("DEBUG fold inputs inserting EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new());
}
addresstype_to_typeindex_to_addressdatawithsource
.insert_for_type(output_type, typeindex, addressdata);
}
@@ -1383,15 +1405,15 @@ impl Vecs {
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> Option<WithAddressDataSource<LoadedAddressData>> {
let typeindex_usize: usize = typeindex.into();
if typeindex_usize == 254909199 {
eprintln!("DEBUG get_addressdatawithsource EXACT: address_type={:?}, typeindex={}", address_type, typeindex_usize);
}
let first = *first_addressindexes.get(address_type).unwrap();
if first <= typeindex {
let typeindex_usize: usize = typeindex.into();
let first_usize: usize = first.into();
if typeindex_usize > 100_000_000 {
eprintln!(
"DEBUG get_addressdatawithsource NEW: address_type={:?}, typeindex={}, first_addressindex={}",
address_type, typeindex_usize, first_usize
);
if typeindex_usize == 254909199 {
eprintln!("DEBUG get_addressdatawithsource returning New EXACT: address_type={:?}, first={}", address_type, first_usize);
}
return Some(WithAddressDataSource::New(LoadedAddressData::default()));
}
@@ -1478,13 +1500,6 @@ impl Vecs {
addresstype_to_typeindex_to_emptyaddressdata.into_sorted_iter()
{
for (typeindex, emptyaddressdata_with_source) in sorted.into_iter() {
let typeindex_usize: usize = typeindex.into();
if typeindex_usize > 100_000_000 {
eprintln!(
"DEBUG emptyaddressdata: address_type={:?}, typeindex={}, variant={:?}",
address_type, typeindex_usize, std::mem::discriminant(&emptyaddressdata_with_source)
);
}
match emptyaddressdata_with_source {
WithAddressDataSource::New(emptyaddressdata) => {
let emptyaddressindex = self
@@ -1494,6 +1509,10 @@ impl Vecs {
let anyaddressindex = AnyAddressIndex::from(emptyaddressindex);
let ti: usize = typeindex.into();
if address_type == OutputType::P2SH && ti > 30_000_000 {
eprintln!("DEBUG insert1 (empty New): P2SH typeindex={}", ti);
}
addresstype_to_typeindex_to_new_or_updated_anyaddressindex
.get_mut(address_type)
.unwrap()
@@ -1519,6 +1538,10 @@ impl Vecs {
let anyaddressindex = emptyaddressindex.into();
let ti: usize = typeindex.into();
if address_type == OutputType::P2SH && ti > 30_000_000 {
eprintln!("DEBUG insert2 (empty FromLoaded): P2SH typeindex={}", ti);
}
addresstype_to_typeindex_to_new_or_updated_anyaddressindex
.get_mut(address_type)
.unwrap()
@@ -1532,13 +1555,6 @@ impl Vecs {
addresstype_to_typeindex_to_loadedaddressdata.into_sorted_iter()
{
for (typeindex, loadedaddressdata_with_source) in sorted.into_iter() {
let typeindex_usize: usize = typeindex.into();
if typeindex_usize > 100_000_000 {
eprintln!(
"DEBUG loadedaddressdata: address_type={:?}, typeindex={}, variant={:?}",
address_type, typeindex_usize, std::mem::discriminant(&loadedaddressdata_with_source)
);
}
match loadedaddressdata_with_source {
WithAddressDataSource::New(loadedaddressdata) => {
let loadedaddressindex = self
@@ -1548,6 +1564,10 @@ impl Vecs {
let anyaddressindex = AnyAddressIndex::from(loadedaddressindex);
let ti: usize = typeindex.into();
if address_type == OutputType::P2SH && ti > 30_000_000 {
eprintln!("DEBUG insert3 (loaded New): P2SH typeindex={}", ti);
}
addresstype_to_typeindex_to_new_or_updated_anyaddressindex
.get_mut(address_type)
.unwrap()
@@ -1573,6 +1593,10 @@ impl Vecs {
let anyaddressindex = loadedaddressindex.into();
let ti: usize = typeindex.into();
if address_type == OutputType::P2SH && ti > 30_000_000 {
eprintln!("DEBUG insert4 (loaded FromEmpty): P2SH typeindex={}", ti);
}
addresstype_to_typeindex_to_new_or_updated_anyaddressindex
.get_mut(address_type)
.unwrap()
@@ -1586,6 +1610,14 @@ impl Vecs {
addresstype_to_typeindex_to_new_or_updated_anyaddressindex.into_sorted_iter()
{
for (typeindex, anyaddressindex) in sorted {
// Debug: log right before the call that fails
let typeindex_usize: usize = typeindex.into();
if address_type == OutputType::P2SH && typeindex_usize > 30_000_000 {
eprintln!(
"DEBUG flush update_or_push: address_type={:?}, typeindex={}, anyaddressindex={:?}",
address_type, typeindex_usize, anyaddressindex
);
}
self.any_address_indexes.update_or_push(
address_type,
typeindex,
@@ -1611,4 +1643,3 @@ impl Vecs {
Ok(())
}
}

72
crates/brk_computer/src/stateful/transaction_processing.rs
View File

@@ -1,16 +1,15 @@
use brk_error::Result;
use brk_grouper::{ByAddressType, Filtered};
use brk_types::{CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex};
use brk_types::{
CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex,
};
use vecdb::VecIndex;
use crate::utils::OptionExt;
use super::{
address_cohorts,
addresstype::{
AddressTypeToTypeIndexMap, AddressTypeToVec,
HeightToAddressTypeToVec,
},
addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec},
withaddressdatasource::WithAddressDataSource,
};
@@ -34,6 +33,11 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
) {
self.unwrap().into_iter().for_each(|(_type, vec)| {
vec.into_iter().for_each(|(type_index, value)| {
let type_index_usize: usize = type_index.into();
if type_index_usize == 254909199 {
eprintln!("DEBUG process_received EXACT: _type={:?}, type_index={}", _type, type_index_usize);
}
let mut is_new = false;
let mut from_any_empty = false;
@@ -48,12 +52,18 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
.remove(&type_index)
.map(|ad| {
from_any_empty = true;
if type_index_usize == 254909199 {
eprintln!("DEBUG process_received from_empty EXACT: _type={:?}, is_new={}", _type, ad.is_new());
}
ad.into()
})
.unwrap_or_else(|| {
let addressdata =
stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
.remove_for_type(_type, &type_index);
if type_index_usize == 254909199 {
eprintln!("DEBUG process_received from_stored_or_new EXACT: _type={:?}, is_new={}", _type, addressdata.is_new());
}
is_new = addressdata.is_new();
from_any_empty = addressdata.is_from_emptyaddressdata();
addressdata
@@ -73,14 +83,15 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
let amount = prev_amount + value;
let filters_differ =
vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter();
let filters_differ = vecs.amount_range.get(amount).filter()
!= vecs.amount_range.get(prev_amount).filter();
if is_new || from_any_empty || filters_differ {
if !is_new && !from_any_empty {
vecs.amount_range
.get_mut(prev_amount)
.state.um()
.state
.um()
.subtract(addressdata);
}
@@ -88,12 +99,14 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
vecs.amount_range
.get_mut(amount)
.state.um()
.state
.um()
.add(addressdata);
} else {
vecs.amount_range
.get_mut(amount)
.state.um()
.state
.um()
.receive(addressdata, value, price);
}
});
@@ -143,6 +156,11 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
v.unwrap().into_iter().try_for_each(|(_type, vec)| {
vec.into_iter().try_for_each(|(type_index, value)| {
let type_index_usize: usize = type_index.into();
if type_index_usize == 254909199 {
eprintln!("DEBUG process_sent EXACT: _type={:?}, type_index={}", _type, type_index_usize);
}
let typeindex_to_loadedaddressdata =
addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type);
@@ -161,13 +179,14 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
let will_be_empty = addressdata.has_1_utxos();
let filters_differ =
vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter();
let filters_differ = vecs.amount_range.get(amount).filter()
!= vecs.amount_range.get(prev_amount).filter();
if will_be_empty || filters_differ {
vecs.amount_range
.get_mut(prev_amount)
.state.um()
.state
.um()
.subtract(addressdata);
addressdata.send(value, prev_price)?;
@@ -183,6 +202,9 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
let addressdata =
typeindex_to_loadedaddressdata.remove(&type_index).unwrap();
if type_index_usize == 254909199 {
eprintln!("DEBUG process_sent will_be_empty EXACT: _type={:?}", _type);
}
addresstype_to_typeindex_to_emptyaddressdata
.get_mut(_type)
.unwrap()
@@ -190,22 +212,20 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
} else {
vecs.amount_range
.get_mut(amount)
.state.um()
.state
.um()
.add(addressdata);
}
} else {
vecs.amount_range
.get_mut(amount)
.state.um()
.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
vecs.amount_range.get_mut(amount).state.um().send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
}
Ok(())