From dc2fa233ab275601fa7d28586daef38b1a93a3ce Mon Sep 17 00:00:00 2001 From: nym21 Date: Thu, 4 Dec 2025 21:30:08 +0100 Subject: [PATCH] indexer: fix bug --- crates/brk_computer/examples/debug_indexer.rs | 96 +++++++++++++++ .../src/stateful/address_indexes.rs | 20 +--- crates/brk_computer/src/stateful/mod.rs | 109 +++++++++++------- .../src/stateful/transaction_processing.rs | 72 +++++++----- 4 files changed, 213 insertions(+), 84 deletions(-) create mode 100644 crates/brk_computer/examples/debug_indexer.rs diff --git a/crates/brk_computer/examples/debug_indexer.rs b/crates/brk_computer/examples/debug_indexer.rs new file mode 100644 index 000000000..1e551a069 --- /dev/null +++ b/crates/brk_computer/examples/debug_indexer.rs @@ -0,0 +1,96 @@ +use std::{env, path::Path}; + +use brk_indexer::Indexer; +use brk_types::{Height, P2PKHAddressIndex, P2SHAddressIndex, TxOutIndex, TypeIndex}; +use vecdb::GenericStoredVec; + +fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + + let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk"); + + let indexer = Indexer::forced_import(&outputs_dir)?; + + let reader_outputtype = indexer.vecs.txoutindex_to_outputtype.create_reader(); + let reader_typeindex = indexer.vecs.txoutindex_to_typeindex.create_reader(); + let reader_txindex = indexer.vecs.txoutindex_to_txindex.create_reader(); + let reader_txid = indexer.vecs.txindex_to_txid.create_reader(); + let reader_height_to_first_txoutindex = indexer.vecs.height_to_first_txoutindex.create_reader(); + let reader_p2pkh = indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.create_reader(); + let reader_p2sh = indexer.vecs.p2shaddressindex_to_p2shbytes.create_reader(); + + // Check what's stored at typeindex 254909199 in both P2PKH and P2SH vecs + let typeindex = TypeIndex::from(254909199_usize); + + let p2pkh_bytes = indexer + .vecs + .p2pkhaddressindex_to_p2pkhbytes + .read(P2PKHAddressIndex::from(typeindex), &reader_p2pkh); + println!("P2PKH at typeindex 254909199: {:?}", p2pkh_bytes); + + let p2sh_bytes = indexer + .vecs + .p2shaddressindex_to_p2shbytes + .read(P2SHAddressIndex::from(typeindex), &reader_p2sh); + println!("P2SH at typeindex 254909199: {:?}", p2sh_bytes); + + // Check first P2SH index at height 476152 + let reader_first_p2sh = indexer.vecs.height_to_first_p2shaddressindex.create_reader(); + let reader_first_p2pkh = indexer.vecs.height_to_first_p2pkhaddressindex.create_reader(); + let first_p2sh_at_476152 = indexer.vecs.height_to_first_p2shaddressindex.read(Height::from(476152_usize), &reader_first_p2sh); + let first_p2pkh_at_476152 = indexer.vecs.height_to_first_p2pkhaddressindex.read(Height::from(476152_usize), &reader_first_p2pkh); + println!("First P2SH index at height 476152: {:?}", first_p2sh_at_476152); + println!("First P2PKH index at height 476152: {:?}", first_p2pkh_at_476152); + + // Check the problematic txoutindexes found during debugging + for txoutindex_usize in [653399433_usize, 653399443_usize] { + let txoutindex = TxOutIndex::from(txoutindex_usize); + let outputtype = indexer + .vecs + .txoutindex_to_outputtype + .read(txoutindex, &reader_outputtype) + .unwrap(); + let typeindex = indexer + .vecs + .txoutindex_to_typeindex + .read(txoutindex, &reader_typeindex) + .unwrap(); + let txindex = indexer + .vecs + .txoutindex_to_txindex + .read(txoutindex, &reader_txindex) + .unwrap(); + let txid = indexer + .vecs + .txindex_to_txid + .read(txindex, &reader_txid) + .unwrap(); + + // Find height by binary search + let mut height = Height::from(0_usize); + for h in 0..900_000_usize { + let first_txoutindex = indexer + .vecs + .height_to_first_txoutindex + .read(Height::from(h), &reader_height_to_first_txoutindex); + if let Ok(first) = first_txoutindex { + if usize::from(first) > txoutindex_usize { + break; + } + height = Height::from(h); + } + } + + println!( + "txoutindex={}, outputtype={:?}, typeindex={}, txindex={}, txid={}, height={}", + txoutindex_usize, + outputtype, + usize::from(typeindex), + usize::from(txindex), + txid, + usize::from(height) + ); + } + + Ok(()) +} diff --git a/crates/brk_computer/src/stateful/address_indexes.rs b/crates/brk_computer/src/stateful/address_indexes.rs index 6541fb28e..1206c9f67 100644 --- a/crates/brk_computer/src/stateful/address_indexes.rs +++ b/crates/brk_computer/src/stateful/address_indexes.rs @@ -6,7 +6,7 @@ use brk_types::{ P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, TypeIndex, }; -use vecdb::{AnyStoredVec, AnyVec, BytesVec, GenericStoredVec, Reader, Stamp}; +use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp}; #[derive(Clone, Traversable)] pub struct AnyAddressIndexesVecs { @@ -99,24 +99,6 @@ impl AnyAddressIndexesVecs { typeindex: TypeIndex, anyaddressindex: AnyAddressIndex, ) -> Result<()> { - let vec_len = match address_type { - OutputType::P2PK33 => self.p2pk33.len(), - OutputType::P2PK65 => self.p2pk65.len(), - OutputType::P2PKH => self.p2pkh.len(), - OutputType::P2SH => self.p2sh.len(), - OutputType::P2TR => self.p2tr.len(), - OutputType::P2WPKH => self.p2wpkh.len(), - OutputType::P2WSH => self.p2wsh.len(), - OutputType::P2A => self.p2a.len(), - _ => unreachable!(), - }; - let typeindex_usize: usize = typeindex.into(); - if typeindex_usize > vec_len { - eprintln!( - "DEBUG update_or_push: address_type={:?}, typeindex={}, vec_len={}, anyaddressindex={:?}", - address_type, typeindex_usize, vec_len, anyaddressindex - ); - } (match address_type { OutputType::P2PK33 => self .p2pk33 diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 95a7565b1..208fe380d 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -147,11 +147,7 @@ impl Vecs { .with_saved_stamped_changes(SAVED_STAMPED_CHANGES), )?, - height_to_unspendable_supply: EagerVec::forced_import( - &db, - "unspendable_supply", - v0, - )?, + height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( &db, "unspendable_supply", @@ -161,11 +157,7 @@ impl Vecs { compute_dollars, indexes, )?, - height_to_opreturn_supply: EagerVec::forced_import( - &db, - "opreturn_supply", - v0, - )?, + height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?, indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( &db, "opreturn_supply", @@ -219,12 +211,20 @@ impl Vecs { }), addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::from( ByAddressType::new_with_name(|name| { - Ok(EagerVec::forced_import(&db, &format!("{name}_addr_count"), v0)?) + Ok(EagerVec::forced_import( + &db, + &format!("{name}_addr_count"), + v0, + )?) })?, ), addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount::from( ByAddressType::new_with_name(|name| { - Ok(EagerVec::forced_import(&db, &format!("{name}_empty_addr_count"), v0)?) + Ok(EagerVec::forced_import( + &db, + &format!("{name}_empty_addr_count"), + v0, + )?) })?, ), addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::from( @@ -470,7 +470,6 @@ impl Vecs { let starting_height = starting_indexes.height.min(stateful_starting_height); let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp()); if starting_height <= last_height { - let stamp = starting_height.into(); let starting_height = if starting_height.is_not_zero() { let mut set = [self.chain_state.rollback_before(stamp)?] @@ -756,12 +755,12 @@ impl Vecs { let typeindex = txoutindex_to_typeindex .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); - let typeindex_usize: usize = typeindex.into(); - if output_type == OutputType::P2SH && typeindex_usize > 100_000_000 { - let txoutindex_usize: usize = txoutindex.into(); + // Debug: track the exact bad typeindex + let ti: usize = typeindex.into(); + if ti == 254909199 { eprintln!( - "DEBUG P2SH bad typeindex at read: txoutindex={}, typeindex={}, txindex={}", - txoutindex_usize, typeindex_usize, txindex.to_usize() + "DEBUG outputs EXACT: output_type={:?}, txoutindex={}, typeindex={}", + output_type, txoutindex.to_usize(), ti ); } @@ -796,8 +795,15 @@ impl Vecs { transacted.iterate(value, output_type); if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt { + let ti: usize = typeindex.into(); + if ti == 254909199 { + eprintln!("DEBUG fold outputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some()); + } if let Some(addressdata) = addressdata_opt { + if ti == 254909199 { + eprintln!("DEBUG fold inserting to addressdatawithsource EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new()); + } addresstype_to_typeindex_to_addressdatawithsource .insert_for_type(output_type, typeindex, addressdata); } @@ -868,6 +874,15 @@ impl Vecs { let typeindex = txoutindex_to_typeindex .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex); + // Debug: track the exact bad typeindex + let ti: usize = typeindex.into(); + if ti == 254909199 { + eprintln!( + "DEBUG inputs EXACT: input_type={:?}, txoutindex={}, typeindex={}", + input_type, txoutindex.to_usize(), ti + ); + } + let addressdata_opt = Self::get_addressdatawithsource( input_type, typeindex, @@ -917,7 +932,14 @@ impl Vecs { if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt { + let ti: usize = typeindex.into(); + if ti == 254909199 { + eprintln!("DEBUG fold inputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some()); + } if let Some(addressdata) = addressdata_opt { + if ti == 254909199 { + eprintln!("DEBUG fold inputs inserting EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new()); + } addresstype_to_typeindex_to_addressdatawithsource .insert_for_type(output_type, typeindex, addressdata); } @@ -1383,15 +1405,15 @@ impl Vecs { any_address_indexes: &AnyAddressIndexesVecs, addresses_data: &AddressesDataVecs, ) -> Option> { + let typeindex_usize: usize = typeindex.into(); + if typeindex_usize == 254909199 { + eprintln!("DEBUG get_addressdatawithsource EXACT: address_type={:?}, typeindex={}", address_type, typeindex_usize); + } let first = *first_addressindexes.get(address_type).unwrap(); if first <= typeindex { - let typeindex_usize: usize = typeindex.into(); let first_usize: usize = first.into(); - if typeindex_usize > 100_000_000 { - eprintln!( - "DEBUG get_addressdatawithsource NEW: address_type={:?}, typeindex={}, first_addressindex={}", - address_type, typeindex_usize, first_usize - ); + if typeindex_usize == 254909199 { + eprintln!("DEBUG get_addressdatawithsource returning New EXACT: address_type={:?}, first={}", address_type, first_usize); } return Some(WithAddressDataSource::New(LoadedAddressData::default())); } @@ -1478,13 +1500,6 @@ impl Vecs { addresstype_to_typeindex_to_emptyaddressdata.into_sorted_iter() { for (typeindex, emptyaddressdata_with_source) in sorted.into_iter() { - let typeindex_usize: usize = typeindex.into(); - if typeindex_usize > 100_000_000 { - eprintln!( - "DEBUG emptyaddressdata: address_type={:?}, typeindex={}, variant={:?}", - address_type, typeindex_usize, std::mem::discriminant(&emptyaddressdata_with_source) - ); - } match emptyaddressdata_with_source { WithAddressDataSource::New(emptyaddressdata) => { let emptyaddressindex = self @@ -1494,6 +1509,10 @@ impl Vecs { let anyaddressindex = AnyAddressIndex::from(emptyaddressindex); + let ti: usize = typeindex.into(); + if address_type == OutputType::P2SH && ti > 30_000_000 { + eprintln!("DEBUG insert1 (empty New): P2SH typeindex={}", ti); + } addresstype_to_typeindex_to_new_or_updated_anyaddressindex .get_mut(address_type) .unwrap() @@ -1519,6 +1538,10 @@ impl Vecs { let anyaddressindex = emptyaddressindex.into(); + let ti: usize = typeindex.into(); + if address_type == OutputType::P2SH && ti > 30_000_000 { + eprintln!("DEBUG insert2 (empty FromLoaded): P2SH typeindex={}", ti); + } addresstype_to_typeindex_to_new_or_updated_anyaddressindex .get_mut(address_type) .unwrap() @@ -1532,13 +1555,6 @@ impl Vecs { addresstype_to_typeindex_to_loadedaddressdata.into_sorted_iter() { for (typeindex, loadedaddressdata_with_source) in sorted.into_iter() { - let typeindex_usize: usize = typeindex.into(); - if typeindex_usize > 100_000_000 { - eprintln!( - "DEBUG loadedaddressdata: address_type={:?}, typeindex={}, variant={:?}", - address_type, typeindex_usize, std::mem::discriminant(&loadedaddressdata_with_source) - ); - } match loadedaddressdata_with_source { WithAddressDataSource::New(loadedaddressdata) => { let loadedaddressindex = self @@ -1548,6 +1564,10 @@ impl Vecs { let anyaddressindex = AnyAddressIndex::from(loadedaddressindex); + let ti: usize = typeindex.into(); + if address_type == OutputType::P2SH && ti > 30_000_000 { + eprintln!("DEBUG insert3 (loaded New): P2SH typeindex={}", ti); + } addresstype_to_typeindex_to_new_or_updated_anyaddressindex .get_mut(address_type) .unwrap() @@ -1573,6 +1593,10 @@ impl Vecs { let anyaddressindex = loadedaddressindex.into(); + let ti: usize = typeindex.into(); + if address_type == OutputType::P2SH && ti > 30_000_000 { + eprintln!("DEBUG insert4 (loaded FromEmpty): P2SH typeindex={}", ti); + } addresstype_to_typeindex_to_new_or_updated_anyaddressindex .get_mut(address_type) .unwrap() @@ -1586,6 +1610,14 @@ impl Vecs { addresstype_to_typeindex_to_new_or_updated_anyaddressindex.into_sorted_iter() { for (typeindex, anyaddressindex) in sorted { + // Debug: log right before the call that fails + let typeindex_usize: usize = typeindex.into(); + if address_type == OutputType::P2SH && typeindex_usize > 30_000_000 { + eprintln!( + "DEBUG flush update_or_push: address_type={:?}, typeindex={}, anyaddressindex={:?}", + address_type, typeindex_usize, anyaddressindex + ); + } self.any_address_indexes.update_or_push( address_type, typeindex, @@ -1611,4 +1643,3 @@ impl Vecs { Ok(()) } } - diff --git a/crates/brk_computer/src/stateful/transaction_processing.rs b/crates/brk_computer/src/stateful/transaction_processing.rs index b95716ccb..380bc7e34 100644 --- a/crates/brk_computer/src/stateful/transaction_processing.rs +++ b/crates/brk_computer/src/stateful/transaction_processing.rs @@ -1,16 +1,15 @@ use brk_error::Result; use brk_grouper::{ByAddressType, Filtered}; -use brk_types::{CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex}; +use brk_types::{ + CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex, +}; use vecdb::VecIndex; use crate::utils::OptionExt; use super::{ address_cohorts, - addresstype::{ - AddressTypeToTypeIndexMap, AddressTypeToVec, - HeightToAddressTypeToVec, - }, + addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec}, withaddressdatasource::WithAddressDataSource, }; @@ -34,6 +33,11 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { ) { self.unwrap().into_iter().for_each(|(_type, vec)| { vec.into_iter().for_each(|(type_index, value)| { + let type_index_usize: usize = type_index.into(); + if type_index_usize == 254909199 { + eprintln!("DEBUG process_received EXACT: _type={:?}, type_index={}", _type, type_index_usize); + } + let mut is_new = false; let mut from_any_empty = false; @@ -48,12 +52,18 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { .remove(&type_index) .map(|ad| { from_any_empty = true; + if type_index_usize == 254909199 { + eprintln!("DEBUG process_received from_empty EXACT: _type={:?}, is_new={}", _type, ad.is_new()); + } ad.into() }) .unwrap_or_else(|| { let addressdata = stored_or_new_addresstype_to_typeindex_to_addressdatawithsource .remove_for_type(_type, &type_index); + if type_index_usize == 254909199 { + eprintln!("DEBUG process_received from_stored_or_new EXACT: _type={:?}, is_new={}", _type, addressdata.is_new()); + } is_new = addressdata.is_new(); from_any_empty = addressdata.is_from_emptyaddressdata(); addressdata @@ -73,14 +83,15 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { let amount = prev_amount + value; - let filters_differ = - vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter(); + let filters_differ = vecs.amount_range.get(amount).filter() + != vecs.amount_range.get(prev_amount).filter(); if is_new || from_any_empty || filters_differ { if !is_new && !from_any_empty { vecs.amount_range .get_mut(prev_amount) - .state.um() + .state + .um() .subtract(addressdata); } @@ -88,12 +99,14 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { vecs.amount_range .get_mut(amount) - .state.um() + .state + .um() .add(addressdata); } else { vecs.amount_range .get_mut(amount) - .state.um() + .state + .um() .receive(addressdata, value, price); } }); @@ -143,6 +156,11 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { v.unwrap().into_iter().try_for_each(|(_type, vec)| { vec.into_iter().try_for_each(|(type_index, value)| { + let type_index_usize: usize = type_index.into(); + if type_index_usize == 254909199 { + eprintln!("DEBUG process_sent EXACT: _type={:?}, type_index={}", _type, type_index_usize); + } + let typeindex_to_loadedaddressdata = addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type); @@ -161,13 +179,14 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { let will_be_empty = addressdata.has_1_utxos(); - let filters_differ = - vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter(); + let filters_differ = vecs.amount_range.get(amount).filter() + != vecs.amount_range.get(prev_amount).filter(); if will_be_empty || filters_differ { vecs.amount_range .get_mut(prev_amount) - .state.um() + .state + .um() .subtract(addressdata); addressdata.send(value, prev_price)?; @@ -183,6 +202,9 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { let addressdata = typeindex_to_loadedaddressdata.remove(&type_index).unwrap(); + if type_index_usize == 254909199 { + eprintln!("DEBUG process_sent will_be_empty EXACT: _type={:?}", _type); + } addresstype_to_typeindex_to_emptyaddressdata .get_mut(_type) .unwrap() @@ -190,22 +212,20 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { } else { vecs.amount_range .get_mut(amount) - .state.um() + .state + .um() .add(addressdata); } } else { - vecs.amount_range - .get_mut(amount) - .state.um() - .send( - addressdata, - value, - price, - prev_price, - blocks_old, - days_old, - older_than_hour, - )?; + vecs.amount_range.get_mut(amount).state.um().send( + addressdata, + value, + price, + prev_price, + blocks_old, + days_old, + older_than_hour, + )?; } Ok(())