indexer: fix bug

crates/brk_computer/examples/debug_indexer.rs

@@ -0,0 +1,96 @@
+use std::{env, path::Path};
+
+use brk_indexer::Indexer;
+use brk_types::{Height, P2PKHAddressIndex, P2SHAddressIndex, TxOutIndex, TypeIndex};
+use vecdb::GenericStoredVec;
+
+fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
+
+    let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
+
+    let indexer = Indexer::forced_import(&outputs_dir)?;
+
+    let reader_outputtype = indexer.vecs.txoutindex_to_outputtype.create_reader();
+    let reader_typeindex = indexer.vecs.txoutindex_to_typeindex.create_reader();
+    let reader_txindex = indexer.vecs.txoutindex_to_txindex.create_reader();
+    let reader_txid = indexer.vecs.txindex_to_txid.create_reader();
+    let reader_height_to_first_txoutindex = indexer.vecs.height_to_first_txoutindex.create_reader();
+    let reader_p2pkh = indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.create_reader();
+    let reader_p2sh = indexer.vecs.p2shaddressindex_to_p2shbytes.create_reader();
+
+    // Check what's stored at typeindex 254909199 in both P2PKH and P2SH vecs
+    let typeindex = TypeIndex::from(254909199_usize);
+
+    let p2pkh_bytes = indexer
+        .vecs
+        .p2pkhaddressindex_to_p2pkhbytes
+        .read(P2PKHAddressIndex::from(typeindex), &reader_p2pkh);
+    println!("P2PKH at typeindex 254909199: {:?}", p2pkh_bytes);
+
+    let p2sh_bytes = indexer
+        .vecs
+        .p2shaddressindex_to_p2shbytes
+        .read(P2SHAddressIndex::from(typeindex), &reader_p2sh);
+    println!("P2SH at typeindex 254909199: {:?}", p2sh_bytes);
+
+    // Check first P2SH index at height 476152
+    let reader_first_p2sh = indexer.vecs.height_to_first_p2shaddressindex.create_reader();
+    let reader_first_p2pkh = indexer.vecs.height_to_first_p2pkhaddressindex.create_reader();
+    let first_p2sh_at_476152 = indexer.vecs.height_to_first_p2shaddressindex.read(Height::from(476152_usize), &reader_first_p2sh);
+    let first_p2pkh_at_476152 = indexer.vecs.height_to_first_p2pkhaddressindex.read(Height::from(476152_usize), &reader_first_p2pkh);
+    println!("First P2SH index at height 476152: {:?}", first_p2sh_at_476152);
+    println!("First P2PKH index at height 476152: {:?}", first_p2pkh_at_476152);
+
+    // Check the problematic txoutindexes found during debugging
+    for txoutindex_usize in [653399433_usize, 653399443_usize] {
+        let txoutindex = TxOutIndex::from(txoutindex_usize);
+        let outputtype = indexer
+            .vecs
+            .txoutindex_to_outputtype
+            .read(txoutindex, &reader_outputtype)
+            .unwrap();
+        let typeindex = indexer
+            .vecs
+            .txoutindex_to_typeindex
+            .read(txoutindex, &reader_typeindex)
+            .unwrap();
+        let txindex = indexer
+            .vecs
+            .txoutindex_to_txindex
+            .read(txoutindex, &reader_txindex)
+            .unwrap();
+        let txid = indexer
+            .vecs
+            .txindex_to_txid
+            .read(txindex, &reader_txid)
+            .unwrap();
+
+        // Find height by binary search
+        let mut height = Height::from(0_usize);
+        for h in 0..900_000_usize {
+            let first_txoutindex = indexer
+                .vecs
+                .height_to_first_txoutindex
+                .read(Height::from(h), &reader_height_to_first_txoutindex);
+            if let Ok(first) = first_txoutindex {
+                if usize::from(first) > txoutindex_usize {
+                    break;
+                }
+                height = Height::from(h);
+            }
+        }
+
+        println!(
+            "txoutindex={}, outputtype={:?}, typeindex={}, txindex={}, txid={}, height={}",
+            txoutindex_usize,
+            outputtype,
+            usize::from(typeindex),
+            usize::from(txindex),
+            txid,
+            usize::from(height)
+        );
+    }
+
+    Ok(())
+}

crates/brk_computer/src/stateful/address_indexes.rs

@@ -6,7 +6,7 @@ use brk_types::{
     P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
     TypeIndex,
 };
-use vecdb::{AnyStoredVec, AnyVec, BytesVec, GenericStoredVec, Reader, Stamp};
+use vecdb::{AnyStoredVec, BytesVec, GenericStoredVec, Reader, Stamp};
 
 #[derive(Clone, Traversable)]
 pub struct AnyAddressIndexesVecs {
@@ -99,24 +99,6 @@ impl AnyAddressIndexesVecs {
         typeindex: TypeIndex,
         anyaddressindex: AnyAddressIndex,
     ) -> Result<()> {
-        let vec_len = match address_type {
-            OutputType::P2PK33 => self.p2pk33.len(),
-            OutputType::P2PK65 => self.p2pk65.len(),
-            OutputType::P2PKH => self.p2pkh.len(),
-            OutputType::P2SH => self.p2sh.len(),
-            OutputType::P2TR => self.p2tr.len(),
-            OutputType::P2WPKH => self.p2wpkh.len(),
-            OutputType::P2WSH => self.p2wsh.len(),
-            OutputType::P2A => self.p2a.len(),
-            _ => unreachable!(),
-        };
-        let typeindex_usize: usize = typeindex.into();
-        if typeindex_usize > vec_len {
-            eprintln!(
-                "DEBUG update_or_push: address_type={:?}, typeindex={}, vec_len={}, anyaddressindex={:?}",
-                address_type, typeindex_usize, vec_len, anyaddressindex
-            );
-        }
         (match address_type {
             OutputType::P2PK33 => self
                 .p2pk33

crates/brk_computer/src/stateful/mod.rs

@@ -147,11 +147,7 @@ impl Vecs {
                     .with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
             )?,
 
-            height_to_unspendable_supply: EagerVec::forced_import(
+            height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
-                &db,
-                "unspendable_supply",
-                v0,
-            )?,
             indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
                 &db,
                 "unspendable_supply",
@@ -161,11 +157,7 @@ impl Vecs {
                 compute_dollars,
                 indexes,
             )?,
-            height_to_opreturn_supply: EagerVec::forced_import(
+            height_to_opreturn_supply: EagerVec::forced_import(&db, "opreturn_supply", v0)?,
-                &db,
-                "opreturn_supply",
-                v0,
-            )?,
             indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
                 &db,
                 "opreturn_supply",
@@ -219,12 +211,20 @@ impl Vecs {
             }),
             addresstype_to_height_to_addr_count: AddressTypeToHeightToAddressCount::from(
                 ByAddressType::new_with_name(|name| {
-                    Ok(EagerVec::forced_import(&db, &format!("{name}_addr_count"), v0)?)
+                    Ok(EagerVec::forced_import(
+                        &db,
+                        &format!("{name}_addr_count"),
+                        v0,
+                    )?)
                 })?,
             ),
             addresstype_to_height_to_empty_addr_count: AddressTypeToHeightToAddressCount::from(
                 ByAddressType::new_with_name(|name| {
-                    Ok(EagerVec::forced_import(&db, &format!("{name}_empty_addr_count"), v0)?)
+                    Ok(EagerVec::forced_import(
+                        &db,
+                        &format!("{name}_empty_addr_count"),
+                        v0,
+                    )?)
                 })?,
             ),
             addresstype_to_indexes_to_addr_count: AddressTypeToIndexesToAddressCount::from(
@@ -470,7 +470,6 @@ impl Vecs {
         let starting_height = starting_indexes.height.min(stateful_starting_height);
         let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp());
         if starting_height <= last_height {
-
             let stamp = starting_height.into();
             let starting_height = if starting_height.is_not_zero() {
                 let mut set = [self.chain_state.rollback_before(stamp)?]
@@ -756,12 +755,12 @@ impl Vecs {
                             let typeindex = txoutindex_to_typeindex
                                 .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
 
-                            let typeindex_usize: usize = typeindex.into();
+                            // Debug: track the exact bad typeindex
-                            if output_type == OutputType::P2SH && typeindex_usize > 100_000_000 {
+                            let ti: usize = typeindex.into();
-                                let txoutindex_usize: usize = txoutindex.into();
+                            if ti == 254909199 {
                                 eprintln!(
-                                    "DEBUG P2SH bad typeindex at read: txoutindex={}, typeindex={}, txindex={}",
+                                    "DEBUG outputs EXACT: output_type={:?}, txoutindex={}, typeindex={}",
-                                    txoutindex_usize, typeindex_usize, txindex.to_usize()
+                                    output_type, txoutindex.to_usize(), ti
                                 );
                             }
 
@@ -796,8 +795,15 @@ impl Vecs {
                             transacted.iterate(value, output_type);
 
                             if let Some((typeindex, addressdata_opt)) = typeindex_with_addressdata_opt {
+                                let ti: usize = typeindex.into();
+                                if ti == 254909199 {
+                                    eprintln!("DEBUG fold outputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some());
+                                }
                                 if let Some(addressdata) = addressdata_opt
                                 {
+                                    if ti == 254909199 {
+                                        eprintln!("DEBUG fold inserting to addressdatawithsource EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new());
+                                    }
                                     addresstype_to_typeindex_to_addressdatawithsource
                                         .insert_for_type(output_type, typeindex, addressdata);
                                 }
@@ -868,6 +874,15 @@ impl Vecs {
                             let typeindex = txoutindex_to_typeindex
                                 .read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
 
+                            // Debug: track the exact bad typeindex
+                            let ti: usize = typeindex.into();
+                            if ti == 254909199 {
+                                eprintln!(
+                                    "DEBUG inputs EXACT: input_type={:?}, txoutindex={}, typeindex={}",
+                                    input_type, txoutindex.to_usize(), ti
+                                );
+                            }
+
                             let addressdata_opt = Self::get_addressdatawithsource(
                                 input_type,
                                 typeindex,
@@ -917,7 +932,14 @@ impl Vecs {
                                 if let Some((typeindex, addressdata_opt)) =
                                     typeindex_with_addressdata_opt
                                 {
+                                    let ti: usize = typeindex.into();
+                                    if ti == 254909199 {
+                                        eprintln!("DEBUG fold inputs EXACT: output_type={:?}, typeindex={}, has_addressdata={}", output_type, ti, addressdata_opt.is_some());
+                                    }
                                     if let Some(addressdata) = addressdata_opt {
+                                        if ti == 254909199 {
+                                            eprintln!("DEBUG fold inputs inserting EXACT: output_type={:?}, is_new={}", output_type, addressdata.is_new());
+                                        }
                                         addresstype_to_typeindex_to_addressdatawithsource
                                             .insert_for_type(output_type, typeindex, addressdata);
                                     }
@@ -1383,15 +1405,15 @@ impl Vecs {
         any_address_indexes: &AnyAddressIndexesVecs,
         addresses_data: &AddressesDataVecs,
     ) -> Option<WithAddressDataSource<LoadedAddressData>> {
+        let typeindex_usize: usize = typeindex.into();
+        if typeindex_usize == 254909199 {
+            eprintln!("DEBUG get_addressdatawithsource EXACT: address_type={:?}, typeindex={}", address_type, typeindex_usize);
+        }
         let first = *first_addressindexes.get(address_type).unwrap();
         if first <= typeindex {
-            let typeindex_usize: usize = typeindex.into();
             let first_usize: usize = first.into();
-            if typeindex_usize > 100_000_000 {
+            if typeindex_usize == 254909199 {
-                eprintln!(
+                eprintln!("DEBUG get_addressdatawithsource returning New EXACT: address_type={:?}, first={}", address_type, first_usize);
-                    "DEBUG get_addressdatawithsource NEW: address_type={:?}, typeindex={}, first_addressindex={}",
-                    address_type, typeindex_usize, first_usize
-                );
             }
             return Some(WithAddressDataSource::New(LoadedAddressData::default()));
         }
@@ -1478,13 +1500,6 @@ impl Vecs {
             addresstype_to_typeindex_to_emptyaddressdata.into_sorted_iter()
         {
             for (typeindex, emptyaddressdata_with_source) in sorted.into_iter() {
-                let typeindex_usize: usize = typeindex.into();
-                if typeindex_usize > 100_000_000 {
-                    eprintln!(
-                        "DEBUG emptyaddressdata: address_type={:?}, typeindex={}, variant={:?}",
-                        address_type, typeindex_usize, std::mem::discriminant(&emptyaddressdata_with_source)
-                    );
-                }
                 match emptyaddressdata_with_source {
                     WithAddressDataSource::New(emptyaddressdata) => {
                         let emptyaddressindex = self
@@ -1494,6 +1509,10 @@ impl Vecs {
 
                         let anyaddressindex = AnyAddressIndex::from(emptyaddressindex);
 
+                        let ti: usize = typeindex.into();
+                        if address_type == OutputType::P2SH && ti > 30_000_000 {
+                            eprintln!("DEBUG insert1 (empty New): P2SH typeindex={}", ti);
+                        }
                         addresstype_to_typeindex_to_new_or_updated_anyaddressindex
                             .get_mut(address_type)
                             .unwrap()
@@ -1519,6 +1538,10 @@ impl Vecs {
 
                         let anyaddressindex = emptyaddressindex.into();
 
+                        let ti: usize = typeindex.into();
+                        if address_type == OutputType::P2SH && ti > 30_000_000 {
+                            eprintln!("DEBUG insert2 (empty FromLoaded): P2SH typeindex={}", ti);
+                        }
                         addresstype_to_typeindex_to_new_or_updated_anyaddressindex
                             .get_mut(address_type)
                             .unwrap()
@@ -1532,13 +1555,6 @@ impl Vecs {
             addresstype_to_typeindex_to_loadedaddressdata.into_sorted_iter()
         {
             for (typeindex, loadedaddressdata_with_source) in sorted.into_iter() {
-                let typeindex_usize: usize = typeindex.into();
-                if typeindex_usize > 100_000_000 {
-                    eprintln!(
-                        "DEBUG loadedaddressdata: address_type={:?}, typeindex={}, variant={:?}",
-                        address_type, typeindex_usize, std::mem::discriminant(&loadedaddressdata_with_source)
-                    );
-                }
                 match loadedaddressdata_with_source {
                     WithAddressDataSource::New(loadedaddressdata) => {
                         let loadedaddressindex = self
@@ -1548,6 +1564,10 @@ impl Vecs {
 
                         let anyaddressindex = AnyAddressIndex::from(loadedaddressindex);
 
+                        let ti: usize = typeindex.into();
+                        if address_type == OutputType::P2SH && ti > 30_000_000 {
+                            eprintln!("DEBUG insert3 (loaded New): P2SH typeindex={}", ti);
+                        }
                         addresstype_to_typeindex_to_new_or_updated_anyaddressindex
                             .get_mut(address_type)
                             .unwrap()
@@ -1573,6 +1593,10 @@ impl Vecs {
 
                         let anyaddressindex = loadedaddressindex.into();
 
+                        let ti: usize = typeindex.into();
+                        if address_type == OutputType::P2SH && ti > 30_000_000 {
+                            eprintln!("DEBUG insert4 (loaded FromEmpty): P2SH typeindex={}", ti);
+                        }
                         addresstype_to_typeindex_to_new_or_updated_anyaddressindex
                             .get_mut(address_type)
                             .unwrap()
@@ -1586,6 +1610,14 @@ impl Vecs {
             addresstype_to_typeindex_to_new_or_updated_anyaddressindex.into_sorted_iter()
         {
             for (typeindex, anyaddressindex) in sorted {
+                // Debug: log right before the call that fails
+                let typeindex_usize: usize = typeindex.into();
+                if address_type == OutputType::P2SH && typeindex_usize > 30_000_000 {
+                    eprintln!(
+                        "DEBUG flush update_or_push: address_type={:?}, typeindex={}, anyaddressindex={:?}",
+                        address_type, typeindex_usize, anyaddressindex
+                    );
+                }
                 self.any_address_indexes.update_or_push(
                     address_type,
                     typeindex,
@@ -1611,4 +1643,3 @@ impl Vecs {
         Ok(())
     }
 }
-

crates/brk_computer/src/stateful/transaction_processing.rs

@@ -1,16 +1,15 @@
 use brk_error::Result;
 use brk_grouper::{ByAddressType, Filtered};
-use brk_types::{CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex};
+use brk_types::{
+    CheckedSub, Dollars, EmptyAddressData, Height, LoadedAddressData, Sats, Timestamp, TypeIndex,
+};
 use vecdb::VecIndex;
 
 use crate::utils::OptionExt;
 
 use super::{
     address_cohorts,
-    addresstype::{
+    addresstype::{AddressTypeToTypeIndexMap, AddressTypeToVec, HeightToAddressTypeToVec},
-        AddressTypeToTypeIndexMap, AddressTypeToVec,
-        HeightToAddressTypeToVec,
-    },
     withaddressdatasource::WithAddressDataSource,
 };
 
@@ -34,6 +33,11 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
     ) {
         self.unwrap().into_iter().for_each(|(_type, vec)| {
             vec.into_iter().for_each(|(type_index, value)| {
+                let type_index_usize: usize = type_index.into();
+                if type_index_usize == 254909199 {
+                    eprintln!("DEBUG process_received EXACT: _type={:?}, type_index={}", _type, type_index_usize);
+                }
+
                 let mut is_new = false;
                 let mut from_any_empty = false;
 
@@ -48,12 +52,18 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
                             .remove(&type_index)
                             .map(|ad| {
                                 from_any_empty = true;
+                                if type_index_usize == 254909199 {
+                                    eprintln!("DEBUG process_received from_empty EXACT: _type={:?}, is_new={}", _type, ad.is_new());
+                                }
                                 ad.into()
                             })
                             .unwrap_or_else(|| {
                                 let addressdata =
                                     stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
                                         .remove_for_type(_type, &type_index);
+                                if type_index_usize == 254909199 {
+                                    eprintln!("DEBUG process_received from_stored_or_new EXACT: _type={:?}, is_new={}", _type, addressdata.is_new());
+                                }
                                 is_new = addressdata.is_new();
                                 from_any_empty = addressdata.is_from_emptyaddressdata();
                                 addressdata
@@ -73,14 +83,15 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
 
                 let amount = prev_amount + value;
 
-                let filters_differ =
+                let filters_differ = vecs.amount_range.get(amount).filter()
-                    vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter();
+                    != vecs.amount_range.get(prev_amount).filter();
 
                 if is_new || from_any_empty || filters_differ {
                     if !is_new && !from_any_empty {
                         vecs.amount_range
                             .get_mut(prev_amount)
-                            .state.um()
+                            .state
+                            .um()
                             .subtract(addressdata);
                     }
 
@@ -88,12 +99,14 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
 
                     vecs.amount_range
                         .get_mut(amount)
-                        .state.um()
+                        .state
+                        .um()
                         .add(addressdata);
                 } else {
                     vecs.amount_range
                         .get_mut(amount)
-                        .state.um()
+                        .state
+                        .um()
                         .receive(addressdata, value, price);
                 }
             });
@@ -143,6 +156,11 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
 
             v.unwrap().into_iter().try_for_each(|(_type, vec)| {
                 vec.into_iter().try_for_each(|(type_index, value)| {
+                    let type_index_usize: usize = type_index.into();
+                    if type_index_usize == 254909199 {
+                        eprintln!("DEBUG process_sent EXACT: _type={:?}, type_index={}", _type, type_index_usize);
+                    }
+
                     let typeindex_to_loadedaddressdata =
                         addresstype_to_typeindex_to_loadedaddressdata.get_mut_unwrap(_type);
 
@@ -161,13 +179,14 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
 
                     let will_be_empty = addressdata.has_1_utxos();
 
-                    let filters_differ =
+                    let filters_differ = vecs.amount_range.get(amount).filter()
-                        vecs.amount_range.get(amount).filter() != vecs.amount_range.get(prev_amount).filter();
+                        != vecs.amount_range.get(prev_amount).filter();
 
                     if will_be_empty || filters_differ {
                         vecs.amount_range
                             .get_mut(prev_amount)
-                            .state.um()
+                            .state
+                            .um()
                             .subtract(addressdata);
 
                         addressdata.send(value, prev_price)?;
@@ -183,6 +202,9 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
                             let addressdata =
                                 typeindex_to_loadedaddressdata.remove(&type_index).unwrap();
 
+                            if type_index_usize == 254909199 {
+                                eprintln!("DEBUG process_sent will_be_empty EXACT: _type={:?}", _type);
+                            }
                             addresstype_to_typeindex_to_emptyaddressdata
                                 .get_mut(_type)
                                 .unwrap()
@@ -190,22 +212,20 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
                         } else {
                             vecs.amount_range
                                 .get_mut(amount)
-                                .state.um()
+                                .state
+                                .um()
                                 .add(addressdata);
                         }
                     } else {
-                        vecs.amount_range
+                        vecs.amount_range.get_mut(amount).state.um().send(
-                            .get_mut(amount)
+                            addressdata,
-                            .state.um()
+                            value,
-                            .send(
+                            price,
-                                addressdata,
+                            prev_price,
-                                value,
+                            blocks_old,
-                                price,
+                            days_old,
-                                prev_price,
+                            older_than_hour,
-                                blocks_old,
+                        )?;
-                                days_old,
-                                older_than_hour,
-                            )?;
                     }
 
                     Ok(())