computer: convert stores to vecs part 2

README.md

@@ -40,7 +40,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible
 - **[Website](https://bitcoinresearchkit.org)** \
   Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
   It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
-  Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
+  Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
 - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
   Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \
   Just like the website, it's entirely free, with no authentication or rate-limiting.

crates/brk_computer/src/cointime.rs

@@ -302,7 +302,7 @@ impl Vecs {
         stateful: &stateful::Vecs,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
-        let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply;
+        let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply;
 
         self.indexes_to_coinblocks_created.compute_all(
             indexer,
@@ -320,7 +320,7 @@ impl Vecs {
         )?;
 
         let indexes_to_coinblocks_destroyed =
-            &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed;
+            &stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed;
 
         self.indexes_to_coinblocks_stored.compute_all(
             indexer,
@@ -430,7 +430,7 @@ impl Vecs {
 
         if let Some(fetched) = fetched {
             let realized_cap = stateful
-                .utxo_vecs
+                .utxo_cohorts
                 .all
                 .1
                 .height_to_realized_cap
@@ -438,7 +438,7 @@ impl Vecs {
                 .unwrap();
 
             let realized_price = stateful
-                .utxo_vecs
+                .utxo_cohorts
                 .all
                 .1
                 .indexes_to_realized_price

crates/brk_computer/src/stateful/mod.rs

@@ -1,14 +1,17 @@
 use std::{cmp::Ordering, collections::BTreeMap, mem, ops::ControlFlow, path::Path, thread};
 
 use brk_core::{
-    AddressData, ByAddressType, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height,
+    AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex,
-    InputIndex, OutputIndex, OutputType, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version,
+    Dollars, EmptyAddressData, EmptyAddressIndex, Height, InputIndex, LoadedAddressData,
+    LoadedAddressIndex, OutputIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex,
+    P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex,
+    P2WSHAddressIndex, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version,
 };
 use brk_exit::Exit;
 use brk_indexer::Indexer;
 use brk_vec::{
-    AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec,
+    AnyCollectableVec, AnyIndexedVec, AnyVec, CollectableVec, Computation, EagerVec, Format,
-    StoredIndex, StoredVec, VecIterator,
+    GenericStoredVec, IndexedVec, Mmap, StoredIndex, StoredVec, VecIterator,
 };
 use either::Either;
 use log::info;
@@ -18,12 +21,6 @@ use crate::{
     BlockState, SupplyState, Transacted,
     grouped::{ComputedVecsFromHeight, Source},
     market,
-    stateful::{
-        addresstype_to_addresscount::AddressTypeToAddressCount,
-        addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount,
-        addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount,
-        addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs,
-    },
 };
 
 use super::{
@@ -48,11 +45,14 @@ mod utxo_cohort;
 mod utxo_cohorts;
 mod withaddressdatasource;
 
+use addresstype_to_addresscount::*;
+use addresstype_to_height_to_addresscount::*;
+use addresstype_to_indexes_to_addresscount::*;
 pub use addresstype_to_typeindex_tree::*;
 pub use addresstype_to_vec::*;
 use height_to_addresstype_to_vec::*;
 use range_map::*;
-use r#trait::CohortVecs;
+use r#trait::*;
 pub use withaddressdatasource::WithAddressDataSource;
 
 const VERSION: Version = Version::new(18);
@@ -69,8 +69,19 @@ pub struct Vecs {
     pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount,
     pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount,
     pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount,
-    pub utxo_vecs: utxo_cohorts::Vecs,
+    pub utxo_cohorts: utxo_cohorts::Vecs,
-    pub address_vecs: address_cohorts::Vecs,
+    pub address_cohorts: address_cohorts::Vecs,
+
+    pub p2pk33addressindex_to_anyaddressindex: IndexedVec<P2PK33AddressIndex, AnyAddressIndex>,
+    pub p2pk65addressindex_to_anyaddressindex: IndexedVec<P2PK65AddressIndex, AnyAddressIndex>,
+    pub p2pkhaddressindex_to_anyaddressindex: IndexedVec<P2PKHAddressIndex, AnyAddressIndex>,
+    pub p2shaddressindex_to_anyaddressindex: IndexedVec<P2SHAddressIndex, AnyAddressIndex>,
+    pub p2traddressindex_to_anyaddressindex: IndexedVec<P2TRAddressIndex, AnyAddressIndex>,
+    pub p2wpkhaddressindex_to_anyaddressindex: IndexedVec<P2WPKHAddressIndex, AnyAddressIndex>,
+    pub p2wshaddressindex_to_anyaddressindex: IndexedVec<P2WSHAddressIndex, AnyAddressIndex>,
+    pub p2aaddressindex_to_anyaddressindex: IndexedVec<P2AAddressIndex, AnyAddressIndex>,
+    pub loadedaddressindex_to_loadedaddressdata: IndexedVec<LoadedAddressIndex, LoadedAddressData>,
+    pub emptyaddressindex_to_emptyaddressdata: IndexedVec<EmptyAddressIndex, EmptyAddressData>,
 
     pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
     pub indexes_to_empty_address_count: ComputedVecsFromHeight<StoredUsize>,
@@ -426,7 +437,7 @@ impl Vecs {
                     )?,
                 },
             ),
-            utxo_vecs: utxo_cohorts::Vecs::forced_import(
+            utxo_cohorts: utxo_cohorts::Vecs::forced_import(
                 path,
                 version,
                 computation,
@@ -435,7 +446,7 @@ impl Vecs {
                 fetched,
                 &states_path,
             )?,
-            address_vecs: address_cohorts::Vecs::forced_import(
+            address_cohorts: address_cohorts::Vecs::forced_import(
                 path,
                 version,
                 computation,
@@ -444,6 +455,68 @@ impl Vecs {
                 fetched,
                 &states_path,
             )?,
+
+            p2aaddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2pk33addressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2pk65addressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2pkhaddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2shaddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2traddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2wpkhaddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            p2wshaddressindex_to_anyaddressindex: IndexedVec::forced_import(
+                path,
+                "anyaddressindex",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+
+            loadedaddressindex_to_loadedaddressdata: IndexedVec::forced_import(
+                path,
+                "loadedaddressdata",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
+            emptyaddressindex_to_emptyaddressdata: IndexedVec::forced_import(
+                path,
+                "emptyaddressdata",
+                version + VERSION + Version::ZERO,
+                Format::Raw,
+            )?,
         })
     }
 
@@ -555,11 +628,10 @@ impl Vecs {
                 .map_or(Version::ZERO, |v| v.version())
             + height_to_date_fixed.version()
             + dateindex_to_first_height.version()
-            + dateindex_to_height_count.version()
+            + dateindex_to_height_count.version();
-            + stores.as_slice().into_iter().map(|s| s.version()).sum();
 
-        let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs();
+        let mut separate_utxo_vecs = self.utxo_cohorts.as_mut_separate_vecs();
-        let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs();
+        let mut separate_address_vecs = self.address_cohorts.as_mut_separate_vecs();
 
         separate_utxo_vecs
             .par_iter_mut()
@@ -592,7 +664,56 @@ impl Vecs {
                     .unwrap_or_default(),
             )
             .min(chain_state_starting_height)
-            .min(stores.starting_height())
+            .min(
+                self.p2pk33addressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2pk65addressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2pkhaddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2shaddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2traddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2wpkhaddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2wshaddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.p2aaddressindex_to_anyaddressindex
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.loadedaddressindex_to_loadedaddressdata
+                    .height()
+                    .incremented(),
+            )
+            .min(
+                self.emptyaddressindex_to_emptyaddressdata
+                    .height()
+                    .incremented(),
+            )
             .min(Height::from(self.height_to_unspendable_supply.len()))
             .min(Height::from(self.height_to_opreturn_supply.len()))
             .cmp(&chain_state_starting_height)
@@ -627,11 +748,21 @@ impl Vecs {
         if starting_height.is_zero() {
             info!("Starting processing utxos from the start");
 
-            // todo!("rollback instead");
+            // TODO: rollback instead
+
             chain_state = vec![];
             chain_state_starting_height = Height::ZERO;
 
-            stores.reset()?;
+            self.p2pk33addressindex_to_anyaddressindex.reset()?;
+            self.p2pk65addressindex_to_anyaddressindex.reset()?;
+            self.p2pkhaddressindex_to_anyaddressindex.reset()?;
+            self.p2shaddressindex_to_anyaddressindex.reset()?;
+            self.p2traddressindex_to_anyaddressindex.reset()?;
+            self.p2wpkhaddressindex_to_anyaddressindex.reset()?;
+            self.p2wshaddressindex_to_anyaddressindex.reset()?;
+            self.p2aaddressindex_to_anyaddressindex.reset()?;
+            self.loadedaddressindex_to_loadedaddressdata.reset()?;
+            self.emptyaddressindex_to_emptyaddressdata.reset()?;
 
             info!("Resetting utxo price maps...");
 
@@ -688,10 +819,19 @@ impl Vecs {
 
             let mut height = starting_height;
 
-            let mut addresstype_to_typeindex_to_addressdata =
+            let mut addresstype_to_typeindex_to_loadedaddressdata =
-                AddressTypeToTypeIndexTree::<WithAddressDataSource<AddressData>>::default();
+                AddressTypeToTypeIndexTree::<WithAddressDataSource<LoadedAddressData>>::default();
             let mut addresstype_to_typeindex_to_emptyaddressdata =
                 AddressTypeToTypeIndexTree::<WithAddressDataSource<EmptyAddressData>>::default();
+            let mut addresstypeindex_to_anyaddressindex_mmap_opt =
+                ByAddressType::<Option<Mmap>>::default();
+            let mut anyaddressindex_to_anyaddressdata_mmap_opt =
+                ByAnyAddress::<Option<Mmap>>::default();
+
+            self.reset_mmaps_options(
+                &mut addresstypeindex_to_anyaddressindex_mmap_opt,
+                &mut anyaddressindex_to_anyaddressdata_mmap_opt,
+            );
 
             (height.unwrap_to_usize()..height_to_date_fixed.len())
                 .map(Height::from)
@@ -700,12 +840,12 @@ impl Vecs {
 
                     info!("Processing chain at {height}...");
 
-                    self.utxo_vecs
+                    self.utxo_cohorts
                         .as_mut_separate_vecs()
                         .iter_mut()
                         .for_each(|(_, v)| v.state.reset_single_iteration_values());
 
-                    self.address_vecs
+                    self.address_cohorts
                         .as_mut_separate_vecs()
                         .iter_mut()
                         .for_each(|(_, v)| v.state.reset_single_iteration_values());
@@ -755,10 +895,11 @@ impl Vecs {
                         addresstype_to_typedindex_to_received_data,
                         mut height_to_sent,
                         addresstype_to_typedindex_to_sent_data,
-                        mut stored_addressindexes,
+                        mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource,
                     ) = thread::scope(|scope| {
                         scope.spawn(|| {
-                            self.utxo_vecs.tick_tock_next_block(&chain_state, timestamp);
+                            self.utxo_cohorts
+                                .tick_tock_next_block(&chain_state, timestamp);
                         });
 
                         let received = scope.spawn(|| {
@@ -767,7 +908,8 @@ impl Vecs {
                             let mut addresstype_to_typedindex_to_received_data =
                                 AddressTypeToVec::<(TypeIndex, Sats)>::default();
 
-                            let mut receiving_addressindexes = AddressTypeToTypeIndexSet::default();
+                            let mut addresstype_to_typeindex_to_addressdatawithsource =
+                                AddressTypeToTypeIndexTree::default();
 
                             let _ = (first_outputindex..first_outputindex + *output_count)
                                 .map(OutputIndex::from)
@@ -787,10 +929,30 @@ impl Vecs {
                                     let typeindex = outputindex_to_typeindex
                                         .unwrap_read(outputindex, &outputindex_to_typeindex_mmap);
 
-                                    receiving_addressindexes
+                                    if let Some(addressdata) = Self::get_addressdatawithsource(
-                                        .get_mut(output_type)
+                                        output_type,
-                                        .unwrap()
+                                        typeindex,
-                                        .insert(typeindex);
+                                        &first_addressindexes,
+                                        &addresstype_to_typeindex_to_loadedaddressdata,
+                                        &addresstype_to_typeindex_to_emptyaddressdata,
+                                        &addresstypeindex_to_anyaddressindex_mmap_opt,
+                                        &anyaddressindex_to_anyaddressdata_mmap_opt,
+                                        &self.p2pk33addressindex_to_anyaddressindex,
+                                        &self.p2pk65addressindex_to_anyaddressindex,
+                                        &self.p2pkhaddressindex_to_anyaddressindex,
+                                        &self.p2shaddressindex_to_anyaddressindex,
+                                        &self.p2traddressindex_to_anyaddressindex,
+                                        &self.p2wpkhaddressindex_to_anyaddressindex,
+                                        &self.p2wshaddressindex_to_anyaddressindex,
+                                        &self.p2aaddressindex_to_anyaddressindex,
+                                        &self.loadedaddressindex_to_loadedaddressdata,
+                                        &self.emptyaddressindex_to_emptyaddressdata,
+                                    ) {
+                                        addresstype_to_typeindex_to_addressdatawithsource
+                                            .get_mut(output_type)
+                                            .unwrap()
+                                            .insert(typeindex, addressdata);
+                                    }
 
                                     addresstype_to_typedindex_to_received_data
                                         .get_mut(output_type)
@@ -803,7 +965,7 @@ impl Vecs {
                             (
                                 transacted,
                                 addresstype_to_typedindex_to_received_data,
-                                receiving_addressindexes,
+                                addresstype_to_typeindex_to_addressdatawithsource,
                             )
                         });
 
@@ -812,7 +974,8 @@ impl Vecs {
                         let mut addresstype_to_typedindex_to_sent_data =
                             HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default();
 
-                        let mut sending_addressindexes = AddressTypeToTypeIndexSet::default();
+                        let mut sending_addresstype_to_typeindex_to_addressdatawithsource =
+                            AddressTypeToTypeIndexTree::default();
 
                         // Skip coinbase
                         let _ = (first_inputindex + 1..first_inputindex + *input_count)
@@ -842,10 +1005,30 @@ impl Vecs {
                                 let typeindex = outputindex_to_typeindex
                                     .unwrap_read(outputindex, &outputindex_to_typeindex_mmap);
 
-                                sending_addressindexes
+                                if let Some(addressdata) = Self::get_addressdatawithsource(
-                                    .get_mut(input_type)
+                                    input_type,
-                                    .unwrap()
+                                    typeindex,
-                                    .insert(typeindex);
+                                    &first_addressindexes,
+                                    &addresstype_to_typeindex_to_loadedaddressdata,
+                                    &addresstype_to_typeindex_to_emptyaddressdata,
+                                    &addresstypeindex_to_anyaddressindex_mmap_opt,
+                                    &anyaddressindex_to_anyaddressdata_mmap_opt,
+                                    &self.p2pk33addressindex_to_anyaddressindex,
+                                    &self.p2pk65addressindex_to_anyaddressindex,
+                                    &self.p2pkhaddressindex_to_anyaddressindex,
+                                    &self.p2shaddressindex_to_anyaddressindex,
+                                    &self.p2traddressindex_to_anyaddressindex,
+                                    &self.p2wpkhaddressindex_to_anyaddressindex,
+                                    &self.p2wshaddressindex_to_anyaddressindex,
+                                    &self.p2aaddressindex_to_anyaddressindex,
+                                    &self.loadedaddressindex_to_loadedaddressdata,
+                                    &self.emptyaddressindex_to_emptyaddressdata,
+                                ) {
+                                    sending_addresstype_to_typeindex_to_addressdatawithsource
+                                        .get_mut(input_type)
+                                        .unwrap()
+                                        .insert(typeindex, addressdata);
+                                }
 
                                 addresstype_to_typedindex_to_sent_data
                                     .entry(prev_height)
@@ -860,91 +1043,38 @@ impl Vecs {
                         let (
                             transacted,
                             addresstype_to_typedindex_to_received_data,
-                            receiving_addressindexes,
+                            receiving_addresstype_to_typeindex_to_addressdatawithsource,
                         ) = received.join().unwrap();
 
-                        let stored_addressindexes =
+                        let addresstype_to_typeindex_to_addressdatawithsource =
-                            sending_addressindexes.merge(receiving_addressindexes);
+                            receiving_addresstype_to_typeindex_to_addressdatawithsource
+                                .merge(sending_addresstype_to_typeindex_to_addressdatawithsource);
 
                         (
                             transacted,
                             addresstype_to_typedindex_to_received_data,
                             height_to_sent,
                             addresstype_to_typedindex_to_sent_data,
-                            stored_addressindexes,
+                            addresstype_to_typeindex_to_addressdatawithsource,
                         )
                     });
 
-                    let mut stored_addressdata_with_source = stored_addressindexes
-                        .into_typed_vec()
-                        .into_par_iter()
-                        .flat_map(|(_type, set)| {
-                            set.into_par_iter().map(move |index| (_type, index))
-                        })
-                        .flat_map(|(_type, index)| {
-                            if *first_addressindexes.get(_type).unwrap() <= index {
-                                Some((
-                                    _type,
-                                    index,
-                                    WithAddressDataSource::New(AddressData::default()),
-                                ))
-                            } else if addresstype_to_typeindex_to_addressdata
-                                .get(_type)
-                                .unwrap()
-                                .contains_key(&index)
-                                || addresstype_to_typeindex_to_emptyaddressdata
-                                    .get(_type)
-                                    .unwrap()
-                                    .contains_key(&index)
-                            {
-                                None
-                            } else if let Some(addressdata) =
-                                stores.get_addressdata(_type, index).unwrap()
-                            {
-                                Some((
-                                    _type,
-                                    index,
-                                    WithAddressDataSource::FromAddressDataStore(addressdata),
-                                ))
-                            } else if let Some(emptyaddressdata) =
-                                stores.get_emptyaddressdata(_type, index).unwrap()
-                            {
-                                Some((
-                                    _type,
-                                    index,
-                                    WithAddressDataSource::FromEmptyAddressDataStore(
-                                        emptyaddressdata.into(),
-                                    ),
-                                ))
-                            } else {
-                                unreachable!()
-                            }
-                        })
-                        .fold(
-                            AddressTypeToTypeIndexTree::default,
-                            |mut tree, (_type, index, addressdata)| {
-                                tree.get_mut(_type).unwrap().insert(index, addressdata);
-                                tree
-                            },
-                        )
-                        .reduce(AddressTypeToTypeIndexTree::default, |a, b| a.merge(b));
-
                     thread::scope(|scope| {
                         scope.spawn(|| {
                             addresstype_to_typedindex_to_received_data.process_received(
-                                &mut self.address_vecs,
+                                &mut self.address_cohorts,
-                                &mut addresstype_to_typeindex_to_addressdata,
+                                &mut addresstype_to_typeindex_to_loadedaddressdata,
                                 &mut addresstype_to_typeindex_to_emptyaddressdata,
                                 price,
                                 &mut addresstype_to_address_count,
                                 &mut addresstype_to_empty_address_count,
-                                &mut stored_addressdata_with_source,
+                                &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource,
                             );
 
                             addresstype_to_typedindex_to_sent_data
                                 .process_sent(
-                                    &mut self.address_vecs,
+                                    &mut self.address_cohorts,
-                                    &mut addresstype_to_typeindex_to_addressdata,
+                                    &mut addresstype_to_typeindex_to_loadedaddressdata,
                                     &mut addresstype_to_typeindex_to_emptyaddressdata,
                                     price,
                                     &mut addresstype_to_address_count,
@@ -953,7 +1083,7 @@ impl Vecs {
                                     &height_to_timestamp_fixed_vec,
                                     height,
                                     timestamp,
-                                    &mut stored_addressdata_with_source,
+                                    &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource,
                                 )
                                 .unwrap();
                         });
@@ -993,18 +1123,18 @@ impl Vecs {
                             timestamp,
                         });
 
-                        self.utxo_vecs.receive(transacted, height, price);
+                        self.utxo_cohorts.receive(transacted, height, price);
 
-                        self.utxo_vecs.send(height_to_sent, &mut chain_state);
+                        self.utxo_cohorts.send(height_to_sent, &mut chain_state);
                     });
 
-                    let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs();
+                    let mut separate_utxo_vecs = self.utxo_cohorts.as_mut_separate_vecs();
 
                     separate_utxo_vecs
                         .iter_mut()
                         .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
 
-                    let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs();
+                    let mut separate_address_vecs = self.address_cohorts.as_mut_separate_vecs();
 
                     separate_address_vecs
                         .iter_mut()
@@ -1062,16 +1192,22 @@ impl Vecs {
                         self.flush_states(height, &chain_state, exit)?;
 
                         // Maybe keep some from the end for both
-                        let addresstype_to_typeindex_to_addressdata_to_consume =
+                        let addresstype_to_typeindex_to_loadedaddressdata_to_consume =
-                            mem::take(&mut addresstype_to_typeindex_to_addressdata);
+                            mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata);
                         let addresstype_to_typeindex_to_emptyaddressdata_to_consume =
                             mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata);
 
-                        stores.commit(
+                        // stores.commit(
-                            height,
+                        //     height,
-                            addresstype_to_typeindex_to_addressdata_to_consume,
+                        //     addresstype_to_typeindex_to_loadedaddressdata_to_consume,
-                            addresstype_to_typeindex_to_emptyaddressdata_to_consume,
+                        //     addresstype_to_typeindex_to_emptyaddressdata_to_consume,
-                        )?;
+                        // )?;
+
+                        self.reset_mmaps_options(
+                            &mut addresstypeindex_to_anyaddressindex_mmap_opt,
+                            &mut anyaddressindex_to_anyaddressdata_mmap_opt,
+                        );
+
                         exit.release();
                     }
 
@@ -1083,11 +1219,11 @@ impl Vecs {
             info!("Flushing...");
 
             self.flush_states(height, &chain_state, exit)?;
-            stores.commit(
+            // stores.commit(
-                height,
+            //     height,
-                mem::take(&mut addresstype_to_typeindex_to_addressdata),
+            //     mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata),
-                mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata),
+            //     mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata),
-            )?;
+            // )?;
         } else {
             exit.block();
         }
@@ -1096,12 +1232,12 @@ impl Vecs {
 
         thread::scope(|scope| {
             scope.spawn(|| {
-                self.utxo_vecs
+                self.utxo_cohorts
                     .compute_overlapping_vecs(starting_indexes, exit)
                     .unwrap();
             });
             scope.spawn(|| {
-                self.address_vecs
+                self.address_cohorts
                     .compute_overlapping_vecs(starting_indexes, exit)
                     .unwrap();
             });
@@ -1160,13 +1296,13 @@ impl Vecs {
             &self.addresstype_to_height_to_empty_address_count,
         )?;
 
-        self.utxo_vecs
+        self.utxo_cohorts
             .as_mut_vecs()
             .into_iter()
             .map(|(_, v)| v)
             .map(Either::Left)
             .chain(
-                self.address_vecs
+                self.address_cohorts
                     .as_mut_vecs()
                     .into_iter()
                     .map(|(_, v)| v)
@@ -1185,18 +1321,24 @@ impl Vecs {
 
         info!("Computing rest part 2...");
 
-        let height_to_supply = self.utxo_vecs.all.1.height_to_supply_value.bitcoin.clone();
+        let height_to_supply = self
+            .utxo_cohorts
+            .all
+            .1
+            .height_to_supply_value
+            .bitcoin
+            .clone();
         let dateindex_to_supply = self
-            .utxo_vecs
+            .utxo_cohorts
             .all
             .1
             .indexes_to_supply
             .bitcoin
             .dateindex
             .clone();
-        let height_to_realized_cap = self.utxo_vecs.all.1.height_to_realized_cap.clone();
+        let height_to_realized_cap = self.utxo_cohorts.all.1.height_to_realized_cap.clone();
         let dateindex_to_realized_cap = self
-            .utxo_vecs
+            .utxo_cohorts
             .all
             .1
             .indexes_to_realized_cap
@@ -1206,13 +1348,13 @@ impl Vecs {
         let height_to_realized_cap_ref = height_to_realized_cap.as_ref();
         let dateindex_to_realized_cap_ref = dateindex_to_realized_cap.as_ref();
 
-        self.utxo_vecs
+        self.utxo_cohorts
             .as_mut_vecs()
             .into_iter()
             .map(|(_, v)| v)
             .map(Either::Left)
             .chain(
-                self.address_vecs
+                self.address_cohorts
                     .as_mut_vecs()
                     .into_iter()
                     .map(|(_, v)| v)
@@ -1269,17 +1411,182 @@ impl Vecs {
         Ok(())
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn get_addressdatawithsource(
+        address_type: OutputType,
+        typeindex: TypeIndex,
+        first_addressindexes: &ByAddressType<TypeIndex>,
+        addresstype_to_typeindex_to_loadedaddressdata: &AddressTypeToTypeIndexTree<
+            WithAddressDataSource<LoadedAddressData>,
+        >,
+        addresstype_to_typeindex_to_emptyaddressdata: &AddressTypeToTypeIndexTree<
+            WithAddressDataSource<EmptyAddressData>,
+        >,
+        addresstypeindex_to_anyaddressindex_mmap_opt: &ByAddressType<Option<Mmap>>,
+        anyaddressindex_to_anyaddressdata_mmap_opt: &ByAnyAddress<Option<Mmap>>,
+        p2pk33addressindex_to_anyaddressindex: &IndexedVec<P2PK33AddressIndex, AnyAddressIndex>,
+        p2pk65addressindex_to_anyaddressindex: &IndexedVec<P2PK65AddressIndex, AnyAddressIndex>,
+        p2pkhaddressindex_to_anyaddressindex: &IndexedVec<P2PKHAddressIndex, AnyAddressIndex>,
+        p2shaddressindex_to_anyaddressindex: &IndexedVec<P2SHAddressIndex, AnyAddressIndex>,
+        p2traddressindex_to_anyaddressindex: &IndexedVec<P2TRAddressIndex, AnyAddressIndex>,
+        p2wpkhaddressindex_to_anyaddressindex: &IndexedVec<P2WPKHAddressIndex, AnyAddressIndex>,
+        p2wshaddressindex_to_anyaddressindex: &IndexedVec<P2WSHAddressIndex, AnyAddressIndex>,
+        p2aaddressindex_to_anyaddressindex: &IndexedVec<P2AAddressIndex, AnyAddressIndex>,
+        loadedaddressindex_to_loadedaddressdata: &IndexedVec<LoadedAddressIndex, LoadedAddressData>,
+        emptyaddressindex_to_emptyaddressdata: &IndexedVec<EmptyAddressIndex, EmptyAddressData>,
+    ) -> Option<WithAddressDataSource<LoadedAddressData>> {
+        if *first_addressindexes.get(address_type).unwrap() <= typeindex {
+            return Some(WithAddressDataSource::New(LoadedAddressData::default()));
+        }
+
+        if addresstype_to_typeindex_to_loadedaddressdata
+            .get(address_type)
+            .unwrap()
+            .contains_key(&typeindex)
+            || addresstype_to_typeindex_to_emptyaddressdata
+                .get(address_type)
+                .unwrap()
+                .contains_key(&typeindex)
+        {
+            return None;
+        }
+
+        let mmap = addresstypeindex_to_anyaddressindex_mmap_opt
+            .get_unwrap(address_type)
+            .as_ref()
+            .unwrap();
+
+        let anyaddressindex = match address_type {
+            OutputType::P2PK33 => {
+                p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2PK65 => {
+                p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2PKH => {
+                p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2SH => {
+                p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2TR => {
+                p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2WPKH => {
+                p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2WSH => {
+                p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            OutputType::P2A => {
+                p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
+            }
+            _ => unreachable!(),
+        }
+        .unwrap()
+        .unwrap()
+        .into_owned();
+
+        Some(match anyaddressindex.to_enum() {
+            AnyAddressDataIndexEnum::Loaded(index) => {
+                let mmap = anyaddressindex_to_anyaddressdata_mmap_opt
+                    .loaded
+                    .as_ref()
+                    .unwrap();
+
+                let loadedaddressdata = loadedaddressindex_to_loadedaddressdata
+                    .get_or_read(index, mmap)
+                    .unwrap()
+                    .unwrap()
+                    .into_owned();
+
+                WithAddressDataSource::FromLoadedAddressDataVec(loadedaddressdata)
+            }
+            AnyAddressDataIndexEnum::Empty(index) => {
+                let mmap = anyaddressindex_to_anyaddressdata_mmap_opt
+                    .empty
+                    .as_ref()
+                    .unwrap();
+
+                let emptyaddressdata = emptyaddressindex_to_emptyaddressdata
+                    .get_or_read(index, mmap)
+                    .unwrap()
+                    .unwrap()
+                    .into_owned();
+
+                WithAddressDataSource::FromEmptyAddressDataVec(emptyaddressdata.into())
+            }
+        })
+    }
+
+    fn reset_mmaps_options(
+        &self,
+        addresstypeindex_to_anyaddressindex_mmap_opt: &mut ByAddressType<Option<Mmap>>,
+        anyaddressindex_to_anyaddressdata_mmap_opt: &mut ByAnyAddress<Option<Mmap>>,
+    ) {
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2pk65.replace(
+            self.p2pk65addressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2pk33.replace(
+            self.p2pk33addressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2pkh.replace(
+            self.p2pkhaddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2sh.replace(
+            self.p2shaddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2wpkh.replace(
+            self.p2wpkhaddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2wsh.replace(
+            self.p2wshaddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2tr.replace(
+            self.p2traddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        addresstypeindex_to_anyaddressindex_mmap_opt.p2a.replace(
+            self.p2aaddressindex_to_anyaddressindex
+                .create_mmap()
+                .unwrap(),
+        );
+        anyaddressindex_to_anyaddressdata_mmap_opt.loaded.replace(
+            self.loadedaddressindex_to_loadedaddressdata
+                .create_mmap()
+                .unwrap(),
+        );
+        anyaddressindex_to_anyaddressdata_mmap_opt.empty.replace(
+            self.emptyaddressindex_to_emptyaddressdata
+                .create_mmap()
+                .unwrap(),
+        );
+    }
+
     fn flush_states(
         &mut self,
         height: Height,
         chain_state: &[BlockState],
         exit: &Exit,
     ) -> Result<()> {
-        self.utxo_vecs
+        self.utxo_cohorts
             .as_mut_separate_vecs()
             .par_iter_mut()
             .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
-        self.address_vecs
+        self.address_cohorts
             .as_mut_separate_vecs()
             .par_iter_mut()
             .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
@@ -1294,6 +1601,17 @@ impl Vecs {
             .into_iter()
             .try_for_each(|v| v.safe_flush(exit))?;
 
+        self.p2pk33addressindex_to_anyaddressindex.flush(height)?;
+        self.p2pk65addressindex_to_anyaddressindex.flush(height)?;
+        self.p2pkhaddressindex_to_anyaddressindex.flush(height)?;
+        self.p2shaddressindex_to_anyaddressindex.flush(height)?;
+        self.p2traddressindex_to_anyaddressindex.flush(height)?;
+        self.p2wpkhaddressindex_to_anyaddressindex.flush(height)?;
+        self.p2wshaddressindex_to_anyaddressindex.flush(height)?;
+        self.p2aaddressindex_to_anyaddressindex.flush(height)?;
+        self.loadedaddressindex_to_loadedaddressdata.flush(height)?;
+        self.emptyaddressindex_to_emptyaddressdata.flush(height)?;
+
         self.chain_state.truncate_if_needed(Height::ZERO)?;
         chain_state.iter().for_each(|block_state| {
             self.chain_state.push(block_state.supply.clone());
@@ -1305,12 +1623,12 @@ impl Vecs {
 
     pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
         [
-            self.utxo_vecs
+            self.utxo_cohorts
                 .vecs()
                 .into_iter()
                 .flat_map(|v| v.vecs())
                 .collect::<Vec<_>>(),
-            self.address_vecs
+            self.address_cohorts
                 .vecs()
                 .into_iter()
                 .flat_map(|v| v.vecs())
@@ -1347,8 +1665,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
     fn process_received(
         mut self,
         vecs: &mut address_cohorts::Vecs,
-        addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree<
+        addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexTree<
-            WithAddressDataSource<AddressData>,
+            WithAddressDataSource<LoadedAddressData>,
         >,
         addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree<
             WithAddressDataSource<EmptyAddressData>,
@@ -1356,8 +1674,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
         price: Option<Dollars>,
         addresstype_to_address_count: &mut ByAddressType<usize>,
         addresstype_to_empty_address_count: &mut ByAddressType<usize>,
-        stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree<
+        stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree<
-            WithAddressDataSource<AddressData>,
+            WithAddressDataSource<LoadedAddressData>,
         >,
     ) {
         self.into_typed_vec().into_iter().for_each(|(_type, vec)| {
@@ -1365,7 +1683,7 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
                 let mut is_new = false;
                 let mut from_any_empty = false;
 
-                let addressdata_withsource = addresstype_to_typeindex_to_addressdata
+                let addressdata_withsource = addresstype_to_typeindex_to_loadedaddressdata
                     .get_mut(_type)
                     .unwrap()
                     .entry(type_index)
@@ -1379,11 +1697,12 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
                                 ad.into()
                             })
                             .unwrap_or_else(|| {
-                                let addressdata = stored_addressdata_with_source
+                                let addressdata =
-                                    .get_mut(_type)
+                                    stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
-                                    .unwrap()
+                                        .get_mut(_type)
-                                    .remove(&type_index)
+                                        .unwrap()
-                                    .unwrap();
+                                        .remove(&type_index)
+                                        .unwrap();
                                 is_new = addressdata.is_new();
                                 from_any_empty = addressdata.is_from_emptyaddressdata();
                                 addressdata
@@ -1436,8 +1755,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
     fn process_sent(
         self,
         vecs: &mut address_cohorts::Vecs,
-        addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree<
+        addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexTree<
-            WithAddressDataSource<AddressData>,
+            WithAddressDataSource<LoadedAddressData>,
         >,
         addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree<
             WithAddressDataSource<EmptyAddressData>,
@@ -1449,8 +1768,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
         height_to_timestamp_fixed_vec: &[Timestamp],
         height: Height,
         timestamp: Timestamp,
-        stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree<
+        stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree<
-            WithAddressDataSource<AddressData>,
+            WithAddressDataSource<LoadedAddressData>,
         >,
     ) -> Result<()> {
         self.0.into_iter().try_for_each(|(prev_height, mut v)| {
@@ -1473,14 +1792,15 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
 
             v.into_typed_vec().into_iter().try_for_each(|(_type, vec)| {
                 vec.into_iter().try_for_each(|(type_index, value)| {
-                    let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata
+                    let typeindex_to_loadedaddressdata =
-                        .get_mut(_type)
+                        addresstype_to_typeindex_to_loadedaddressdata
-                        .unwrap();
+                            .get_mut(_type)
+                            .unwrap();
 
-                    let addressdata_withsource = typeindex_to_addressdata
+                    let addressdata_withsource = typeindex_to_loadedaddressdata
                         .entry(type_index)
                         .or_insert_with(|| {
-                            stored_addressdata_with_source
+                            stored_or_new_addresstype_to_typeindex_to_addressdatawithsource
                                 .get_mut(_type)
                                 .unwrap()
                                 .remove(&type_index)
@@ -1515,7 +1835,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
                             (*addresstype_to_address_count.get_mut(_type).unwrap()) -= 1;
                             (*addresstype_to_empty_address_count.get_mut(_type).unwrap()) += 1;
 
-                            let addressdata = typeindex_to_addressdata.remove(&type_index).unwrap();
+                            let addressdata =
+                                typeindex_to_loadedaddressdata.remove(&type_index).unwrap();
 
                             addresstype_to_typeindex_to_emptyaddressdata
                                 .get_mut(_type)

crates/brk_computer/src/stateful/withaddressdatasource.rs

@@ -1,10 +1,10 @@
-use brk_core::{AddressData, EmptyAddressData};
+use brk_core::{EmptyAddressData, LoadedAddressData};
 
 #[derive(Debug)]
 pub enum WithAddressDataSource<T> {
     New(T),
-    FromAddressDataStore(T),
+    FromLoadedAddressDataVec(T),
-    FromEmptyAddressDataStore(T),
+    FromEmptyAddressDataVec(T),
 }
 
 impl<T> WithAddressDataSource<T> {
@@ -13,49 +13,53 @@ impl<T> WithAddressDataSource<T> {
     }
 
     pub fn is_from_addressdata(&self) -> bool {
-        matches!(self, Self::FromAddressDataStore(_))
+        matches!(self, Self::FromLoadedAddressDataVec(_))
     }
 
     pub fn is_from_emptyaddressdata(&self) -> bool {
-        matches!(self, Self::FromEmptyAddressDataStore(_))
+        matches!(self, Self::FromEmptyAddressDataVec(_))
     }
 
     pub fn deref(&self) -> &T {
         match self {
             Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
+            Self::FromLoadedAddressDataVec(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
+            Self::FromEmptyAddressDataVec(v) => v,
         }
     }
 
     pub fn deref_mut(&mut self) -> &mut T {
         match self {
             Self::New(v) => v,
-            Self::FromAddressDataStore(v) => v,
+            Self::FromLoadedAddressDataVec(v) => v,
-            Self::FromEmptyAddressDataStore(v) => v,
+            Self::FromEmptyAddressDataVec(v) => v,
         }
     }
 }
 
-impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<AddressData> {
+impl From<WithAddressDataSource<EmptyAddressData>> for WithAddressDataSource<LoadedAddressData> {
     fn from(value: WithAddressDataSource<EmptyAddressData>) -> Self {
         match value {
             WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec(v) => {
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
+                Self::FromLoadedAddressDataVec(v.into())
-                Self::FromEmptyAddressDataStore(v.into())
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec(v) => {
+                Self::FromEmptyAddressDataVec(v.into())
             }
         }
     }
 }
 
-impl From<WithAddressDataSource<AddressData>> for WithAddressDataSource<EmptyAddressData> {
+impl From<WithAddressDataSource<LoadedAddressData>> for WithAddressDataSource<EmptyAddressData> {
-    fn from(value: WithAddressDataSource<AddressData>) -> Self {
+    fn from(value: WithAddressDataSource<LoadedAddressData>) -> Self {
         match value {
             WithAddressDataSource::New(v) => Self::New(v.into()),
-            WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()),
+            WithAddressDataSource::FromLoadedAddressDataVec(v) => {
-            WithAddressDataSource::FromEmptyAddressDataStore(v) => {
+                Self::FromLoadedAddressDataVec(v.into())
-                Self::FromEmptyAddressDataStore(v.into())
+            }
+            WithAddressDataSource::FromEmptyAddressDataVec(v) => {
+                Self::FromEmptyAddressDataVec(v.into())
             }
         }
     }

crates/brk_computer/src/states/cohorts/address.rs

@@ -1,6 +1,6 @@
 use std::path::Path;
 
-use brk_core::{AddressData, Dollars, Height, Result, Sats};
+use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats};
 
 use crate::SupplyState;
 
@@ -35,7 +35,7 @@ impl AddressCohortState {
     #[allow(clippy::too_many_arguments)]
     pub fn send(
         &mut self,
-        addressdata: &mut AddressData,
+        addressdata: &mut LoadedAddressData,
         value: Sats,
         current_price: Option<Dollars>,
         prev_price: Option<Dollars>,
@@ -72,7 +72,12 @@ impl AddressCohortState {
         Ok(())
     }
 
-    pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option<Dollars>) {
+    pub fn receive(
+        &mut self,
+        address_data: &mut LoadedAddressData,
+        value: Sats,
+        price: Option<Dollars>,
+    ) {
         let compute_price = price.is_some();
 
         let prev_realized_price = compute_price.then(|| address_data.realized_price());
@@ -96,7 +101,7 @@ impl AddressCohortState {
         );
     }
 
-    pub fn add(&mut self, addressdata: &AddressData) {
+    pub fn add(&mut self, addressdata: &LoadedAddressData) {
         self.address_count += 1;
         self.inner.increment_(
             &addressdata.into(),
@@ -105,7 +110,7 @@ impl AddressCohortState {
         );
     }
 
-    pub fn subtract(&mut self, addressdata: &AddressData) {
+    pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
         self.address_count = self.address_count.checked_sub(1).unwrap();
         self.inner.decrement_(
             &addressdata.into(),

crates/brk_computer/src/states/supply.rs

@@ -1,6 +1,6 @@
 use std::ops::{Add, AddAssign, SubAssign};
 
-use brk_core::{AddressData, CheckedSub, Sats};
+use brk_core::{CheckedSub, LoadedAddressData, Sats};
 use serde::Serialize;
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
@@ -40,8 +40,8 @@ impl SubAssign<&SupplyState> for SupplyState {
     }
 }
 
-impl From<&AddressData> for SupplyState {
+impl From<&LoadedAddressData> for SupplyState {
-    fn from(value: &AddressData) -> Self {
+    fn from(value: &LoadedAddressData) -> Self {
         Self {
             utxos: value.outputs_len as usize,
             value: value.amount(),

crates/brk_core/src/groups/by_address_type.rs

@@ -19,6 +19,10 @@ pub struct ByAddressType<T> {
 }
 
 impl<T> ByAddressType<T> {
+    pub fn get_unwrap(&self, address_type: OutputType) -> &T {
+        self.get(address_type).unwrap()
+    }
+
     pub fn get(&self, address_type: OutputType) -> Option<&T> {
         match address_type {
             OutputType::P2PK65 => Some(&self.p2pk65),

crates/brk_core/src/groups/by_any_address.rs

@@ -0,0 +1,5 @@
+#[derive(Debug, Default)]
+pub struct ByAnyAddress<T> {
+    pub loaded: T,
+    pub empty: T,
+}

crates/brk_core/src/groups/mod.rs

@@ -2,6 +2,7 @@ mod address;
 mod by_address_type;
 mod by_age_range;
 mod by_amount_range;
+mod by_any_address;
 mod by_epoch;
 mod by_ge_amount;
 mod by_lt_amount;
@@ -18,6 +19,7 @@ pub use address::*;
 pub use by_address_type::*;
 pub use by_age_range::*;
 pub use by_amount_range::*;
+pub use by_any_address::*;
 pub use by_epoch::*;
 pub use by_ge_amount::*;
 pub use by_lt_amount::*;

crates/brk_core/src/structs/anyaddressdataindex.rs

@@ -1,95 +0,0 @@
-use std::ops::Add;
-
-use byteview::ByteView;
-use derive_deref::{Deref, DerefMut};
-use serde::Serialize;
-use zerocopy::{FromBytes, IntoBytes};
-use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
-
-use crate::{CheckedSub, Printable, TypeIndex};
-
-#[derive(
-    Debug,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Clone,
-    Copy,
-    Deref,
-    DerefMut,
-    Default,
-    FromBytes,
-    Immutable,
-    IntoBytes,
-    KnownLayout,
-    Serialize,
-)]
-pub struct AnyAddressDataIndex(TypeIndex);
-// impl From<TypeIndex> for AddressDataIndex {
-//     fn from(value: TypeIndex) -> Self {
-//         Self(value)
-//     }
-// }
-// impl From<AddressDataIndex> for TypeIndex {
-//     fn from(value: AddressDataIndex) -> Self {
-//         value.0
-//     }
-// }
-// impl From<AddressDataIndex> for u32 {
-//     fn from(value: AddressDataIndex) -> Self {
-//         Self::from(*value)
-//     }
-// }
-// impl From<u32> for AddressDataIndex {
-//     fn from(value: u32) -> Self {
-//         Self(TypeIndex::from(value))
-//     }
-// }
-// impl From<AddressDataIndex> for usize {
-//     fn from(value: AddressDataIndex) -> Self {
-//         Self::from(*value)
-//     }
-// }
-// impl From<usize> for AddressDataIndex {
-//     fn from(value: usize) -> Self {
-//         Self(TypeIndex::from(value))
-//     }
-// }
-// impl Add<usize> for AddressDataIndex {
-//     type Output = Self;
-//     fn add(self, rhs: usize) -> Self::Output {
-//         Self(*self + rhs)
-//     }
-// }
-// impl CheckedSub<AddressDataIndex> for AddressDataIndex {
-//     fn checked_sub(self, rhs: Self) -> Option<Self> {
-//         self.0.checked_sub(rhs.0).map(Self)
-//     }
-// }
-
-// impl Printable for AddressDataIndex {
-//     fn to_string() -> &'static str {
-//         "p2pk33addressindex"
-//     }
-
-//     fn to_possible_strings() -> &'static [&'static str] {
-//         &["addr", "p2pk33addr", "p2pk33addressindex"]
-//     }
-// }
-
-// impl From<ByteView> for AddressDataIndex {
-//     fn from(value: ByteView) -> Self {
-//         Self::read_from_bytes(&value).unwrap()
-//     }
-// }
-// impl From<AddressDataIndex> for ByteView {
-//     fn from(value: AddressDataIndex) -> Self {
-//         Self::from(&value)
-//     }
-// }
-// impl From<&AddressDataIndex> for ByteView {
-//     fn from(value: &AddressDataIndex) -> Self {
-//         Self::new(value.as_bytes())
-//     }
-// }

crates/brk_core/src/structs/anyaddressindex.rs

@@ -0,0 +1,48 @@
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{
+    TypeIndex,
+    structs::{EmptyAddressIndex, LoadedAddressIndex},
+};
+
+const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000;
+
+#[derive(
+    Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout,
+)]
+pub struct AnyAddressIndex(TypeIndex);
+
+impl AnyAddressIndex {
+    pub fn to_enum(&self) -> AnyAddressDataIndexEnum {
+        AnyAddressDataIndexEnum::from(*self)
+    }
+}
+
+impl Serialize for AnyAddressIndex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_enum().serialize(serializer)
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub enum AnyAddressDataIndexEnum {
+    Loaded(LoadedAddressIndex),
+    Empty(EmptyAddressIndex),
+}
+
+impl From<AnyAddressIndex> for AnyAddressDataIndexEnum {
+    fn from(value: AnyAddressIndex) -> Self {
+        let uvalue = u32::from(value.0);
+        if uvalue >= MIN_EMPTY_INDEX {
+            Self::Empty(EmptyAddressIndex::from(TypeIndex::from(
+                uvalue - MIN_EMPTY_INDEX,
+            )))
+        } else {
+            Self::Loaded(LoadedAddressIndex::from(value.0))
+        }
+    }
+}

crates/brk_core/src/structs/emptyaddressdata.rs

@@ -1,22 +1,23 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
 use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use crate::{AddressData, Sats};
+use crate::{LoadedAddressData, Sats};
 
-#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)]
+#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
 pub struct EmptyAddressData {
     pub transfered: Sats,
 }
 
-impl From<AddressData> for EmptyAddressData {
+impl From<LoadedAddressData> for EmptyAddressData {
-    fn from(value: AddressData) -> Self {
+    fn from(value: LoadedAddressData) -> Self {
         Self::from(&value)
     }
 }
 
-impl From<&AddressData> for EmptyAddressData {
+impl From<&LoadedAddressData> for EmptyAddressData {
-    fn from(value: &AddressData) -> Self {
+    fn from(value: &LoadedAddressData) -> Self {
         if value.sent != value.received {
             dbg!(&value);
             panic!("Trying to convert not empty wallet to empty !");

crates/brk_core/src/structs/emptyaddressindex.rs

@@ -0,0 +1,62 @@
+use std::ops::Add;
+
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Default,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct EmptyAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for EmptyAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for EmptyAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+
+impl From<EmptyAddressIndex> for usize {
+    fn from(value: EmptyAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+
+impl Add<usize> for EmptyAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<EmptyAddressIndex> for EmptyAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for EmptyAddressIndex {
+    fn to_string() -> &'static str {
+        "emptyaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["emptyaddr", "emptyaddressindex"]
+    }
+}

crates/brk_core/src/structs/emptyoutputindex.rs

@@ -31,7 +31,7 @@ impl From<TypeIndex> for EmptyOutputIndex {
 }
 impl From<EmptyOutputIndex> for usize {
     fn from(value: EmptyOutputIndex) -> Self {
-        Self::from(*value)
+        Self::from(value.0)
     }
 }
 impl From<usize> for EmptyOutputIndex {
@@ -42,7 +42,7 @@ impl From<usize> for EmptyOutputIndex {
 impl Add<usize> for EmptyOutputIndex {
     type Output = Self;
     fn add(self, rhs: usize) -> Self::Output {
-        Self(*self + rhs)
+        Self(self.0 + rhs)
     }
 }
 impl CheckedSub<EmptyOutputIndex> for EmptyOutputIndex {

crates/brk_core/src/structs/loadedaddressdata.rs

@@ -1,18 +1,21 @@
 use byteview::ByteView;
+use serde::Serialize;
 use zerocopy::{FromBytes, IntoBytes};
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
 use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats};
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
 #[repr(C)]
-pub struct AddressData {
+pub struct LoadedAddressData {
     pub sent: Sats,
     pub received: Sats,
     pub realized_cap: Dollars,
     pub outputs_len: u32,
+    padding: u32,
 }
 
-impl AddressData {
+impl LoadedAddressData {
     pub fn amount(&self) -> Sats {
         (u64::from(self.received) - u64::from(self.sent)).into()
     }
@@ -55,23 +58,24 @@ impl AddressData {
     }
 }
 
-impl From<EmptyAddressData> for AddressData {
+impl From<EmptyAddressData> for LoadedAddressData {
     fn from(value: EmptyAddressData) -> Self {
         Self::from(&value)
     }
 }
-impl From<&EmptyAddressData> for AddressData {
+impl From<&EmptyAddressData> for LoadedAddressData {
     fn from(value: &EmptyAddressData) -> Self {
         Self {
             sent: value.transfered,
             received: value.transfered,
             realized_cap: Dollars::ZERO,
             outputs_len: 0,
+            padding: 0,
         }
     }
 }
 
-impl From<ByteView> for AddressData {
+impl From<ByteView> for LoadedAddressData {
     fn from(value: ByteView) -> Self {
         Self {
             // MUST be same order as impl From<&AddressData> for ByteView
@@ -79,16 +83,17 @@ impl From<ByteView> for AddressData {
             received: Sats::read_from_bytes(&value[8..16]).unwrap(),
             realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(),
             outputs_len: u32::read_from_bytes(&value[24..]).unwrap(),
+            padding: 0,
         }
     }
 }
-impl From<AddressData> for ByteView {
+impl From<LoadedAddressData> for ByteView {
-    fn from(value: AddressData) -> Self {
+    fn from(value: LoadedAddressData) -> Self {
         Self::from(&value)
     }
 }
-impl From<&AddressData> for ByteView {
+impl From<&LoadedAddressData> for ByteView {
-    fn from(value: &AddressData) -> Self {
+    fn from(value: &LoadedAddressData) -> Self {
         Self::new(
             &[
                 value.sent.as_bytes(),

crates/brk_core/src/structs/loadedaddressindex.rs

@@ -0,0 +1,60 @@
+use std::ops::Add;
+
+use serde::Serialize;
+use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::{CheckedSub, Printable, TypeIndex};
+
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Default,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct LoadedAddressIndex(TypeIndex);
+
+impl From<TypeIndex> for LoadedAddressIndex {
+    fn from(value: TypeIndex) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for LoadedAddressIndex {
+    fn from(value: usize) -> Self {
+        Self(TypeIndex::from(value))
+    }
+}
+impl From<LoadedAddressIndex> for usize {
+    fn from(value: LoadedAddressIndex) -> Self {
+        usize::from(value.0)
+    }
+}
+impl Add<usize> for LoadedAddressIndex {
+    type Output = Self;
+    fn add(self, rhs: usize) -> Self::Output {
+        Self(self.0 + rhs)
+    }
+}
+impl CheckedSub<LoadedAddressIndex> for LoadedAddressIndex {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+impl Printable for LoadedAddressIndex {
+    fn to_string() -> &'static str {
+        "loadedaddressindex"
+    }
+
+    fn to_possible_strings() -> &'static [&'static str] {
+        &["loadedaddr", "loadedaddressindex"]
+    }
+}

crates/brk_core/src/structs/mod.rs

@@ -1,7 +1,6 @@
 mod addressbytes;
 mod addressbyteshash;
-mod addressdata;
+mod anyaddressindex;
-mod anyaddressdataindex;
 mod bitcoin;
 mod blockhash;
 mod blockhashprefix;
@@ -12,11 +11,14 @@ mod decadeindex;
 mod difficultyepoch;
 mod dollars;
 mod emptyaddressdata;
+mod emptyaddressindex;
 mod emptyoutputindex;
 mod feerate;
 mod halvingepoch;
 mod height;
 mod inputindex;
+mod loadedaddressdata;
+mod loadedaddressindex;
 mod monthindex;
 mod ohlc;
 mod opreturnindex;
@@ -59,8 +61,7 @@ mod yearindex;
 
 pub use addressbytes::*;
 pub use addressbyteshash::*;
-pub use addressdata::*;
+pub use anyaddressindex::*;
-pub use anyaddressdataindex::*;
 pub use bitcoin::*;
 pub use blockhash::*;
 pub use blockhashprefix::*;
@@ -71,11 +72,14 @@ pub use decadeindex::*;
 pub use difficultyepoch::*;
 pub use dollars::*;
 pub use emptyaddressdata::*;
+pub use emptyaddressindex::*;
 pub use emptyoutputindex::*;
 pub use feerate::*;
 pub use halvingepoch::*;
 pub use height::*;
 pub use inputindex::*;
+pub use loadedaddressdata::*;
+pub use loadedaddressindex::*;
 pub use monthindex::*;
 pub use ohlc::*;
 pub use opreturnindex::*;

crates/brk_interface/src/lib.rs

@@ -29,7 +29,7 @@ pub use params::{IdParam, Params, ParamsOpt};
 pub use table::Tabled;
 use vecs::Vecs;
 
-use crate::{IdToVec, IndexToVec};
+use crate::vecs::{IdToVec, IndexToVec};
 
 pub struct Interface<'a> {
     vecs: Vecs<'a>,

crates/brk_interface/src/vecs.rs

@@ -101,33 +101,6 @@ impl<'a> Vecs<'a> {
     fn insert(&mut self, vec: &'a dyn AnyCollectableVec) {
         let name = vec.name();
         let serialized_index = vec.index_type_to_string();
-        // let split = name.split("_to_").collect::<Vec<_>>();
-        // if split.len() != 1
-        //     && !(split.len() == 2
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.ends_with("relative")
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s.starts_with("cumulative_start")
-        //                 || s.starts_with("cumulative_from")
-        //                 || s == &"activity"
-        //         }))
-        //     && !(split.len() == 3
-        //         && split.first().is_some_and(|s| {
-        //             s == &"up"
-        //                 || s == &"start"
-        //                 || s.starts_with("from")
-        //                 || s == &"cumulative_up"
-        //                 || s == &"cumulative_start"
-        //                 || s.starts_with("cumulative_from")
-        //         })
-        //         && split.get(1).is_some_and(|s| s.ends_with("relative")))
-        // {
-        //     dbg!((&serialized_index, &name, &split));
-        //     unreachable!();
-        // }
         let index = Index::try_from(serialized_index)
             .inspect_err(|_| {
                 dbg!(&serialized_index);

crates/brk_parser/src/lib.rs

@@ -43,11 +43,15 @@ pub struct Parser {
 }
 
 impl Parser {
-    pub fn new(blocks_dir: PathBuf, outputs_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self {
+    pub fn new(
+        blocks_dir: PathBuf,
+        outputs_dir: PathBuf,
+        rpc: &'static bitcoincore_rpc::Client,
+    ) -> Self {
         Self {
             blocks_dir,
-            outputs_dir: outputs_dir,
+            outputs_dir,
-            rpc 
+            rpc,
         }
     }
 

crates/brk_vec/examples/main.rs

@@ -1,6 +1,6 @@
 use std::{fs, path::Path};
 
-use brk_core::{DateIndex, Height, Printable, Version};
+use brk_core::{DateIndex, Height, Version};
 use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator};
 
 type I = DateIndex;
@@ -104,7 +104,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
         let mmap = vec.create_mmap()?;
         dbg!(vec.take(10.into(), &mmap)?);
-        dbg!(vec.get_or_read(10.into(), &mmap));
+        dbg!(vec.get_or_read(10.into(), &mmap)?);
         dbg!(vec.holes());
         vec.flush()?;
         dbg!(vec.holes());
@@ -118,11 +118,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         dbg!(vec.holes());
         dbg!(vec.get_or_read(10.into(), &mmap)?);
 
-        vec.update(10.into(), 10);
+        vec.update(10.into(), 10)?;
-        vec.update(0.into(), 10);
+        vec.update(0.into(), 10)?;
         dbg!(
             vec.holes(),
-            vec.get_or_read(0.into(), &mmap),
+            vec.get_or_read(0.into(), &mmap)?,
             vec.get_or_read(10.into(), &mmap)?
         );
 
@@ -130,9 +130,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     {
-        let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
+        let vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
 
-        dbg!(vec.collect());
+        dbg!(vec.collect()?);
     }
 
     Ok(())

crates/brk_vec/src/variants/compressed.rs

@@ -57,6 +57,7 @@ where
         }
     }
 
+    #[allow(unreachable_code, unused_variables)]
     pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
         panic!("Compressed vecs are a work in progress right now, please use raw vecs instead");
 

crates/brk_vec/src/variants/indexed.rs

@@ -62,6 +62,11 @@ where
         self.0.mut_header().update_height(height);
     }
 
+    pub fn reset(&mut self) -> Result<()> {
+        self.update_height(Height::ZERO);
+        self.0.reset()
+    }
+
     pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> {
         self.update_height(height);
         self.0.truncate_if_needed(index)?;