diff --git a/README.md b/README.md index 32a725b04..d7a82eac5 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible - **[Website](https://bitcoinresearchkit.org)** \ Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \ It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \ - Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz) + Also available at: [brekit.org](https://brekit.org) // [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz) - **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \ Researchers and developers are free to use BRK's public API with ![Datasets variant count](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbitcoinresearchkit.org%2Fapi%2Fvecs%2Fvec-count&query=%24&style=flat&label=%20&color=white) dataset variants at their disposal. \ Just like the website, it's entirely free, with no authentication or rate-limiting. diff --git a/crates/brk_computer/src/cointime.rs b/crates/brk_computer/src/cointime.rs index 78a9d462c..c0eb5d8e8 100644 --- a/crates/brk_computer/src/cointime.rs +++ b/crates/brk_computer/src/cointime.rs @@ -302,7 +302,7 @@ impl Vecs { stateful: &stateful::Vecs, exit: &Exit, ) -> color_eyre::Result<()> { - let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply; + let circulating_supply = &stateful.utxo_cohorts.all.1.height_to_supply; self.indexes_to_coinblocks_created.compute_all( indexer, @@ -320,7 +320,7 @@ impl Vecs { )?; let indexes_to_coinblocks_destroyed = - &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed; + &stateful.utxo_cohorts.all.1.indexes_to_coinblocks_destroyed; self.indexes_to_coinblocks_stored.compute_all( indexer, @@ -430,7 +430,7 @@ impl Vecs { if let Some(fetched) = fetched { let realized_cap = stateful - .utxo_vecs + .utxo_cohorts .all .1 .height_to_realized_cap @@ -438,7 +438,7 @@ impl Vecs { .unwrap(); let realized_price = stateful - .utxo_vecs + .utxo_cohorts .all .1 .indexes_to_realized_price diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 6f14e4f51..edbfe0d1c 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1,14 +1,17 @@ use std::{cmp::Ordering, collections::BTreeMap, mem, ops::ControlFlow, path::Path, thread}; use brk_core::{ - AddressData, ByAddressType, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height, - InputIndex, OutputIndex, OutputType, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version, + AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex, + Dollars, EmptyAddressData, EmptyAddressIndex, Height, InputIndex, LoadedAddressData, + LoadedAddressIndex, OutputIndex, OutputType, P2AAddressIndex, P2PK33AddressIndex, + P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, + P2WSHAddressIndex, Result, Sats, StoredUsize, Timestamp, TypeIndex, Version, }; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ - AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec, Format, GenericStoredVec, - StoredIndex, StoredVec, VecIterator, + AnyCollectableVec, AnyIndexedVec, AnyVec, CollectableVec, Computation, EagerVec, Format, + GenericStoredVec, IndexedVec, Mmap, StoredIndex, StoredVec, VecIterator, }; use either::Either; use log::info; @@ -18,12 +21,6 @@ use crate::{ BlockState, SupplyState, Transacted, grouped::{ComputedVecsFromHeight, Source}, market, - stateful::{ - addresstype_to_addresscount::AddressTypeToAddressCount, - addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount, - addresstype_to_indexes_to_addresscount::AddressTypeToIndexesToAddressCount, - addresstype_to_typeindex_set::AddressTypeToTypeIndexSet, r#trait::DynCohortVecs, - }, }; use super::{ @@ -48,11 +45,14 @@ mod utxo_cohort; mod utxo_cohorts; mod withaddressdatasource; +use addresstype_to_addresscount::*; +use addresstype_to_height_to_addresscount::*; +use addresstype_to_indexes_to_addresscount::*; pub use addresstype_to_typeindex_tree::*; pub use addresstype_to_vec::*; use height_to_addresstype_to_vec::*; use range_map::*; -use r#trait::CohortVecs; +use r#trait::*; pub use withaddressdatasource::WithAddressDataSource; const VERSION: Version = Version::new(18); @@ -69,8 +69,19 @@ pub struct Vecs { pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount, pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount, pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount, - pub utxo_vecs: utxo_cohorts::Vecs, - pub address_vecs: address_cohorts::Vecs, + pub utxo_cohorts: utxo_cohorts::Vecs, + pub address_cohorts: address_cohorts::Vecs, + + pub p2pk33addressindex_to_anyaddressindex: IndexedVec, + pub p2pk65addressindex_to_anyaddressindex: IndexedVec, + pub p2pkhaddressindex_to_anyaddressindex: IndexedVec, + pub p2shaddressindex_to_anyaddressindex: IndexedVec, + pub p2traddressindex_to_anyaddressindex: IndexedVec, + pub p2wpkhaddressindex_to_anyaddressindex: IndexedVec, + pub p2wshaddressindex_to_anyaddressindex: IndexedVec, + pub p2aaddressindex_to_anyaddressindex: IndexedVec, + pub loadedaddressindex_to_loadedaddressdata: IndexedVec, + pub emptyaddressindex_to_emptyaddressdata: IndexedVec, pub indexes_to_address_count: ComputedVecsFromHeight, pub indexes_to_empty_address_count: ComputedVecsFromHeight, @@ -426,7 +437,7 @@ impl Vecs { )?, }, ), - utxo_vecs: utxo_cohorts::Vecs::forced_import( + utxo_cohorts: utxo_cohorts::Vecs::forced_import( path, version, computation, @@ -435,7 +446,7 @@ impl Vecs { fetched, &states_path, )?, - address_vecs: address_cohorts::Vecs::forced_import( + address_cohorts: address_cohorts::Vecs::forced_import( path, version, computation, @@ -444,6 +455,68 @@ impl Vecs { fetched, &states_path, )?, + + p2aaddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2pk33addressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2pk65addressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2pkhaddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2shaddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2traddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2wpkhaddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + p2wshaddressindex_to_anyaddressindex: IndexedVec::forced_import( + path, + "anyaddressindex", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + + loadedaddressindex_to_loadedaddressdata: IndexedVec::forced_import( + path, + "loadedaddressdata", + version + VERSION + Version::ZERO, + Format::Raw, + )?, + emptyaddressindex_to_emptyaddressdata: IndexedVec::forced_import( + path, + "emptyaddressdata", + version + VERSION + Version::ZERO, + Format::Raw, + )?, }) } @@ -555,11 +628,10 @@ impl Vecs { .map_or(Version::ZERO, |v| v.version()) + height_to_date_fixed.version() + dateindex_to_first_height.version() - + dateindex_to_height_count.version() - + stores.as_slice().into_iter().map(|s| s.version()).sum(); + + dateindex_to_height_count.version(); - let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); - let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs(); + let mut separate_utxo_vecs = self.utxo_cohorts.as_mut_separate_vecs(); + let mut separate_address_vecs = self.address_cohorts.as_mut_separate_vecs(); separate_utxo_vecs .par_iter_mut() @@ -592,7 +664,56 @@ impl Vecs { .unwrap_or_default(), ) .min(chain_state_starting_height) - .min(stores.starting_height()) + .min( + self.p2pk33addressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2pk65addressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2pkhaddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2shaddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2traddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2wpkhaddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2wshaddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.p2aaddressindex_to_anyaddressindex + .height() + .incremented(), + ) + .min( + self.loadedaddressindex_to_loadedaddressdata + .height() + .incremented(), + ) + .min( + self.emptyaddressindex_to_emptyaddressdata + .height() + .incremented(), + ) .min(Height::from(self.height_to_unspendable_supply.len())) .min(Height::from(self.height_to_opreturn_supply.len())) .cmp(&chain_state_starting_height) @@ -627,11 +748,21 @@ impl Vecs { if starting_height.is_zero() { info!("Starting processing utxos from the start"); - // todo!("rollback instead"); + // TODO: rollback instead + chain_state = vec![]; chain_state_starting_height = Height::ZERO; - stores.reset()?; + self.p2pk33addressindex_to_anyaddressindex.reset()?; + self.p2pk65addressindex_to_anyaddressindex.reset()?; + self.p2pkhaddressindex_to_anyaddressindex.reset()?; + self.p2shaddressindex_to_anyaddressindex.reset()?; + self.p2traddressindex_to_anyaddressindex.reset()?; + self.p2wpkhaddressindex_to_anyaddressindex.reset()?; + self.p2wshaddressindex_to_anyaddressindex.reset()?; + self.p2aaddressindex_to_anyaddressindex.reset()?; + self.loadedaddressindex_to_loadedaddressdata.reset()?; + self.emptyaddressindex_to_emptyaddressdata.reset()?; info!("Resetting utxo price maps..."); @@ -688,10 +819,19 @@ impl Vecs { let mut height = starting_height; - let mut addresstype_to_typeindex_to_addressdata = - AddressTypeToTypeIndexTree::>::default(); + let mut addresstype_to_typeindex_to_loadedaddressdata = + AddressTypeToTypeIndexTree::>::default(); let mut addresstype_to_typeindex_to_emptyaddressdata = AddressTypeToTypeIndexTree::>::default(); + let mut addresstypeindex_to_anyaddressindex_mmap_opt = + ByAddressType::>::default(); + let mut anyaddressindex_to_anyaddressdata_mmap_opt = + ByAnyAddress::>::default(); + + self.reset_mmaps_options( + &mut addresstypeindex_to_anyaddressindex_mmap_opt, + &mut anyaddressindex_to_anyaddressdata_mmap_opt, + ); (height.unwrap_to_usize()..height_to_date_fixed.len()) .map(Height::from) @@ -700,12 +840,12 @@ impl Vecs { info!("Processing chain at {height}..."); - self.utxo_vecs + self.utxo_cohorts .as_mut_separate_vecs() .iter_mut() .for_each(|(_, v)| v.state.reset_single_iteration_values()); - self.address_vecs + self.address_cohorts .as_mut_separate_vecs() .iter_mut() .for_each(|(_, v)| v.state.reset_single_iteration_values()); @@ -755,10 +895,11 @@ impl Vecs { addresstype_to_typedindex_to_received_data, mut height_to_sent, addresstype_to_typedindex_to_sent_data, - mut stored_addressindexes, + mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, ) = thread::scope(|scope| { scope.spawn(|| { - self.utxo_vecs.tick_tock_next_block(&chain_state, timestamp); + self.utxo_cohorts + .tick_tock_next_block(&chain_state, timestamp); }); let received = scope.spawn(|| { @@ -767,7 +908,8 @@ impl Vecs { let mut addresstype_to_typedindex_to_received_data = AddressTypeToVec::<(TypeIndex, Sats)>::default(); - let mut receiving_addressindexes = AddressTypeToTypeIndexSet::default(); + let mut addresstype_to_typeindex_to_addressdatawithsource = + AddressTypeToTypeIndexTree::default(); let _ = (first_outputindex..first_outputindex + *output_count) .map(OutputIndex::from) @@ -787,10 +929,30 @@ impl Vecs { let typeindex = outputindex_to_typeindex .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); - receiving_addressindexes - .get_mut(output_type) - .unwrap() - .insert(typeindex); + if let Some(addressdata) = Self::get_addressdatawithsource( + output_type, + typeindex, + &first_addressindexes, + &addresstype_to_typeindex_to_loadedaddressdata, + &addresstype_to_typeindex_to_emptyaddressdata, + &addresstypeindex_to_anyaddressindex_mmap_opt, + &anyaddressindex_to_anyaddressdata_mmap_opt, + &self.p2pk33addressindex_to_anyaddressindex, + &self.p2pk65addressindex_to_anyaddressindex, + &self.p2pkhaddressindex_to_anyaddressindex, + &self.p2shaddressindex_to_anyaddressindex, + &self.p2traddressindex_to_anyaddressindex, + &self.p2wpkhaddressindex_to_anyaddressindex, + &self.p2wshaddressindex_to_anyaddressindex, + &self.p2aaddressindex_to_anyaddressindex, + &self.loadedaddressindex_to_loadedaddressdata, + &self.emptyaddressindex_to_emptyaddressdata, + ) { + addresstype_to_typeindex_to_addressdatawithsource + .get_mut(output_type) + .unwrap() + .insert(typeindex, addressdata); + } addresstype_to_typedindex_to_received_data .get_mut(output_type) @@ -803,7 +965,7 @@ impl Vecs { ( transacted, addresstype_to_typedindex_to_received_data, - receiving_addressindexes, + addresstype_to_typeindex_to_addressdatawithsource, ) }); @@ -812,7 +974,8 @@ impl Vecs { let mut addresstype_to_typedindex_to_sent_data = HeightToAddressTypeToVec::<(TypeIndex, Sats)>::default(); - let mut sending_addressindexes = AddressTypeToTypeIndexSet::default(); + let mut sending_addresstype_to_typeindex_to_addressdatawithsource = + AddressTypeToTypeIndexTree::default(); // Skip coinbase let _ = (first_inputindex + 1..first_inputindex + *input_count) @@ -842,10 +1005,30 @@ impl Vecs { let typeindex = outputindex_to_typeindex .unwrap_read(outputindex, &outputindex_to_typeindex_mmap); - sending_addressindexes - .get_mut(input_type) - .unwrap() - .insert(typeindex); + if let Some(addressdata) = Self::get_addressdatawithsource( + input_type, + typeindex, + &first_addressindexes, + &addresstype_to_typeindex_to_loadedaddressdata, + &addresstype_to_typeindex_to_emptyaddressdata, + &addresstypeindex_to_anyaddressindex_mmap_opt, + &anyaddressindex_to_anyaddressdata_mmap_opt, + &self.p2pk33addressindex_to_anyaddressindex, + &self.p2pk65addressindex_to_anyaddressindex, + &self.p2pkhaddressindex_to_anyaddressindex, + &self.p2shaddressindex_to_anyaddressindex, + &self.p2traddressindex_to_anyaddressindex, + &self.p2wpkhaddressindex_to_anyaddressindex, + &self.p2wshaddressindex_to_anyaddressindex, + &self.p2aaddressindex_to_anyaddressindex, + &self.loadedaddressindex_to_loadedaddressdata, + &self.emptyaddressindex_to_emptyaddressdata, + ) { + sending_addresstype_to_typeindex_to_addressdatawithsource + .get_mut(input_type) + .unwrap() + .insert(typeindex, addressdata); + } addresstype_to_typedindex_to_sent_data .entry(prev_height) @@ -860,91 +1043,38 @@ impl Vecs { let ( transacted, addresstype_to_typedindex_to_received_data, - receiving_addressindexes, + receiving_addresstype_to_typeindex_to_addressdatawithsource, ) = received.join().unwrap(); - let stored_addressindexes = - sending_addressindexes.merge(receiving_addressindexes); + let addresstype_to_typeindex_to_addressdatawithsource = + receiving_addresstype_to_typeindex_to_addressdatawithsource + .merge(sending_addresstype_to_typeindex_to_addressdatawithsource); ( transacted, addresstype_to_typedindex_to_received_data, height_to_sent, addresstype_to_typedindex_to_sent_data, - stored_addressindexes, + addresstype_to_typeindex_to_addressdatawithsource, ) }); - let mut stored_addressdata_with_source = stored_addressindexes - .into_typed_vec() - .into_par_iter() - .flat_map(|(_type, set)| { - set.into_par_iter().map(move |index| (_type, index)) - }) - .flat_map(|(_type, index)| { - if *first_addressindexes.get(_type).unwrap() <= index { - Some(( - _type, - index, - WithAddressDataSource::New(AddressData::default()), - )) - } else if addresstype_to_typeindex_to_addressdata - .get(_type) - .unwrap() - .contains_key(&index) - || addresstype_to_typeindex_to_emptyaddressdata - .get(_type) - .unwrap() - .contains_key(&index) - { - None - } else if let Some(addressdata) = - stores.get_addressdata(_type, index).unwrap() - { - Some(( - _type, - index, - WithAddressDataSource::FromAddressDataStore(addressdata), - )) - } else if let Some(emptyaddressdata) = - stores.get_emptyaddressdata(_type, index).unwrap() - { - Some(( - _type, - index, - WithAddressDataSource::FromEmptyAddressDataStore( - emptyaddressdata.into(), - ), - )) - } else { - unreachable!() - } - }) - .fold( - AddressTypeToTypeIndexTree::default, - |mut tree, (_type, index, addressdata)| { - tree.get_mut(_type).unwrap().insert(index, addressdata); - tree - }, - ) - .reduce(AddressTypeToTypeIndexTree::default, |a, b| a.merge(b)); - thread::scope(|scope| { scope.spawn(|| { addresstype_to_typedindex_to_received_data.process_received( - &mut self.address_vecs, - &mut addresstype_to_typeindex_to_addressdata, + &mut self.address_cohorts, + &mut addresstype_to_typeindex_to_loadedaddressdata, &mut addresstype_to_typeindex_to_emptyaddressdata, price, &mut addresstype_to_address_count, &mut addresstype_to_empty_address_count, - &mut stored_addressdata_with_source, + &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, ); addresstype_to_typedindex_to_sent_data .process_sent( - &mut self.address_vecs, - &mut addresstype_to_typeindex_to_addressdata, + &mut self.address_cohorts, + &mut addresstype_to_typeindex_to_loadedaddressdata, &mut addresstype_to_typeindex_to_emptyaddressdata, price, &mut addresstype_to_address_count, @@ -953,7 +1083,7 @@ impl Vecs { &height_to_timestamp_fixed_vec, height, timestamp, - &mut stored_addressdata_with_source, + &mut stored_or_new_addresstype_to_typeindex_to_addressdatawithsource, ) .unwrap(); }); @@ -993,18 +1123,18 @@ impl Vecs { timestamp, }); - self.utxo_vecs.receive(transacted, height, price); + self.utxo_cohorts.receive(transacted, height, price); - self.utxo_vecs.send(height_to_sent, &mut chain_state); + self.utxo_cohorts.send(height_to_sent, &mut chain_state); }); - let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); + let mut separate_utxo_vecs = self.utxo_cohorts.as_mut_separate_vecs(); separate_utxo_vecs .iter_mut() .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; - let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs(); + let mut separate_address_vecs = self.address_cohorts.as_mut_separate_vecs(); separate_address_vecs .iter_mut() @@ -1062,16 +1192,22 @@ impl Vecs { self.flush_states(height, &chain_state, exit)?; // Maybe keep some from the end for both - let addresstype_to_typeindex_to_addressdata_to_consume = - mem::take(&mut addresstype_to_typeindex_to_addressdata); + let addresstype_to_typeindex_to_loadedaddressdata_to_consume = + mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata); let addresstype_to_typeindex_to_emptyaddressdata_to_consume = mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata); - stores.commit( - height, - addresstype_to_typeindex_to_addressdata_to_consume, - addresstype_to_typeindex_to_emptyaddressdata_to_consume, - )?; + // stores.commit( + // height, + // addresstype_to_typeindex_to_loadedaddressdata_to_consume, + // addresstype_to_typeindex_to_emptyaddressdata_to_consume, + // )?; + + self.reset_mmaps_options( + &mut addresstypeindex_to_anyaddressindex_mmap_opt, + &mut anyaddressindex_to_anyaddressdata_mmap_opt, + ); + exit.release(); } @@ -1083,11 +1219,11 @@ impl Vecs { info!("Flushing..."); self.flush_states(height, &chain_state, exit)?; - stores.commit( - height, - mem::take(&mut addresstype_to_typeindex_to_addressdata), - mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), - )?; + // stores.commit( + // height, + // mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata), + // mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), + // )?; } else { exit.block(); } @@ -1096,12 +1232,12 @@ impl Vecs { thread::scope(|scope| { scope.spawn(|| { - self.utxo_vecs + self.utxo_cohorts .compute_overlapping_vecs(starting_indexes, exit) .unwrap(); }); scope.spawn(|| { - self.address_vecs + self.address_cohorts .compute_overlapping_vecs(starting_indexes, exit) .unwrap(); }); @@ -1160,13 +1296,13 @@ impl Vecs { &self.addresstype_to_height_to_empty_address_count, )?; - self.utxo_vecs + self.utxo_cohorts .as_mut_vecs() .into_iter() .map(|(_, v)| v) .map(Either::Left) .chain( - self.address_vecs + self.address_cohorts .as_mut_vecs() .into_iter() .map(|(_, v)| v) @@ -1185,18 +1321,24 @@ impl Vecs { info!("Computing rest part 2..."); - let height_to_supply = self.utxo_vecs.all.1.height_to_supply_value.bitcoin.clone(); + let height_to_supply = self + .utxo_cohorts + .all + .1 + .height_to_supply_value + .bitcoin + .clone(); let dateindex_to_supply = self - .utxo_vecs + .utxo_cohorts .all .1 .indexes_to_supply .bitcoin .dateindex .clone(); - let height_to_realized_cap = self.utxo_vecs.all.1.height_to_realized_cap.clone(); + let height_to_realized_cap = self.utxo_cohorts.all.1.height_to_realized_cap.clone(); let dateindex_to_realized_cap = self - .utxo_vecs + .utxo_cohorts .all .1 .indexes_to_realized_cap @@ -1206,13 +1348,13 @@ impl Vecs { let height_to_realized_cap_ref = height_to_realized_cap.as_ref(); let dateindex_to_realized_cap_ref = dateindex_to_realized_cap.as_ref(); - self.utxo_vecs + self.utxo_cohorts .as_mut_vecs() .into_iter() .map(|(_, v)| v) .map(Either::Left) .chain( - self.address_vecs + self.address_cohorts .as_mut_vecs() .into_iter() .map(|(_, v)| v) @@ -1269,17 +1411,182 @@ impl Vecs { Ok(()) } + #[allow(clippy::too_many_arguments)] + fn get_addressdatawithsource( + address_type: OutputType, + typeindex: TypeIndex, + first_addressindexes: &ByAddressType, + addresstype_to_typeindex_to_loadedaddressdata: &AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + addresstype_to_typeindex_to_emptyaddressdata: &AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + addresstypeindex_to_anyaddressindex_mmap_opt: &ByAddressType>, + anyaddressindex_to_anyaddressdata_mmap_opt: &ByAnyAddress>, + p2pk33addressindex_to_anyaddressindex: &IndexedVec, + p2pk65addressindex_to_anyaddressindex: &IndexedVec, + p2pkhaddressindex_to_anyaddressindex: &IndexedVec, + p2shaddressindex_to_anyaddressindex: &IndexedVec, + p2traddressindex_to_anyaddressindex: &IndexedVec, + p2wpkhaddressindex_to_anyaddressindex: &IndexedVec, + p2wshaddressindex_to_anyaddressindex: &IndexedVec, + p2aaddressindex_to_anyaddressindex: &IndexedVec, + loadedaddressindex_to_loadedaddressdata: &IndexedVec, + emptyaddressindex_to_emptyaddressdata: &IndexedVec, + ) -> Option> { + if *first_addressindexes.get(address_type).unwrap() <= typeindex { + return Some(WithAddressDataSource::New(LoadedAddressData::default())); + } + + if addresstype_to_typeindex_to_loadedaddressdata + .get(address_type) + .unwrap() + .contains_key(&typeindex) + || addresstype_to_typeindex_to_emptyaddressdata + .get(address_type) + .unwrap() + .contains_key(&typeindex) + { + return None; + } + + let mmap = addresstypeindex_to_anyaddressindex_mmap_opt + .get_unwrap(address_type) + .as_ref() + .unwrap(); + + let anyaddressindex = match address_type { + OutputType::P2PK33 => { + p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2PK65 => { + p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2PKH => { + p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2SH => { + p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2TR => { + p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2WPKH => { + p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2WSH => { + p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + OutputType::P2A => { + p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + } + _ => unreachable!(), + } + .unwrap() + .unwrap() + .into_owned(); + + Some(match anyaddressindex.to_enum() { + AnyAddressDataIndexEnum::Loaded(index) => { + let mmap = anyaddressindex_to_anyaddressdata_mmap_opt + .loaded + .as_ref() + .unwrap(); + + let loadedaddressdata = loadedaddressindex_to_loadedaddressdata + .get_or_read(index, mmap) + .unwrap() + .unwrap() + .into_owned(); + + WithAddressDataSource::FromLoadedAddressDataVec(loadedaddressdata) + } + AnyAddressDataIndexEnum::Empty(index) => { + let mmap = anyaddressindex_to_anyaddressdata_mmap_opt + .empty + .as_ref() + .unwrap(); + + let emptyaddressdata = emptyaddressindex_to_emptyaddressdata + .get_or_read(index, mmap) + .unwrap() + .unwrap() + .into_owned(); + + WithAddressDataSource::FromEmptyAddressDataVec(emptyaddressdata.into()) + } + }) + } + + fn reset_mmaps_options( + &self, + addresstypeindex_to_anyaddressindex_mmap_opt: &mut ByAddressType>, + anyaddressindex_to_anyaddressdata_mmap_opt: &mut ByAnyAddress>, + ) { + addresstypeindex_to_anyaddressindex_mmap_opt.p2pk65.replace( + self.p2pk65addressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2pk33.replace( + self.p2pk33addressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2pkh.replace( + self.p2pkhaddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2sh.replace( + self.p2shaddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2wpkh.replace( + self.p2wpkhaddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2wsh.replace( + self.p2wshaddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2tr.replace( + self.p2traddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + addresstypeindex_to_anyaddressindex_mmap_opt.p2a.replace( + self.p2aaddressindex_to_anyaddressindex + .create_mmap() + .unwrap(), + ); + anyaddressindex_to_anyaddressdata_mmap_opt.loaded.replace( + self.loadedaddressindex_to_loadedaddressdata + .create_mmap() + .unwrap(), + ); + anyaddressindex_to_anyaddressdata_mmap_opt.empty.replace( + self.emptyaddressindex_to_emptyaddressdata + .create_mmap() + .unwrap(), + ); + } + fn flush_states( &mut self, height: Height, chain_state: &[BlockState], exit: &Exit, ) -> Result<()> { - self.utxo_vecs + self.utxo_cohorts .as_mut_separate_vecs() .par_iter_mut() .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; - self.address_vecs + self.address_cohorts .as_mut_separate_vecs() .par_iter_mut() .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; @@ -1294,6 +1601,17 @@ impl Vecs { .into_iter() .try_for_each(|v| v.safe_flush(exit))?; + self.p2pk33addressindex_to_anyaddressindex.flush(height)?; + self.p2pk65addressindex_to_anyaddressindex.flush(height)?; + self.p2pkhaddressindex_to_anyaddressindex.flush(height)?; + self.p2shaddressindex_to_anyaddressindex.flush(height)?; + self.p2traddressindex_to_anyaddressindex.flush(height)?; + self.p2wpkhaddressindex_to_anyaddressindex.flush(height)?; + self.p2wshaddressindex_to_anyaddressindex.flush(height)?; + self.p2aaddressindex_to_anyaddressindex.flush(height)?; + self.loadedaddressindex_to_loadedaddressdata.flush(height)?; + self.emptyaddressindex_to_emptyaddressdata.flush(height)?; + self.chain_state.truncate_if_needed(Height::ZERO)?; chain_state.iter().for_each(|block_state| { self.chain_state.push(block_state.supply.clone()); @@ -1305,12 +1623,12 @@ impl Vecs { pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ - self.utxo_vecs + self.utxo_cohorts .vecs() .into_iter() .flat_map(|v| v.vecs()) .collect::>(), - self.address_vecs + self.address_cohorts .vecs() .into_iter() .flat_map(|v| v.vecs()) @@ -1347,8 +1665,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { fn process_received( mut self, vecs: &mut address_cohorts::Vecs, - addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< - WithAddressDataSource, + addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, >, addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree< WithAddressDataSource, @@ -1356,8 +1674,8 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { price: Option, addresstype_to_address_count: &mut ByAddressType, addresstype_to_empty_address_count: &mut ByAddressType, - stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree< - WithAddressDataSource, + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, >, ) { self.into_typed_vec().into_iter().for_each(|(_type, vec)| { @@ -1365,7 +1683,7 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { let mut is_new = false; let mut from_any_empty = false; - let addressdata_withsource = addresstype_to_typeindex_to_addressdata + let addressdata_withsource = addresstype_to_typeindex_to_loadedaddressdata .get_mut(_type) .unwrap() .entry(type_index) @@ -1379,11 +1697,12 @@ impl AddressTypeToVec<(TypeIndex, Sats)> { ad.into() }) .unwrap_or_else(|| { - let addressdata = stored_addressdata_with_source - .get_mut(_type) - .unwrap() - .remove(&type_index) - .unwrap(); + let addressdata = + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource + .get_mut(_type) + .unwrap() + .remove(&type_index) + .unwrap(); is_new = addressdata.is_new(); from_any_empty = addressdata.is_from_emptyaddressdata(); addressdata @@ -1436,8 +1755,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { fn process_sent( self, vecs: &mut address_cohorts::Vecs, - addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< - WithAddressDataSource, + addresstype_to_typeindex_to_loadedaddressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, >, addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree< WithAddressDataSource, @@ -1449,8 +1768,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { height_to_timestamp_fixed_vec: &[Timestamp], height: Height, timestamp: Timestamp, - stored_addressdata_with_source: &mut AddressTypeToTypeIndexTree< - WithAddressDataSource, + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, >, ) -> Result<()> { self.0.into_iter().try_for_each(|(prev_height, mut v)| { @@ -1473,14 +1792,15 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { v.into_typed_vec().into_iter().try_for_each(|(_type, vec)| { vec.into_iter().try_for_each(|(type_index, value)| { - let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata - .get_mut(_type) - .unwrap(); + let typeindex_to_loadedaddressdata = + addresstype_to_typeindex_to_loadedaddressdata + .get_mut(_type) + .unwrap(); - let addressdata_withsource = typeindex_to_addressdata + let addressdata_withsource = typeindex_to_loadedaddressdata .entry(type_index) .or_insert_with(|| { - stored_addressdata_with_source + stored_or_new_addresstype_to_typeindex_to_addressdatawithsource .get_mut(_type) .unwrap() .remove(&type_index) @@ -1515,7 +1835,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> { (*addresstype_to_address_count.get_mut(_type).unwrap()) -= 1; (*addresstype_to_empty_address_count.get_mut(_type).unwrap()) += 1; - let addressdata = typeindex_to_addressdata.remove(&type_index).unwrap(); + let addressdata = + typeindex_to_loadedaddressdata.remove(&type_index).unwrap(); addresstype_to_typeindex_to_emptyaddressdata .get_mut(_type) diff --git a/crates/brk_computer/src/stateful/withaddressdatasource.rs b/crates/brk_computer/src/stateful/withaddressdatasource.rs index 92dcc1b56..4518b4156 100644 --- a/crates/brk_computer/src/stateful/withaddressdatasource.rs +++ b/crates/brk_computer/src/stateful/withaddressdatasource.rs @@ -1,10 +1,10 @@ -use brk_core::{AddressData, EmptyAddressData}; +use brk_core::{EmptyAddressData, LoadedAddressData}; #[derive(Debug)] pub enum WithAddressDataSource { New(T), - FromAddressDataStore(T), - FromEmptyAddressDataStore(T), + FromLoadedAddressDataVec(T), + FromEmptyAddressDataVec(T), } impl WithAddressDataSource { @@ -13,49 +13,53 @@ impl WithAddressDataSource { } pub fn is_from_addressdata(&self) -> bool { - matches!(self, Self::FromAddressDataStore(_)) + matches!(self, Self::FromLoadedAddressDataVec(_)) } pub fn is_from_emptyaddressdata(&self) -> bool { - matches!(self, Self::FromEmptyAddressDataStore(_)) + matches!(self, Self::FromEmptyAddressDataVec(_)) } pub fn deref(&self) -> &T { match self { Self::New(v) => v, - Self::FromAddressDataStore(v) => v, - Self::FromEmptyAddressDataStore(v) => v, + Self::FromLoadedAddressDataVec(v) => v, + Self::FromEmptyAddressDataVec(v) => v, } } pub fn deref_mut(&mut self) -> &mut T { match self { Self::New(v) => v, - Self::FromAddressDataStore(v) => v, - Self::FromEmptyAddressDataStore(v) => v, + Self::FromLoadedAddressDataVec(v) => v, + Self::FromEmptyAddressDataVec(v) => v, } } } -impl From> for WithAddressDataSource { +impl From> for WithAddressDataSource { fn from(value: WithAddressDataSource) -> Self { match value { WithAddressDataSource::New(v) => Self::New(v.into()), - WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()), - WithAddressDataSource::FromEmptyAddressDataStore(v) => { - Self::FromEmptyAddressDataStore(v.into()) + WithAddressDataSource::FromLoadedAddressDataVec(v) => { + Self::FromLoadedAddressDataVec(v.into()) + } + WithAddressDataSource::FromEmptyAddressDataVec(v) => { + Self::FromEmptyAddressDataVec(v.into()) } } } } -impl From> for WithAddressDataSource { - fn from(value: WithAddressDataSource) -> Self { +impl From> for WithAddressDataSource { + fn from(value: WithAddressDataSource) -> Self { match value { WithAddressDataSource::New(v) => Self::New(v.into()), - WithAddressDataSource::FromAddressDataStore(v) => Self::FromAddressDataStore(v.into()), - WithAddressDataSource::FromEmptyAddressDataStore(v) => { - Self::FromEmptyAddressDataStore(v.into()) + WithAddressDataSource::FromLoadedAddressDataVec(v) => { + Self::FromLoadedAddressDataVec(v.into()) + } + WithAddressDataSource::FromEmptyAddressDataVec(v) => { + Self::FromEmptyAddressDataVec(v.into()) } } } diff --git a/crates/brk_computer/src/states/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs index 41e5a3749..3778267e3 100644 --- a/crates/brk_computer/src/states/cohorts/address.rs +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -1,6 +1,6 @@ use std::path::Path; -use brk_core::{AddressData, Dollars, Height, Result, Sats}; +use brk_core::{Dollars, Height, LoadedAddressData, Result, Sats}; use crate::SupplyState; @@ -35,7 +35,7 @@ impl AddressCohortState { #[allow(clippy::too_many_arguments)] pub fn send( &mut self, - addressdata: &mut AddressData, + addressdata: &mut LoadedAddressData, value: Sats, current_price: Option, prev_price: Option, @@ -72,7 +72,12 @@ impl AddressCohortState { Ok(()) } - pub fn receive(&mut self, address_data: &mut AddressData, value: Sats, price: Option) { + pub fn receive( + &mut self, + address_data: &mut LoadedAddressData, + value: Sats, + price: Option, + ) { let compute_price = price.is_some(); let prev_realized_price = compute_price.then(|| address_data.realized_price()); @@ -96,7 +101,7 @@ impl AddressCohortState { ); } - pub fn add(&mut self, addressdata: &AddressData) { + pub fn add(&mut self, addressdata: &LoadedAddressData) { self.address_count += 1; self.inner.increment_( &addressdata.into(), @@ -105,7 +110,7 @@ impl AddressCohortState { ); } - pub fn subtract(&mut self, addressdata: &AddressData) { + pub fn subtract(&mut self, addressdata: &LoadedAddressData) { self.address_count = self.address_count.checked_sub(1).unwrap(); self.inner.decrement_( &addressdata.into(), diff --git a/crates/brk_computer/src/states/supply.rs b/crates/brk_computer/src/states/supply.rs index 4c26830d3..c0e438a36 100644 --- a/crates/brk_computer/src/states/supply.rs +++ b/crates/brk_computer/src/states/supply.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, SubAssign}; -use brk_core::{AddressData, CheckedSub, Sats}; +use brk_core::{CheckedSub, LoadedAddressData, Sats}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -40,8 +40,8 @@ impl SubAssign<&SupplyState> for SupplyState { } } -impl From<&AddressData> for SupplyState { - fn from(value: &AddressData) -> Self { +impl From<&LoadedAddressData> for SupplyState { + fn from(value: &LoadedAddressData) -> Self { Self { utxos: value.outputs_len as usize, value: value.amount(), diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_core/src/groups/by_address_type.rs index 394dca645..17289bcfb 100644 --- a/crates/brk_core/src/groups/by_address_type.rs +++ b/crates/brk_core/src/groups/by_address_type.rs @@ -19,6 +19,10 @@ pub struct ByAddressType { } impl ByAddressType { + pub fn get_unwrap(&self, address_type: OutputType) -> &T { + self.get(address_type).unwrap() + } + pub fn get(&self, address_type: OutputType) -> Option<&T> { match address_type { OutputType::P2PK65 => Some(&self.p2pk65), diff --git a/crates/brk_core/src/groups/by_any_address.rs b/crates/brk_core/src/groups/by_any_address.rs new file mode 100644 index 000000000..13591e9c4 --- /dev/null +++ b/crates/brk_core/src/groups/by_any_address.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Default)] +pub struct ByAnyAddress { + pub loaded: T, + pub empty: T, +} diff --git a/crates/brk_core/src/groups/mod.rs b/crates/brk_core/src/groups/mod.rs index 8ca92df9f..972602dab 100644 --- a/crates/brk_core/src/groups/mod.rs +++ b/crates/brk_core/src/groups/mod.rs @@ -2,6 +2,7 @@ mod address; mod by_address_type; mod by_age_range; mod by_amount_range; +mod by_any_address; mod by_epoch; mod by_ge_amount; mod by_lt_amount; @@ -18,6 +19,7 @@ pub use address::*; pub use by_address_type::*; pub use by_age_range::*; pub use by_amount_range::*; +pub use by_any_address::*; pub use by_epoch::*; pub use by_ge_amount::*; pub use by_lt_amount::*; diff --git a/crates/brk_core/src/structs/anyaddressdataindex.rs b/crates/brk_core/src/structs/anyaddressdataindex.rs deleted file mode 100644 index bc0ac6120..000000000 --- a/crates/brk_core/src/structs/anyaddressdataindex.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::ops::Add; - -use byteview::ByteView; -use derive_deref::{Deref, DerefMut}; -use serde::Serialize; -use zerocopy::{FromBytes, IntoBytes}; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; - -use crate::{CheckedSub, Printable, TypeIndex}; - -#[derive( - Debug, - PartialEq, - Eq, - PartialOrd, - Ord, - Clone, - Copy, - Deref, - DerefMut, - Default, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] -pub struct AnyAddressDataIndex(TypeIndex); -// impl From for AddressDataIndex { -// fn from(value: TypeIndex) -> Self { -// Self(value) -// } -// } -// impl From for TypeIndex { -// fn from(value: AddressDataIndex) -> Self { -// value.0 -// } -// } -// impl From for u32 { -// fn from(value: AddressDataIndex) -> Self { -// Self::from(*value) -// } -// } -// impl From for AddressDataIndex { -// fn from(value: u32) -> Self { -// Self(TypeIndex::from(value)) -// } -// } -// impl From for usize { -// fn from(value: AddressDataIndex) -> Self { -// Self::from(*value) -// } -// } -// impl From for AddressDataIndex { -// fn from(value: usize) -> Self { -// Self(TypeIndex::from(value)) -// } -// } -// impl Add for AddressDataIndex { -// type Output = Self; -// fn add(self, rhs: usize) -> Self::Output { -// Self(*self + rhs) -// } -// } -// impl CheckedSub for AddressDataIndex { -// fn checked_sub(self, rhs: Self) -> Option { -// self.0.checked_sub(rhs.0).map(Self) -// } -// } - -// impl Printable for AddressDataIndex { -// fn to_string() -> &'static str { -// "p2pk33addressindex" -// } - -// fn to_possible_strings() -> &'static [&'static str] { -// &["addr", "p2pk33addr", "p2pk33addressindex"] -// } -// } - -// impl From for AddressDataIndex { -// fn from(value: ByteView) -> Self { -// Self::read_from_bytes(&value).unwrap() -// } -// } -// impl From for ByteView { -// fn from(value: AddressDataIndex) -> Self { -// Self::from(&value) -// } -// } -// impl From<&AddressDataIndex> for ByteView { -// fn from(value: &AddressDataIndex) -> Self { -// Self::new(value.as_bytes()) -// } -// } diff --git a/crates/brk_core/src/structs/anyaddressindex.rs b/crates/brk_core/src/structs/anyaddressindex.rs new file mode 100644 index 000000000..668efbf85 --- /dev/null +++ b/crates/brk_core/src/structs/anyaddressindex.rs @@ -0,0 +1,48 @@ +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{ + TypeIndex, + structs::{EmptyAddressIndex, LoadedAddressIndex}, +}; + +const MIN_EMPTY_INDEX: u32 = u32::MAX - 4_000_000_000; + +#[derive( + Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, +)] +pub struct AnyAddressIndex(TypeIndex); + +impl AnyAddressIndex { + pub fn to_enum(&self) -> AnyAddressDataIndexEnum { + AnyAddressDataIndexEnum::from(*self) + } +} + +impl Serialize for AnyAddressIndex { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.to_enum().serialize(serializer) + } +} + +#[derive(Debug, Serialize)] +pub enum AnyAddressDataIndexEnum { + Loaded(LoadedAddressIndex), + Empty(EmptyAddressIndex), +} + +impl From for AnyAddressDataIndexEnum { + fn from(value: AnyAddressIndex) -> Self { + let uvalue = u32::from(value.0); + if uvalue >= MIN_EMPTY_INDEX { + Self::Empty(EmptyAddressIndex::from(TypeIndex::from( + uvalue - MIN_EMPTY_INDEX, + ))) + } else { + Self::Loaded(LoadedAddressIndex::from(value.0)) + } + } +} diff --git a/crates/brk_core/src/structs/emptyaddressdata.rs b/crates/brk_core/src/structs/emptyaddressdata.rs index d081ed0e8..59eaa41eb 100644 --- a/crates/brk_core/src/structs/emptyaddressdata.rs +++ b/crates/brk_core/src/structs/emptyaddressdata.rs @@ -1,22 +1,23 @@ use byteview::ByteView; +use serde::Serialize; use zerocopy::{FromBytes, IntoBytes}; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::{AddressData, Sats}; +use crate::{LoadedAddressData, Sats}; -#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout)] +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] pub struct EmptyAddressData { pub transfered: Sats, } -impl From for EmptyAddressData { - fn from(value: AddressData) -> Self { +impl From for EmptyAddressData { + fn from(value: LoadedAddressData) -> Self { Self::from(&value) } } -impl From<&AddressData> for EmptyAddressData { - fn from(value: &AddressData) -> Self { +impl From<&LoadedAddressData> for EmptyAddressData { + fn from(value: &LoadedAddressData) -> Self { if value.sent != value.received { dbg!(&value); panic!("Trying to convert not empty wallet to empty !"); diff --git a/crates/brk_core/src/structs/emptyaddressindex.rs b/crates/brk_core/src/structs/emptyaddressindex.rs new file mode 100644 index 000000000..cbba71786 --- /dev/null +++ b/crates/brk_core/src/structs/emptyaddressindex.rs @@ -0,0 +1,62 @@ +use std::ops::Add; + +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{CheckedSub, Printable, TypeIndex}; + +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] +pub struct EmptyAddressIndex(TypeIndex); + +impl From for EmptyAddressIndex { + fn from(value: TypeIndex) -> Self { + Self(value) + } +} + +impl From for EmptyAddressIndex { + fn from(value: usize) -> Self { + Self(TypeIndex::from(value)) + } +} + +impl From for usize { + fn from(value: EmptyAddressIndex) -> Self { + usize::from(value.0) + } +} + +impl Add for EmptyAddressIndex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} +impl CheckedSub for EmptyAddressIndex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} +impl Printable for EmptyAddressIndex { + fn to_string() -> &'static str { + "emptyaddressindex" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["emptyaddr", "emptyaddressindex"] + } +} diff --git a/crates/brk_core/src/structs/emptyoutputindex.rs b/crates/brk_core/src/structs/emptyoutputindex.rs index 78b037f80..c059f293a 100644 --- a/crates/brk_core/src/structs/emptyoutputindex.rs +++ b/crates/brk_core/src/structs/emptyoutputindex.rs @@ -31,7 +31,7 @@ impl From for EmptyOutputIndex { } impl From for usize { fn from(value: EmptyOutputIndex) -> Self { - Self::from(*value) + Self::from(value.0) } } impl From for EmptyOutputIndex { @@ -42,7 +42,7 @@ impl From for EmptyOutputIndex { impl Add for EmptyOutputIndex { type Output = Self; fn add(self, rhs: usize) -> Self::Output { - Self(*self + rhs) + Self(self.0 + rhs) } } impl CheckedSub for EmptyOutputIndex { diff --git a/crates/brk_core/src/structs/addressdata.rs b/crates/brk_core/src/structs/loadedaddressdata.rs similarity index 79% rename from crates/brk_core/src/structs/addressdata.rs rename to crates/brk_core/src/structs/loadedaddressdata.rs index c80148ee6..ca33699d9 100644 --- a/crates/brk_core/src/structs/addressdata.rs +++ b/crates/brk_core/src/structs/loadedaddressdata.rs @@ -1,18 +1,21 @@ use byteview::ByteView; +use serde::Serialize; use zerocopy::{FromBytes, IntoBytes}; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::{Bitcoin, CheckedSub, Dollars, EmptyAddressData, Error, Result, Sats}; -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)] #[repr(C)] -pub struct AddressData { +pub struct LoadedAddressData { pub sent: Sats, pub received: Sats, pub realized_cap: Dollars, pub outputs_len: u32, + padding: u32, } -impl AddressData { +impl LoadedAddressData { pub fn amount(&self) -> Sats { (u64::from(self.received) - u64::from(self.sent)).into() } @@ -55,23 +58,24 @@ impl AddressData { } } -impl From for AddressData { +impl From for LoadedAddressData { fn from(value: EmptyAddressData) -> Self { Self::from(&value) } } -impl From<&EmptyAddressData> for AddressData { +impl From<&EmptyAddressData> for LoadedAddressData { fn from(value: &EmptyAddressData) -> Self { Self { sent: value.transfered, received: value.transfered, realized_cap: Dollars::ZERO, outputs_len: 0, + padding: 0, } } } -impl From for AddressData { +impl From for LoadedAddressData { fn from(value: ByteView) -> Self { Self { // MUST be same order as impl From<&AddressData> for ByteView @@ -79,16 +83,17 @@ impl From for AddressData { received: Sats::read_from_bytes(&value[8..16]).unwrap(), realized_cap: Dollars::read_from_bytes(&value[16..24]).unwrap(), outputs_len: u32::read_from_bytes(&value[24..]).unwrap(), + padding: 0, } } } -impl From for ByteView { - fn from(value: AddressData) -> Self { +impl From for ByteView { + fn from(value: LoadedAddressData) -> Self { Self::from(&value) } } -impl From<&AddressData> for ByteView { - fn from(value: &AddressData) -> Self { +impl From<&LoadedAddressData> for ByteView { + fn from(value: &LoadedAddressData) -> Self { Self::new( &[ value.sent.as_bytes(), diff --git a/crates/brk_core/src/structs/loadedaddressindex.rs b/crates/brk_core/src/structs/loadedaddressindex.rs new file mode 100644 index 000000000..a4abff81b --- /dev/null +++ b/crates/brk_core/src/structs/loadedaddressindex.rs @@ -0,0 +1,60 @@ +use std::ops::Add; + +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::{CheckedSub, Printable, TypeIndex}; + +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] +pub struct LoadedAddressIndex(TypeIndex); + +impl From for LoadedAddressIndex { + fn from(value: TypeIndex) -> Self { + Self(value) + } +} + +impl From for LoadedAddressIndex { + fn from(value: usize) -> Self { + Self(TypeIndex::from(value)) + } +} +impl From for usize { + fn from(value: LoadedAddressIndex) -> Self { + usize::from(value.0) + } +} +impl Add for LoadedAddressIndex { + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} +impl CheckedSub for LoadedAddressIndex { + fn checked_sub(self, rhs: Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } +} +impl Printable for LoadedAddressIndex { + fn to_string() -> &'static str { + "loadedaddressindex" + } + + fn to_possible_strings() -> &'static [&'static str] { + &["loadedaddr", "loadedaddressindex"] + } +} diff --git a/crates/brk_core/src/structs/mod.rs b/crates/brk_core/src/structs/mod.rs index 49628b181..6a6109ecd 100644 --- a/crates/brk_core/src/structs/mod.rs +++ b/crates/brk_core/src/structs/mod.rs @@ -1,7 +1,6 @@ mod addressbytes; mod addressbyteshash; -mod addressdata; -mod anyaddressdataindex; +mod anyaddressindex; mod bitcoin; mod blockhash; mod blockhashprefix; @@ -12,11 +11,14 @@ mod decadeindex; mod difficultyepoch; mod dollars; mod emptyaddressdata; +mod emptyaddressindex; mod emptyoutputindex; mod feerate; mod halvingepoch; mod height; mod inputindex; +mod loadedaddressdata; +mod loadedaddressindex; mod monthindex; mod ohlc; mod opreturnindex; @@ -59,8 +61,7 @@ mod yearindex; pub use addressbytes::*; pub use addressbyteshash::*; -pub use addressdata::*; -pub use anyaddressdataindex::*; +pub use anyaddressindex::*; pub use bitcoin::*; pub use blockhash::*; pub use blockhashprefix::*; @@ -71,11 +72,14 @@ pub use decadeindex::*; pub use difficultyepoch::*; pub use dollars::*; pub use emptyaddressdata::*; +pub use emptyaddressindex::*; pub use emptyoutputindex::*; pub use feerate::*; pub use halvingepoch::*; pub use height::*; pub use inputindex::*; +pub use loadedaddressdata::*; +pub use loadedaddressindex::*; pub use monthindex::*; pub use ohlc::*; pub use opreturnindex::*; diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 5ecac38dc..e9bf3df43 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -29,7 +29,7 @@ pub use params::{IdParam, Params, ParamsOpt}; pub use table::Tabled; use vecs::Vecs; -use crate::{IdToVec, IndexToVec}; +use crate::vecs::{IdToVec, IndexToVec}; pub struct Interface<'a> { vecs: Vecs<'a>, diff --git a/crates/brk_interface/src/vecs.rs b/crates/brk_interface/src/vecs.rs index 93ea6c2c0..8f081b960 100644 --- a/crates/brk_interface/src/vecs.rs +++ b/crates/brk_interface/src/vecs.rs @@ -101,33 +101,6 @@ impl<'a> Vecs<'a> { fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { let name = vec.name(); let serialized_index = vec.index_type_to_string(); - // let split = name.split("_to_").collect::>(); - // if split.len() != 1 - // && !(split.len() == 2 - // && split.first().is_some_and(|s| { - // s == &"up" - // || s == &"start" - // || s.ends_with("relative") - // || s.starts_with("from") - // || s == &"cumulative_up" - // || s.starts_with("cumulative_start") - // || s.starts_with("cumulative_from") - // || s == &"activity" - // })) - // && !(split.len() == 3 - // && split.first().is_some_and(|s| { - // s == &"up" - // || s == &"start" - // || s.starts_with("from") - // || s == &"cumulative_up" - // || s == &"cumulative_start" - // || s.starts_with("cumulative_from") - // }) - // && split.get(1).is_some_and(|s| s.ends_with("relative"))) - // { - // dbg!((&serialized_index, &name, &split)); - // unreachable!(); - // } let index = Index::try_from(serialized_index) .inspect_err(|_| { dbg!(&serialized_index); diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index ff2247067..aff73767c 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -43,11 +43,15 @@ pub struct Parser { } impl Parser { - pub fn new(blocks_dir: PathBuf, outputs_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self { - Self { + pub fn new( + blocks_dir: PathBuf, + outputs_dir: PathBuf, + rpc: &'static bitcoincore_rpc::Client, + ) -> Self { + Self { blocks_dir, - outputs_dir: outputs_dir, - rpc + outputs_dir, + rpc, } } diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index dccb39b02..7a79de218 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -1,6 +1,6 @@ use std::{fs, path::Path}; -use brk_core::{DateIndex, Height, Printable, Version}; +use brk_core::{DateIndex, Height, Version}; use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator}; type I = DateIndex; @@ -104,7 +104,7 @@ fn main() -> Result<(), Box> { let mmap = vec.create_mmap()?; dbg!(vec.take(10.into(), &mmap)?); - dbg!(vec.get_or_read(10.into(), &mmap)); + dbg!(vec.get_or_read(10.into(), &mmap)?); dbg!(vec.holes()); vec.flush()?; dbg!(vec.holes()); @@ -118,11 +118,11 @@ fn main() -> Result<(), Box> { dbg!(vec.holes()); dbg!(vec.get_or_read(10.into(), &mmap)?); - vec.update(10.into(), 10); - vec.update(0.into(), 10); + vec.update(10.into(), 10)?; + vec.update(0.into(), 10)?; dbg!( vec.holes(), - vec.get_or_read(0.into(), &mmap), + vec.get_or_read(0.into(), &mmap)?, vec.get_or_read(10.into(), &mmap)? ); @@ -130,9 +130,9 @@ fn main() -> Result<(), Box> { } { - let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?; + let vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?; - dbg!(vec.collect()); + dbg!(vec.collect()?); } Ok(()) diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index c3bd11a6f..716d23598 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -57,6 +57,7 @@ where } } + #[allow(unreachable_code, unused_variables)] pub fn import(parent: &Path, name: &str, version: Version) -> Result { panic!("Compressed vecs are a work in progress right now, please use raw vecs instead"); diff --git a/crates/brk_vec/src/variants/indexed.rs b/crates/brk_vec/src/variants/indexed.rs index 6ab8c836b..4b0ba0dc7 100644 --- a/crates/brk_vec/src/variants/indexed.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -62,6 +62,11 @@ where self.0.mut_header().update_height(height); } + pub fn reset(&mut self) -> Result<()> { + self.update_height(Height::ZERO); + self.0.reset() + } + pub fn truncate_if_needed(&mut self, index: I, height: Height) -> Result<()> { self.update_height(height); self.0.truncate_if_needed(index)?;