diff --git a/crates/brk_computer/src/states/block.rs b/crates/brk_computer/src/states/block.rs index 53aeba6e6..7920affd2 100644 --- a/crates/brk_computer/src/states/block.rs +++ b/crates/brk_computer/src/states/block.rs @@ -5,4 +5,5 @@ use brk_core::{Sats, StoredU32}; pub struct BlockState { utxos: StoredU32, value: Sats, + unspendable: Sats, } diff --git a/crates/brk_computer/src/states/mod.rs b/crates/brk_computer/src/states/mod.rs index 5512ea589..743422ef2 100644 --- a/crates/brk_computer/src/states/mod.rs +++ b/crates/brk_computer/src/states/mod.rs @@ -1,7 +1,13 @@ mod block; mod cohort; mod outputs; +mod realized; +mod received; +mod sent; pub use block::*; pub use cohort::*; pub use outputs::*; +pub use realized::*; +pub use received::*; +pub use sent::*; diff --git a/crates/brk_computer/src/states/realized.rs b/crates/brk_computer/src/states/realized.rs new file mode 100644 index 000000000..8fb0fc3b9 --- /dev/null +++ b/crates/brk_computer/src/states/realized.rs @@ -0,0 +1,11 @@ +use brk_core::Dollars; + +#[derive(Debug, Default)] +pub struct RealizedState { + realized_profit: Dollars, + realized_loss: Dollars, + value_created: Dollars, + adjusted_value_created: Dollars, + value_destroyed: Dollars, + adjusted_value_destroyed: Dollars, +} diff --git a/crates/brk_computer/src/states/received.rs b/crates/brk_computer/src/states/received.rs new file mode 100644 index 000000000..a3f8266c1 --- /dev/null +++ b/crates/brk_computer/src/states/received.rs @@ -0,0 +1,8 @@ +use brk_core::{Sats, StoredUsize}; + +#[derive(Debug, Default)] +pub struct ReceivedState { + utxos: StoredUsize, + sats: Sats, + unspendable: Sats, +} diff --git a/crates/brk_computer/src/states/sent.rs b/crates/brk_computer/src/states/sent.rs new file mode 100644 index 000000000..c351cfd50 --- /dev/null +++ b/crates/brk_computer/src/states/sent.rs @@ -0,0 +1,7 @@ +use brk_core::{Sats, StoredUsize}; + +#[derive(Debug, Default)] +pub struct SentState { + utxos: StoredUsize, + sats: Sats, +} diff --git a/crates/brk_computer/src/vecs/transactions.rs b/crates/brk_computer/src/vecs/transactions.rs index 26c5124e2..fb09256ba 100644 --- a/crates/brk_computer/src/vecs/transactions.rs +++ b/crates/brk_computer/src/vecs/transactions.rs @@ -1,8 +1,8 @@ use std::{fs, path::Path}; use brk_core::{ - CheckedSub, Feerate, Height, InputIndex, OutputIndex, Sats, StoredU32, StoredUsize, TxIndex, - TxVersion, Weight, + CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32, + StoredUsize, TxIndex, TxVersion, Weight, }; use brk_exit::Exit; use brk_indexer::Indexer; @@ -65,6 +65,7 @@ pub struct Vecs { pub indexes_to_p2wpkh_count: ComputedVecsFromHeight, pub indexes_to_p2wsh_count: ComputedVecsFromHeight, pub indexes_to_subsidy: ComputedValueVecsFromHeight, + pub indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight, pub indexes_to_tx_count: ComputedVecsFromHeight, pub indexes_to_tx_v1: ComputedVecsFromHeight, pub indexes_to_tx_v2: ComputedVecsFromHeight, @@ -476,6 +477,15 @@ impl Vecs { .add_average(), compute_dollars, )?, + indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import( + path, + "unclaimed_rewards", + true, + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_sum().add_total(), + compute_dollars, + )?, indexes_to_p2a_count: ComputedVecsFromHeight::forced_import( path, "p2a_count", @@ -850,9 +860,35 @@ impl Vecs { .as_ref() .unwrap() .as_ref(), - |(height, subsidy, ..)| { + |(height, coinbase, ..)| { let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height); - (height, subsidy.checked_sub(fees).unwrap()) + (height, coinbase.checked_sub(fees).unwrap()) + }, + exit, + ) + }, + )?; + + self.indexes_to_unclaimed_rewards.compute_all( + indexer, + indexes, + fetched, + starting_indexes, + exit, + |vec, _, _, starting_indexes, exit| { + vec.compute_transform( + starting_indexes.height, + self.indexes_to_subsidy + .sats + .height + .as_ref() + .unwrap() + .as_ref(), + |(height, subsidy, ..)| { + let halving = HalvingEpoch::from(height); + let expected = + Sats::FIFTY_BTC / 2_usize.pow(halving.unwrap_to_usize() as u32); + (height, expected.checked_sub(subsidy).unwrap()) }, exit, ) @@ -1131,6 +1167,7 @@ impl Vecs { self.indexes_to_tx_weight.vecs(), self.indexes_to_unknownoutput_count.vecs(), self.indexes_to_exact_utxo_count.vecs(), + self.indexes_to_unclaimed_rewards.vecs(), ] .concat() } diff --git a/crates/brk_computer/src/vecs/utxos.rs b/crates/brk_computer/src/vecs/utxos.rs index 04b16c117..8dbe69ca3 100644 --- a/crates/brk_computer/src/vecs/utxos.rs +++ b/crates/brk_computer/src/vecs/utxos.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path, thread}; +use std::{collections::BTreeMap, fs, path::Path, thread}; use brk_core::{ Bitcoin, CheckedSub, Dollars, Height, InputIndex, OutputIndex, OutputType, Sats, StoredUsize, @@ -9,11 +9,10 @@ use brk_vec::{ AnyCollectableVec, AnyVec, BaseVecIterator, Compressed, Computation, EagerVec, StoredIndex, VecIterator, Version, }; -use color_eyre::eyre::ContextCompat; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; -use crate::states::{CohortState, Outputs}; +use crate::states::{CohortState, Outputs, RealizedState, ReceivedState, SentState}; use super::{ Indexes, fetched, @@ -21,23 +20,11 @@ use super::{ indexes, transactions, }; +const VERSION: Version = Version::new(3); + #[derive(Clone, Deref, DerefMut)] pub struct Vecs(Outputs); -#[derive(Clone)] -pub struct Vecs_ { - pub height_to_realized_cap: Option>, - pub indexes_to_realized_cap: Option>, - pub height_to_supply: EagerVec, - pub indexes_to_supply: ComputedValueVecsFromHeight, - pub height_to_unspendable_supply: EagerVec, - pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, - pub height_to_utxo_count: EagerVec, - pub indexes_to_utxo_count: ComputedVecsFromHeight, -} - -const VERSION: Version = Version::new(3); - impl Vecs { pub fn forced_import( path: &Path, @@ -45,73 +32,8 @@ impl Vecs { compressed: Compressed, fetched: Option<&fetched::Vecs>, ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); - - fs::create_dir_all(path)?; - Ok(Self(Outputs { - all: Vecs_ { - height_to_realized_cap: compute_dollars.then(|| { - EagerVec::forced_import( - &path.join("height_to_realized_cap"), - VERSION + Version::ZERO, - compressed, - ) - .unwrap() - }), - indexes_to_realized_cap: compute_dollars.then(|| { - ComputedVecsFromHeight::forced_import( - path, - "realized_cap", - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - ) - .unwrap() - }), - height_to_supply: EagerVec::forced_import( - &path.join("height_to_supply"), - VERSION + Version::ZERO, - compressed, - )?, - indexes_to_supply: ComputedValueVecsFromHeight::forced_import( - path, - "supply", - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - compute_dollars, - )?, - height_to_unspendable_supply: EagerVec::forced_import( - &path.join("height_to_unspendable_supply"), - VERSION + Version::ZERO, - compressed, - )?, - indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( - path, - "unspendable_supply", - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - compute_dollars, - )?, - height_to_utxo_count: EagerVec::forced_import( - &path.join("height_to_utxo_count"), - VERSION + Version::new(111), - compressed, - )?, - indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( - path, - "utxo_count", - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - )?, - }, + all: Vecs_::forced_import(path, _computation, compressed, fetched)?, })) } @@ -133,8 +55,16 @@ impl Vecs { let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex; let outputindex_to_value = &indexer_vecs.outputindex_to_value; let txindex_to_height = &indexes.txindex_to_height; + let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; let outputindex_to_txindex = &indexes.outputindex_to_txindex; let outputindex_to_outputtype = &indexer_vecs.outputindex_to_outputtype; + let height_to_unclaimed_rewards = transactions + .indexes_to_unclaimed_rewards + .sats + .height + .as_ref() + .unwrap() + .as_ref(); let height_to_close = &fetched .as_ref() .map(|fetched| &fetched.chainindexes_to_close.height); @@ -157,10 +87,13 @@ impl Vecs { let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter()); let mut height_to_opreturn_count_iter = height_to_opreturn_count.into_iter(); let mut outputindex_to_outputtype_iter = outputindex_to_outputtype.into_iter(); + let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); + let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); let base_version = Version::ZERO + height_to_first_outputindex.version() + height_to_first_inputindex.version() + + height_to_timestamp_fixed.version() + height_to_output_count.version() + height_to_input_count.version() + inputindex_to_outputindex.version() @@ -169,6 +102,7 @@ impl Vecs { + outputindex_to_txindex.version() + height_to_opreturn_count.version() + outputindex_to_outputtype.version() + + height_to_unclaimed_rewards.version() + height_to_close .as_ref() .map_or(Version::ZERO, |v| v.version()); @@ -227,6 +161,10 @@ impl Vecs { (starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len()) .map(Height::from) .try_for_each(|height| -> color_eyre::Result<()> { + let sent_state = SentState::default(); + let received_state = ReceivedState::default(); + let realized_state = RealizedState::default(); + let first_outputindex = height_to_first_outputindex_iter .unwrap_get_inner(height) .unwrap_to_usize(); @@ -237,35 +175,52 @@ impl Vecs { let input_count = height_to_input_count_iter.unwrap_get_inner(height); let opreturn_count = height_to_opreturn_count_iter.unwrap_get_inner(height); - let (sent_sats_dollars, (mut received_spendable, mut received_unspendable)) = + let (sent_sats_price_tuple, (mut received_spendable, mut received_unspendable)) = thread::scope(|s| { // Skip coinbase - let sent_sats_dollars = s.spawn(|| { + let sent_sats_price_tuple = s.spawn(|| { + let mut txindex_to_height = BTreeMap::new(); + let mut height_to_timestamp_price_sats = BTreeMap::new(); + (first_inputindex + 1..first_inputindex + *input_count) .map(InputIndex::from) .map(|inputindex| { inputindex_to_outputindex_iter.unwrap_get_inner(inputindex) }) - .map(|outputindex| { + .for_each(|outputindex| { let value = outputindex_to_value_iter.unwrap_get_inner(outputindex); - if let Some(height_to_close_iter) = - height_to_close_iter.as_mut() - { - let txindex = outputindex_to_txindex_iter - .unwrap_get_inner(outputindex); - let height = - txindex_to_height_iter.unwrap_get_inner(txindex); - let dollars = - *height_to_close_iter.unwrap_get_inner(height); + let txindex = + outputindex_to_txindex_iter.unwrap_get_inner(outputindex); - (value, dollars) - } else { - (value, Dollars::ZERO) - } - }) - .collect::>() + let height = + *txindex_to_height.entry(txindex).or_insert_with(|| { + txindex_to_height_iter.unwrap_get_inner(txindex) + }); + + let entry = height_to_timestamp_price_sats + .entry(height) + .or_insert_with(|| { + let timestamp = height_to_timestamp_fixed_iter + .unwrap_get_inner(height); + + if let Some(height_to_close_iter) = + height_to_close_iter.as_mut() + { + let dollars = + *height_to_close_iter.unwrap_get_inner(height); + + (timestamp, dollars, Sats::ZERO) + } else { + (timestamp, Dollars::ZERO, Sats::ZERO) + } + }); + + entry.2 += value; + }); + + height_to_timestamp_price_sats }); let received = s.spawn(|| { @@ -277,8 +232,12 @@ impl Vecs { let value = outputindex_to_value_iter_2.unwrap_get_inner(outputindex); - if outputindex_to_outputtype_iter.unwrap_get_inner(outputindex) - == OutputType::OpReturn + let outputtype = outputindex_to_outputtype_iter + .unwrap_get_inner(outputindex); + + if outputtype == OutputType::OpReturn + || outputtype == OutputType::Empty + || outputtype == OutputType::Unknown { unspendable += value } else { @@ -288,12 +247,19 @@ impl Vecs { (spendable, unspendable) }); - (sent_sats_dollars.join().unwrap(), received.join().unwrap()) + ( + sent_sats_price_tuple.join().unwrap(), + received.join().unwrap(), + ) }); - let (sent, realized_cap_destroyed) = sent_sats_dollars - .into_par_iter() - .map(|(sats, dollars)| (sats, dollars * Bitcoin::from(sats))) + let (sent, realized_cap_destroyed) = sent_sats_price_tuple + .par_iter() + .map(|(_, (_, dollars, sats))| { + let dollars = *dollars; + let sats = *sats; + (sats, dollars * Bitcoin::from(sats)) + }) .reduce( || (Sats::ZERO, Dollars::ZERO), |acc, (sats, dollars)| (acc.0 + sats, acc.1 + dollars), @@ -313,6 +279,8 @@ impl Vecs { *input_count - 1 }; + received_unspendable += height_to_unclaimed_rewards_iter.unwrap_get_inner(height); + state.supply -= sent; state.supply += received_spendable; @@ -327,15 +295,6 @@ impl Vecs { let realized_cap_created = price * Bitcoin::from(received); state.realized_cap = (state.realized_cap + realized_cap_created) .checked_sub(realized_cap_destroyed) - .context("to work") - .inspect_err(|_| { - dbg!(( - height, - state.realized_cap, - realized_cap_created, - realized_cap_destroyed - )); - }) .unwrap(); } @@ -394,22 +353,112 @@ impl Vecs { Ok(()) } + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + [self.all.vecs()].concat() + } +} + +#[derive(Clone)] +pub struct Vecs_ { + pub height_to_realized_cap: Option>, + pub indexes_to_realized_cap: Option>, + pub height_to_supply: EagerVec, + pub indexes_to_supply: ComputedValueVecsFromHeight, + pub height_to_unspendable_supply: EagerVec, + pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, + pub height_to_utxo_count: EagerVec, + pub indexes_to_utxo_count: ComputedVecsFromHeight, +} + +impl Vecs_ { + pub fn forced_import( + path: &Path, + _computation: Computation, + compressed: Compressed, + fetched: Option<&fetched::Vecs>, + ) -> color_eyre::Result { + let compute_dollars = fetched.is_some(); + + fs::create_dir_all(path)?; + + Ok(Self { + height_to_realized_cap: compute_dollars.then(|| { + EagerVec::forced_import( + &path.join("height_to_realized_cap"), + VERSION + Version::ZERO, + compressed, + ) + .unwrap() + }), + indexes_to_realized_cap: compute_dollars.then(|| { + ComputedVecsFromHeight::forced_import( + path, + "realized_cap", + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + ) + .unwrap() + }), + height_to_supply: EagerVec::forced_import( + &path.join("height_to_supply"), + VERSION + Version::ZERO, + compressed, + )?, + indexes_to_supply: ComputedValueVecsFromHeight::forced_import( + path, + "supply", + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + compute_dollars, + )?, + height_to_unspendable_supply: EagerVec::forced_import( + &path.join("height_to_unspendable_supply"), + VERSION + Version::ZERO, + compressed, + )?, + indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( + path, + "unspendable_supply", + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + compute_dollars, + )?, + height_to_utxo_count: EagerVec::forced_import( + &path.join("height_to_utxo_count"), + VERSION + Version::new(111), + compressed, + )?, + indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( + path, + "utxo_count", + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + )?, + }) + } + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ vec![ - &self.all.height_to_supply as &dyn AnyCollectableVec, - &self.all.height_to_unspendable_supply, - &self.all.height_to_utxo_count, + &self.height_to_supply as &dyn AnyCollectableVec, + &self.height_to_unspendable_supply, + &self.height_to_utxo_count, ], - self.all - .height_to_realized_cap + self.height_to_realized_cap .as_ref() .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]), - self.all.indexes_to_supply.vecs(), - self.all.indexes_to_unspendable_supply.vecs(), - self.all.indexes_to_utxo_count.vecs(), - self.all - .indexes_to_realized_cap + self.indexes_to_supply.vecs(), + self.indexes_to_unspendable_supply.vecs(), + self.indexes_to_utxo_count.vecs(), + self.indexes_to_realized_cap .as_ref() .map_or(vec![], |v| v.vecs()), ]