diff --git a/Cargo.lock b/Cargo.lock index f480008ab..ec09946b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,6 @@ dependencies = [ "brk_mcp", "brk_parser", "brk_server", - "brk_state", "brk_store", "brk_vec", ] @@ -500,6 +499,7 @@ dependencies = [ name = "brk_computer" version = "0.0.71" dependencies = [ + "bincode", "bitcoin", "bitcoincore-rpc", "brk_core", @@ -508,7 +508,6 @@ dependencies = [ "brk_indexer", "brk_logger", "brk_parser", - "brk_state", "brk_store", "brk_vec", "color-eyre", @@ -517,6 +516,9 @@ dependencies = [ "jiff", "log", "rayon", + "serde", + "zerocopy", + "zerocopy-derive", ] [[package]] @@ -1011,18 +1013,6 @@ dependencies = [ "zip", ] -[[package]] -name = "brk_state" -version = "0.0.71" -dependencies = [ - "bincode", - "brk_core", - "derive_deref", - "serde", - "zerocopy", - "zerocopy-derive", -] - [[package]] name = "brk_store" version = "0.0.71" diff --git a/Cargo.toml b/Cargo.toml index b3af8e2a9..5ac9eda8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,6 @@ brk_parser = { version = "0.0.71", path = "crates/brk_parser" } brk_rmcp = { version = "0.1.8", features = ["transport-streamable-http-server", "transport-worker"]} # brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]} brk_server = { version = "0.0.71", path = "crates/brk_server" } -brk_state = { version = "0.0.71", path = "crates/brk_state" } brk_store = { version = "0.0.71", path = "crates/brk_store" } brk_vec = { version = "0.0.71", path = "crates/brk_vec" } byteview = "=0.6.1" diff --git a/README.md b/README.md index 72620cf74..b0dafe7f3 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,6 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust - [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer` -- [`brk_state`](https://crates.io/crates/brk_state): Various states used mainly by the computer - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) - [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec - [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index 99b24ae24..4d872f4b9 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -21,7 +21,6 @@ full = [ "parser", "interface", "server", - "state", "store", "vec", ] @@ -36,7 +35,6 @@ mcp = ["brk_mcp"] parser = ["brk_parser"] interface = ["brk_interface"] server = ["brk_server"] -state = ["brk_state"] store = ["brk_store"] vec = ["brk_vec"] @@ -53,7 +51,6 @@ brk_mcp = { workspace = true, optional = true } brk_parser = { workspace = true, optional = true } brk_interface = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } -brk_state = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } brk_vec = { workspace = true, optional = true } diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index 12223db75..f568a5ba6 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -47,10 +47,6 @@ pub use brk_interface as interface; #[doc(inline)] pub use brk_server as server; -#[cfg(feature = "state")] -#[doc(inline)] -pub use brk_state as state; - #[cfg(feature = "store")] #[doc(inline)] pub use brk_store as store; diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index e0c91ba3f..bdf7cd04d 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -8,6 +8,7 @@ homepage.workspace = true repository.workspace = true [dependencies] +bincode = { workspace = true } bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_core = { workspace = true } @@ -16,7 +17,6 @@ brk_fetcher = { workspace = true } brk_indexer = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } -brk_state = { workspace = true } brk_store = { workspace = true } brk_vec = { workspace = true } color-eyre = { workspace = true } @@ -25,3 +25,6 @@ fjall = { workspace = true } jiff = { workspace = true } log = { workspace = true } rayon = { workspace = true } +serde = { workspace = true } +zerocopy = { workspace = true } +zerocopy-derive = { workspace = true } diff --git a/crates/brk_core/src/groups/address.rs b/crates/brk_computer/src/groups/address.rs similarity index 96% rename from crates/brk_core/src/groups/address.rs rename to crates/brk_computer/src/groups/address.rs index e9488d7ab..64f6a8aa6 100644 --- a/crates/brk_core/src/groups/address.rs +++ b/crates/brk_computer/src/groups/address.rs @@ -1,4 +1,4 @@ -use crate::{GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize}; +use super::{GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize}; #[derive(Default, Clone)] pub struct AddressGroups { diff --git a/crates/brk_core/src/groups/by_address_type.rs b/crates/brk_computer/src/groups/by_address_type.rs similarity index 99% rename from crates/brk_core/src/groups/by_address_type.rs rename to crates/brk_computer/src/groups/by_address_type.rs index bd3094ece..70818a0df 100644 --- a/crates/brk_core/src/groups/by_address_type.rs +++ b/crates/brk_computer/src/groups/by_address_type.rs @@ -3,7 +3,7 @@ use std::{ ops::{Add, AddAssign}, }; -use crate::OutputType; +use brk_core::OutputType; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_date_range.rs b/crates/brk_computer/src/groups/by_date_range.rs similarity index 100% rename from crates/brk_core/src/groups/by_date_range.rs rename to crates/brk_computer/src/groups/by_date_range.rs diff --git a/crates/brk_core/src/groups/by_epoch.rs b/crates/brk_computer/src/groups/by_epoch.rs similarity index 97% rename from crates/brk_core/src/groups/by_epoch.rs rename to crates/brk_computer/src/groups/by_epoch.rs index 13e7de2f2..23213a758 100644 --- a/crates/brk_core/src/groups/by_epoch.rs +++ b/crates/brk_computer/src/groups/by_epoch.rs @@ -1,4 +1,4 @@ -use crate::{HalvingEpoch, Height}; +use brk_core::{HalvingEpoch, Height}; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_from_date.rs b/crates/brk_computer/src/groups/by_from_date.rs similarity index 100% rename from crates/brk_core/src/groups/by_from_date.rs rename to crates/brk_computer/src/groups/by_from_date.rs diff --git a/crates/brk_core/src/groups/by_from_size.rs b/crates/brk_computer/src/groups/by_from_size.rs similarity index 98% rename from crates/brk_core/src/groups/by_from_size.rs rename to crates/brk_computer/src/groups/by_from_size.rs index 23d338706..a2d9f15fe 100644 --- a/crates/brk_core/src/groups/by_from_size.rs +++ b/crates/brk_computer/src/groups/by_from_size.rs @@ -1,4 +1,4 @@ -use crate::Sats; +use brk_core::Sats; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_size_range.rs b/crates/brk_computer/src/groups/by_size_range.rs similarity index 99% rename from crates/brk_core/src/groups/by_size_range.rs rename to crates/brk_computer/src/groups/by_size_range.rs index fe903bf24..2bd150d8c 100644 --- a/crates/brk_core/src/groups/by_size_range.rs +++ b/crates/brk_computer/src/groups/by_size_range.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign}; -use crate::Sats; +use brk_core::Sats; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_spendable_type.rs b/crates/brk_computer/src/groups/by_spendable_type.rs similarity index 99% rename from crates/brk_core/src/groups/by_spendable_type.rs rename to crates/brk_computer/src/groups/by_spendable_type.rs index a616229f2..c14ffe1c4 100644 --- a/crates/brk_core/src/groups/by_spendable_type.rs +++ b/crates/brk_computer/src/groups/by_spendable_type.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign}; -use crate::OutputType; +use brk_core::OutputType; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_term.rs b/crates/brk_computer/src/groups/by_term.rs similarity index 100% rename from crates/brk_core/src/groups/by_term.rs rename to crates/brk_computer/src/groups/by_term.rs diff --git a/crates/brk_core/src/groups/by_type.rs b/crates/brk_computer/src/groups/by_type.rs similarity index 98% rename from crates/brk_core/src/groups/by_type.rs rename to crates/brk_computer/src/groups/by_type.rs index daa80dcd6..738a93b01 100644 --- a/crates/brk_core/src/groups/by_type.rs +++ b/crates/brk_computer/src/groups/by_type.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign}; -use crate::OutputType; +use brk_core::OutputType; use super::{GroupedBySpendableType, GroupedByUnspendableType}; diff --git a/crates/brk_core/src/groups/by_unspendable_type.rs b/crates/brk_computer/src/groups/by_unspendable_type.rs similarity index 100% rename from crates/brk_core/src/groups/by_unspendable_type.rs rename to crates/brk_computer/src/groups/by_unspendable_type.rs diff --git a/crates/brk_core/src/groups/by_up_to_date.rs b/crates/brk_computer/src/groups/by_up_to_date.rs similarity index 100% rename from crates/brk_core/src/groups/by_up_to_date.rs rename to crates/brk_computer/src/groups/by_up_to_date.rs diff --git a/crates/brk_core/src/groups/by_up_to_size.rs b/crates/brk_computer/src/groups/by_up_to_size.rs similarity index 98% rename from crates/brk_core/src/groups/by_up_to_size.rs rename to crates/brk_computer/src/groups/by_up_to_size.rs index 2bb189c6d..48b4ad8af 100644 --- a/crates/brk_core/src/groups/by_up_to_size.rs +++ b/crates/brk_computer/src/groups/by_up_to_size.rs @@ -1,4 +1,4 @@ -use crate::Sats; +use brk_core::Sats; use super::GroupFilter; diff --git a/crates/brk_core/src/groups/by_value.rs b/crates/brk_computer/src/groups/by_value.rs similarity index 100% rename from crates/brk_core/src/groups/by_value.rs rename to crates/brk_computer/src/groups/by_value.rs diff --git a/crates/brk_core/src/groups/filter.rs b/crates/brk_computer/src/groups/filter.rs similarity index 97% rename from crates/brk_core/src/groups/filter.rs rename to crates/brk_computer/src/groups/filter.rs index 0107d573c..32bcd534e 100644 --- a/crates/brk_core/src/groups/filter.rs +++ b/crates/brk_computer/src/groups/filter.rs @@ -1,6 +1,6 @@ use std::ops::Range; -use crate::{HalvingEpoch, OutputType}; +use brk_core::{HalvingEpoch, OutputType}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum GroupFilter { diff --git a/crates/brk_core/src/groups/mod.rs b/crates/brk_computer/src/groups/mod.rs similarity index 100% rename from crates/brk_core/src/groups/mod.rs rename to crates/brk_computer/src/groups/mod.rs diff --git a/crates/brk_core/src/groups/utxo.rs b/crates/brk_computer/src/groups/utxo.rs similarity index 100% rename from crates/brk_core/src/groups/utxo.rs rename to crates/brk_computer/src/groups/utxo.rs diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index dab249b97..a31136cae 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -10,12 +10,16 @@ use brk_exit::Exit; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_vec::{Computation, Format}; +use log::info; +mod groups; +mod states; mod stores; mod utils; mod vecs; -use log::info; +use groups::*; +use states::*; use stores::Stores; use vecs::Vecs; diff --git a/crates/brk_state/src/block.rs b/crates/brk_computer/src/states/block.rs similarity index 100% rename from crates/brk_state/src/block.rs rename to crates/brk_computer/src/states/block.rs diff --git a/crates/brk_state/src/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs similarity index 57% rename from crates/brk_state/src/cohorts/address.rs rename to crates/brk_computer/src/states/cohorts/address.rs index f3ce22575..2933e694d 100644 --- a/crates/brk_state/src/cohorts/address.rs +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -1,11 +1,8 @@ -use std::{ - ops::{Deref, DerefMut}, - path::Path, -}; +use std::path::Path; -use brk_core::{Dollars, Height, Result}; +use brk_core::{AddressData, Dollars, Height, Result, Sats}; -use crate::{SupplyState, UnrealizedState}; +use crate::SupplyState; use super::CohortState; @@ -26,11 +23,46 @@ impl AddressCohortState { pub fn reset_single_iteration_values(&mut self) { self.inner.reset_single_iteration_values(); } -} -// fn increment(&mut self, supply_state: &SupplyState, price: Option) { -// self.inner.increment(supply_state, price); -// } + pub fn send( + &mut self, + value: Sats, + current_price: Option, + prev_price: Option, + blocks_old: usize, + days_old: f64, + older_than_hour: bool, + ) { + self.inner.send( + &SupplyState { utxos: 1, value }, + current_price, + prev_price, + blocks_old, + days_old, + older_than_hour, + ); + } + + pub fn receive(&mut self, value: Sats, price: Option) { + self.inner.receive(&SupplyState { utxos: 1, value }, price); + } + + pub fn add(&mut self, addressdata: &AddressData) { + self.address_count += 1; + self.inner + .increment_(&addressdata.into(), addressdata.realized_cap); + } + + pub fn subtract(&mut self, addressdata: &AddressData) { + self.address_count.checked_sub(1).unwrap(); + self.inner + .decrement_(&addressdata.into(), addressdata.realized_cap); + } + + pub fn commit(&mut self, height: Height) -> Result<()> { + self.inner.commit(height) + } +} // fn decrement(&mut self, supply_state: &SupplyState, price: Option) { // self.inner.decrement(supply_state, price); @@ -44,25 +76,6 @@ impl AddressCohortState { // self.inner.receive(supply_state, price); // } -// fn send( -// &mut self, -// supply_state: &SupplyState, -// current_price: Option, -// prev_price: Option, -// blocks_old: usize, -// days_old: f64, -// older_than_hour: bool, -// ) { -// self.inner.send( -// supply_state, -// current_price, -// prev_price, -// blocks_old, -// days_old, -// older_than_hour, -// ); -// } - // fn compute_unrealized_states( // &self, // height_price: Dollars, @@ -72,9 +85,6 @@ impl AddressCohortState { // .compute_unrealized_states(height_price, date_price) // } -// fn commit(&mut self, height: Height) -> Result<()> { -// self.inner.commit(height) -// } // } // impl Deref for AddressCohortState { diff --git a/crates/brk_state/src/cohorts/common.rs b/crates/brk_computer/src/states/cohorts/common.rs similarity index 75% rename from crates/brk_state/src/cohorts/common.rs rename to crates/brk_computer/src/states/cohorts/common.rs index d0535a181..7dba4bf26 100644 --- a/crates/brk_state/src/cohorts/common.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -1,6 +1,6 @@ use std::{cmp::Ordering, path::Path}; -use brk_core::{CheckedSub, Dollars, Height, Result, Sats}; +use brk_core::{Bitcoin, CheckedSub, Dollars, Height, Result, Sats}; use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState}; @@ -44,6 +44,20 @@ impl CohortState { } } + pub fn increment_(&mut self, supply_state: &SupplyState, realized_cap: Dollars) { + self.supply += supply_state; + + if supply_state.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + realized.increment_(realized_cap); + *self + .price_to_amount + .entry(realized_cap / Bitcoin::from(supply_state.value)) + .or_default() += supply_state.value; + } + } + } + pub fn decrement(&mut self, supply_state: &SupplyState, price: Option) { self.supply -= supply_state; @@ -56,6 +70,20 @@ impl CohortState { } } + pub fn decrement_(&mut self, supply_state: &SupplyState, realized_cap: Dollars) { + self.supply -= supply_state; + + if supply_state.value > Sats::ZERO { + if let Some(realized) = self.realized.as_mut() { + realized.decrement_(realized_cap); + self.decrement_price_to_amount( + supply_state, + realized_cap / Bitcoin::from(supply_state.value), + ); + } + } + } + fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) { let amount = self.price_to_amount.get_mut(&price).unwrap(); *amount -= supply_state.value; @@ -85,21 +113,23 @@ impl CohortState { days_old: f64, older_than_hour: bool, ) { - if supply_state.utxos > 0 { - self.supply -= supply_state; + if supply_state.utxos == 0 { + return; + } - if supply_state.value > Sats::ZERO { - self.satblocks_destroyed += supply_state.value * blocks_old; + self.supply -= supply_state; - self.satdays_destroyed += - Sats::from((u64::from(supply_state.value) as f64 * days_old).floor() as u64); + if supply_state.value > Sats::ZERO { + self.satblocks_destroyed += supply_state.value * blocks_old; - if let Some(realized) = self.realized.as_mut() { - let current_price = current_price.unwrap(); - let prev_price = prev_price.unwrap(); - realized.send(supply_state, current_price, prev_price, older_than_hour); - self.decrement_price_to_amount(supply_state, prev_price); - } + self.satdays_destroyed += + Sats::from((u64::from(supply_state.value) as f64 * days_old).floor() as u64); + + if let Some(realized) = self.realized.as_mut() { + let current_price = current_price.unwrap(); + let prev_price = prev_price.unwrap(); + realized.send(supply_state, current_price, prev_price, older_than_hour); + self.decrement_price_to_amount(supply_state, prev_price); } } } diff --git a/crates/brk_state/src/cohorts/mod.rs b/crates/brk_computer/src/states/cohorts/mod.rs similarity index 70% rename from crates/brk_state/src/cohorts/mod.rs rename to crates/brk_computer/src/states/cohorts/mod.rs index 4c579d1a0..b3dc7b22f 100644 --- a/crates/brk_state/src/cohorts/mod.rs +++ b/crates/brk_computer/src/states/cohorts/mod.rs @@ -1,9 +1,7 @@ mod address; mod common; -// mod r#trait; mod utxo; pub use address::*; pub use common::*; -// pub use r#trait::*; pub use utxo::*; diff --git a/crates/brk_computer/src/states/cohorts/utxo.rs b/crates/brk_computer/src/states/cohorts/utxo.rs new file mode 100644 index 000000000..89987d2ab --- /dev/null +++ b/crates/brk_computer/src/states/cohorts/utxo.rs @@ -0,0 +1,19 @@ +use std::path::Path; + +use brk_core::Result; +use derive_deref::{Deref, DerefMut}; + +use super::CohortState; + +#[derive(Clone, Deref, DerefMut)] +pub struct UTXOCohortState(CohortState); + +impl UTXOCohortState { + pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { + Ok(Self(CohortState::default_and_import( + path, + name, + compute_dollars, + )?)) + } +} diff --git a/crates/brk_state/src/lib.rs b/crates/brk_computer/src/states/mod.rs similarity index 63% rename from crates/brk_state/src/lib.rs rename to crates/brk_computer/src/states/mod.rs index a77d5877c..5976a47b9 100644 --- a/crates/brk_state/src/lib.rs +++ b/crates/brk_computer/src/states/mod.rs @@ -1,8 +1,3 @@ -#![doc = include_str!("../README.md")] -// #![doc = "\n## Example\n\n```rust"] -// #![doc = include_str!("../examples/main.rs")] -// #![doc = "```"] - mod block; mod cohorts; mod price_to_amount; diff --git a/crates/brk_state/src/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs similarity index 100% rename from crates/brk_state/src/price_to_amount.rs rename to crates/brk_computer/src/states/price_to_amount.rs diff --git a/crates/brk_state/src/realized.rs b/crates/brk_computer/src/states/realized.rs similarity index 89% rename from crates/brk_state/src/realized.rs rename to crates/brk_computer/src/states/realized.rs index 7d673ae8d..a4eea6f66 100644 --- a/crates/brk_state/src/realized.rs +++ b/crates/brk_computer/src/states/realized.rs @@ -42,6 +42,10 @@ impl RealizedState { return; } + self.increment_(price * supply_state.value) + } + + pub fn increment_(&mut self, realized_cap: Dollars) { if self.cap == Dollars::NAN { self.cap = Dollars::ZERO; self.profit = Dollars::ZERO; @@ -52,13 +56,15 @@ impl RealizedState { self.adj_value_destroyed = Dollars::ZERO; } - let value = price * supply_state.value; - self.cap += value; + self.cap += realized_cap; } pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) { - let value = price * supply_state.value; - self.cap = self.cap.checked_sub(value).unwrap(); + self.decrement_(price * supply_state.value); + } + + pub fn decrement_(&mut self, realized_cap: Dollars) { + self.cap = self.cap.checked_sub(realized_cap).unwrap(); } pub fn receive(&mut self, supply_state: &SupplyState, current_price: Dollars) { diff --git a/crates/brk_state/src/supply.rs b/crates/brk_computer/src/states/supply.rs similarity index 80% rename from crates/brk_state/src/supply.rs rename to crates/brk_computer/src/states/supply.rs index c9bd95442..4c26830d3 100644 --- a/crates/brk_state/src/supply.rs +++ b/crates/brk_computer/src/states/supply.rs @@ -1,6 +1,6 @@ use std::ops::{Add, AddAssign, SubAssign}; -use brk_core::{CheckedSub, Sats}; +use brk_core::{AddressData, CheckedSub, Sats}; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -39,3 +39,12 @@ impl SubAssign<&SupplyState> for SupplyState { self.value = self.value.checked_sub(rhs.value).unwrap(); } } + +impl From<&AddressData> for SupplyState { + fn from(value: &AddressData) -> Self { + Self { + utxos: value.outputs_len as usize, + value: value.amount(), + } + } +} diff --git a/crates/brk_state/src/transacted.rs b/crates/brk_computer/src/states/transacted.rs similarity index 93% rename from crates/brk_state/src/transacted.rs rename to crates/brk_computer/src/states/transacted.rs index c9b0fc9ef..57b37ff3d 100644 --- a/crates/brk_state/src/transacted.rs +++ b/crates/brk_computer/src/states/transacted.rs @@ -1,6 +1,8 @@ use std::ops::{Add, AddAssign}; -use brk_core::{GroupedBySizeRange, GroupedByType, OutputType, Sats}; +use brk_core::{OutputType, Sats}; + +use crate::{GroupedBySizeRange, GroupedByType}; use super::SupplyState; diff --git a/crates/brk_state/src/unrealized.rs b/crates/brk_computer/src/states/unrealized.rs similarity index 100% rename from crates/brk_state/src/unrealized.rs rename to crates/brk_computer/src/states/unrealized.rs diff --git a/crates/brk_computer/src/stores.rs b/crates/brk_computer/src/stores.rs index b400f7157..71875aa73 100644 --- a/crates/brk_computer/src/stores.rs +++ b/crates/brk_computer/src/stores.rs @@ -1,8 +1,8 @@ use std::{path::Path, thread}; use brk_core::{ - AddressData, EmptyAddressData, GroupedByAddressType, Height, OutputIndex, OutputType, - P2AAddressIndex, P2AAddressIndexOutputindex, P2PK33AddressIndex, P2PK33AddressIndexOutputindex, + AddressData, EmptyAddressData, Height, OutputIndex, OutputType, P2AAddressIndex, + P2AAddressIndexOutputindex, P2PK33AddressIndex, P2PK33AddressIndexOutputindex, P2PK65AddressIndex, P2PK65AddressIndexOutputindex, P2PKHAddressIndex, P2PKHAddressIndexOutputindex, P2SHAddressIndex, P2SHAddressIndexOutputindex, P2TRAddressIndex, P2TRAddressIndexOutputindex, P2WPKHAddressIndex, P2WPKHAddressIndexOutputindex, @@ -12,8 +12,11 @@ use brk_store::{AnyStore, Store}; use fjall::{PersistMode, TransactionalKeyspace}; use rayon::prelude::*; -use crate::vecs::stateful::{ - AddressTypeToTypeIndexTree, AddressTypeToTypeIndexVec, WithAddressDataSource, +use crate::{ + GroupedByAddressType, + vecs::stateful::{ + AddressTypeToTypeIndexTree, AddressTypeToTypeIndexVec, WithAddressDataSource, + }, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/cointime.rs b/crates/brk_computer/src/vecs/cointime.rs index ecb6563ef..a6c49cd2a 100644 --- a/crates/brk_computer/src/vecs/cointime.rs +++ b/crates/brk_computer/src/vecs/cointime.rs @@ -264,7 +264,7 @@ impl Vecs { stateful: &stateful::Vecs, exit: &Exit, ) -> color_eyre::Result<()> { - let circulating_supply = &stateful.utxos_vecs.all.1.height_to_supply; + let circulating_supply = &stateful.utxo_vecs.all.1.height_to_supply; self.indexes_to_coinblocks_created.compute_all( indexer, @@ -282,7 +282,7 @@ impl Vecs { )?; let indexes_to_coinblocks_destroyed = - &stateful.utxos_vecs.all.1.indexes_to_coinblocks_destroyed; + &stateful.utxo_vecs.all.1.indexes_to_coinblocks_destroyed; self.indexes_to_coinblocks_stored.compute_all( indexer, @@ -392,7 +392,7 @@ impl Vecs { if let Some(fetched) = fetched { let realized_cap = stateful - .utxos_vecs + .utxo_vecs .all .1 .height_to_realized_cap @@ -400,7 +400,7 @@ impl Vecs { .unwrap(); let realized_price = stateful - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_realized_price diff --git a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs index 845a2d08e..52b2226c7 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs @@ -5,7 +5,7 @@ use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{AnyCollectableVec, EagerVec, Format}; -use crate::vecs::{Indexes, indexes}; +use crate::vecs::{indexes, Indexes}; use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; diff --git a/crates/brk_computer/src/vecs/stateful/address_cohort.rs b/crates/brk_computer/src/vecs/stateful/address_cohort.rs index 9ddbc6f0d..1e8006164 100644 --- a/crates/brk_computer/src/vecs/stateful/address_cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/address_cohort.rs @@ -3,14 +3,16 @@ use std::{ops::Deref, path::Path}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_state::AddressCohortState; use brk_vec::{ AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator, }; -use crate::vecs::{ - Indexes, fetched, indexes, market, - stateful::{common, r#trait::CohortVecs}, +use crate::{ + states::AddressCohortState, + vecs::{ + Indexes, fetched, indexes, market, + stateful::{common, r#trait::CohortVecs}, + }, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/stateful/address_cohorts.rs b/crates/brk_computer/src/vecs/stateful/address_cohorts.rs index 84685a783..5d06d1338 100644 --- a/crates/brk_computer/src/vecs/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/vecs/stateful/address_cohorts.rs @@ -1,17 +1,17 @@ use std::path::Path; -use brk_core::{ - AddressGroups, GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize, Result, - Version, -}; +use brk_core::{Height, Result, Version}; use brk_exit::Exit; use brk_vec::{Computation, Format}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; -use crate::vecs::{ - Indexes, fetched, - stateful::{address_cohort, r#trait::CohortVecs}, +use crate::{ + AddressGroups, GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize, + vecs::{ + Indexes, fetched, + stateful::{address_cohort, r#trait::CohortVecs}, + }, }; const VERSION: Version = Version::new(0); @@ -326,4 +326,10 @@ impl Vecs { vecs.compute_from_stateful(starting_indexes, &stateful, exit) }) } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit)) + } } diff --git a/crates/brk_computer/src/vecs/stateful/common.rs b/crates/brk_computer/src/vecs/stateful/common.rs index eb7a88318..6467c11d3 100644 --- a/crates/brk_computer/src/vecs/stateful/common.rs +++ b/crates/brk_computer/src/vecs/stateful/common.rs @@ -5,18 +5,21 @@ use brk_core::{ }; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_state::CohortState; use brk_vec::{ AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator, }; -use crate::vecs::{ - Indexes, fetched, - grouped::{ - ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, ComputedValueVecsFromDateIndex, - ComputedVecsFromDateIndex, ComputedVecsFromHeight, StorableVecGeneatorOptions, +use crate::{ + states::CohortState, + vecs::{ + Indexes, fetched, + grouped::{ + ComputedHeightValueVecs, ComputedRatioVecsFromDateIndex, + ComputedValueVecsFromDateIndex, ComputedVecsFromDateIndex, ComputedVecsFromHeight, + StorableVecGeneatorOptions, + }, + indexes, market, }, - indexes, market, }; const VERSION: Version = Version::ZERO; diff --git a/crates/brk_computer/src/vecs/stateful/mod.rs b/crates/brk_computer/src/vecs/stateful/mod.rs index 19bdfadb7..1ff1541cf 100644 --- a/crates/brk_computer/src/vecs/stateful/mod.rs +++ b/crates/brk_computer/src/vecs/stateful/mod.rs @@ -1,8 +1,8 @@ use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; use brk_core::{ - AddressData, DateIndex, Dollars, EmptyAddressData, GroupedByAddressType, Height, InputIndex, - OutputIndex, OutputType, Result, Sats, StoredUsize, Version, + AddressData, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height, InputIndex, OutputIndex, + OutputType, Result, Sats, StoredUsize, Version, }; use brk_exit::Exit; use brk_indexer::Indexer; @@ -13,9 +13,9 @@ use brk_vec::{ use log::info; use rayon::prelude::*; -use brk_state::{BlockState, SupplyState, Transacted}; - -use crate::{stores::Stores, vecs::market}; +use crate::{ + BlockState, GroupedByAddressType, SupplyState, Transacted, stores::Stores, vecs::market, +}; use super::{ Indexes, fetched, @@ -49,8 +49,11 @@ pub struct Vecs { pub height_to_opreturn_supply: EagerVec, pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight, pub height_to_address_count: EagerVec, + pub height_to_empty_address_count: EagerVec, pub addresstype_to_height_to_address_count: GroupedByAddressType>, - pub utxos_vecs: utxo_cohorts::Vecs, + pub addresstype_to_height_to_empty_address_count: + GroupedByAddressType>, + pub utxo_vecs: utxo_cohorts::Vecs, pub address_vecs: address_cohorts::Vecs, } @@ -113,6 +116,12 @@ impl Vecs { version + VERSION + Version::ZERO, format, )?, + height_to_empty_address_count: EagerVec::forced_import( + path, + "empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, addresstype_to_height_to_address_count: GroupedByAddressType { p2pk65: EagerVec::forced_import( path, @@ -163,7 +172,57 @@ impl Vecs { format, )?, }, - utxos_vecs: utxo_cohorts::Vecs::forced_import( + addresstype_to_height_to_empty_address_count: GroupedByAddressType { + p2pk65: EagerVec::forced_import( + path, + "p2pk65_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2pk33: EagerVec::forced_import( + path, + "p2pk33_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2pkh: EagerVec::forced_import( + path, + "p2pkh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2sh: EagerVec::forced_import( + path, + "p2sh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wpkh: EagerVec::forced_import( + path, + "p2wpkh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2wsh: EagerVec::forced_import( + path, + "p2wsh_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2tr: EagerVec::forced_import( + path, + "p2tr_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + p2a: EagerVec::forced_import( + path, + "p2a_empty_address_count", + version + VERSION + Version::ZERO, + format, + )?, + }, + utxo_vecs: utxo_cohorts::Vecs::forced_import( path, version, _computation, @@ -229,6 +288,7 @@ impl Vecs { let outputindex_to_txindex_mmap = outputindex_to_txindex.mmap().load(); let txindex_to_height_mmap = txindex_to_height.mmap().load(); let height_to_close_mmap = height_to_close.map(|v| v.mmap().load()); + let height_to_timestamp_fixed_mmap = height_to_timestamp_fixed.mmap().load(); let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); @@ -242,7 +302,7 @@ impl Vecs { let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter(); let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter(); - let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs(); + let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); let base_version = Version::ZERO + height_to_first_outputindex.version() @@ -373,7 +433,7 @@ impl Vecs { .try_for_each(|_height| -> color_eyre::Result<()> { height = _height; - self.utxos_vecs + self.utxo_vecs .as_mut_separate_vecs() .iter_mut() .for_each(|(_, v)| v.state.reset_single_iteration_values()); @@ -398,10 +458,10 @@ impl Vecs { let output_count = height_to_output_count_iter.unwrap_get_inner(height); let input_count = height_to_input_count_iter.unwrap_get_inner(height); - let ((mut height_to_sent, new_addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_sats_and_addressdata_opt), (mut received, new_addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_sats_and_addressdata_opt)) = thread::scope(|s| { + let ((mut height_to_sent, new_addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data), (mut received, new_addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data)) = thread::scope(|s| { if chain_state_starting_height <= height { s.spawn(|| { - self.utxos_vecs + self.utxo_vecs .tick_tock_next_block(&chain_state, timestamp); }); } @@ -452,33 +512,48 @@ impl Vecs { .unwrap() .into_owned(); - let height = txindex_to_height + let prev_height = txindex_to_height .get_or_read(input_txindex, &txindex_to_height_mmap) .unwrap() .unwrap() .into_owned(); - let dollars_opt = height_to_close.map(|m| *m.get_or_read(height, height_to_close_mmap.as_ref().unwrap()).unwrap() + let prev_price = height_to_close.map(|m| *m.get_or_read(prev_height, height_to_close_mmap.as_ref().unwrap()).unwrap() .unwrap() .into_owned()); - (height, value, input_type, typeindex, outputindex, addressdata_opt, dollars_opt) + let prev_timestamp = height_to_timestamp_fixed.get_or_read(prev_height, &height_to_timestamp_fixed_mmap) + .unwrap() + .unwrap() + .into_owned(); + + let blocks_old = height.unwrap_to_usize() - prev_height.unwrap_to_usize(); + + let days_old = prev_timestamp + .difference_in_days_between_float(timestamp); + + let older_than_hour = timestamp + .checked_sub(prev_timestamp) + .unwrap() + .is_more_than_hour(); + + (prev_height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour) }) .fold( || { ( BTreeMap::::default(), AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>, Option)>::default(), + AddressTypeToTypeIndexVec::<(Sats, Option>, Option, usize, f64, bool)>::default(), ) }, - |(mut tree, mut vecs, mut vecs2), (height, value, input_type, typeindex, outputindex, addressdata_opt, dollars_opt)| { + |(mut tree, mut vecs, mut vecs2), (height, value, input_type, typeindex, outputindex, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour)| { tree.entry(height).or_default().iterate(value, input_type); if let Some(vec) = vecs.get_mut(input_type) { vec.push((typeindex, outputindex)); } if let Some(vec) = vecs2.get_mut(input_type) { - vec.push((typeindex, (value, addressdata_opt, dollars_opt))); + vec.push((typeindex, (value, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour))); } (tree, vecs, vecs2) }, @@ -487,7 +562,7 @@ impl Vecs { ( BTreeMap::::default(), AddressTypeToTypeIndexVec::::default(), - AddressTypeToTypeIndexVec::<(Sats, Option>, Option)>::default(), + AddressTypeToTypeIndexVec::<(Sats, Option>, Option, usize, f64, bool)>::default(), ) }, |(first_tree, mut source_vecs,mut source_vecs2), (second_tree, other_vecs, other_vecs2)| { let (mut tree_source, tree_to_consume) = if first_tree.len() > second_tree.len() { @@ -571,14 +646,9 @@ impl Vecs { addresstype_to_typeindex_to_sent_outputindex.merge(new_addresstype_to_typedindex_to_sent_outputindex); addresstype_to_typeindex_to_received_outputindex.merge(new_addresstype_to_typedindex_to_received_outputindex); - addresstype_to_typedindex_to_received_sats_and_addressdata_opt.process_received(&mut self.address_vecs, &mut addresstype_to_typeindex_to_addressdata, &mut addresstype_to_typeindex_to_emptyaddressdata, price); + addresstype_to_typedindex_to_received_data.process_received(&mut self.address_vecs, &mut addresstype_to_typeindex_to_addressdata, &mut addresstype_to_typeindex_to_emptyaddressdata, price); - addresstype_to_typedindex_to_sent_sats_and_addressdata_opt.process_sent(&mut self.address_vecs, &mut addresstype_to_typeindex_to_addressdata)?; - - // addresstype_to_typedindex_to_sent_sats_and_addressdata_opt; - // take from addressdata store - // apply - // update vecs states if cohort changes + addresstype_to_typedindex_to_sent_data.process_sent(&mut self.address_vecs, &mut addresstype_to_typeindex_to_addressdata, &mut addresstype_to_typeindex_to_emptyaddressdata, price)?; unspendable_supply += received .by_type @@ -612,7 +682,7 @@ impl Vecs { timestamp, }); - self.utxos_vecs.receive(received, height, price); + self.utxo_vecs.receive(received, height, price); let unsafe_chain_state = UnsafeSlice::new(&mut chain_state); @@ -621,18 +691,24 @@ impl Vecs { &sent.spendable_supply; }); - self.utxos_vecs.send(height_to_sent, chain_state.as_slice()); + self.utxo_vecs.send(height_to_sent, chain_state.as_slice()); } else { dbg!(chain_state_starting_height, height); panic!("temp, just making sure") } - let mut separate_utxo_vecs = self.utxos_vecs.as_mut_separate_vecs(); + let mut separate_utxo_vecs = self.utxo_vecs.as_mut_separate_vecs(); separate_utxo_vecs .iter_mut() .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; + let mut separate_address_vecs = self.address_vecs.as_mut_separate_vecs(); + + separate_address_vecs + .iter_mut() + .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; + self.height_to_unspendable_supply.forced_push_at( height, unspendable_supply, @@ -655,6 +731,8 @@ impl Vecs { .as_mut() .map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex))); + // thread::scope(|scope| { + // scope.spawn(|| { separate_utxo_vecs.par_iter_mut().try_for_each(|(_, v)| { v.compute_then_force_push_unrealized_states( height, @@ -664,6 +742,20 @@ impl Vecs { exit, ) })?; + // }); + // scope.spawn(|| { + separate_address_vecs.par_iter_mut().try_for_each(|(_, v)| { + v.compute_then_force_push_unrealized_states( + height, + price, + is_date_last_height.then_some(dateindex), + date_price, + exit, + ) + })?; + // }); + // }); + if height != Height::ZERO && height.unwrap_to_usize() % 10_000 == 0 { info!("Flushing..."); @@ -703,39 +795,63 @@ impl Vecs { info!("Computing overlapping..."); - self.utxos_vecs + // thread::scope(|scope| { + // scope.spawn(|| { + self.utxo_vecs .compute_overlapping_vecs(starting_indexes, exit)?; + // }); + // scope.spawn(|| { + self.address_vecs + .compute_overlapping_vecs(starting_indexes, exit)?; + // }); + // }); info!("Computing rest part 1..."); - self.utxos_vecs + // thread::scope(|scope| { + // scope.spawn(|| { + self.utxo_vecs .as_mut_vecs() .par_iter_mut() .try_for_each(|(_, v)| { v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) - })?; + }) + .unwrap(); + // }); + // scope.spawn(|| { + self.address_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit) + }) + .unwrap(); + // }); + // }); info!("Computing rest part 2..."); - let height_to_supply = self.utxos_vecs.all.1.height_to_supply_value.bitcoin.clone(); + let height_to_supply = self.utxo_vecs.all.1.height_to_supply_value.bitcoin.clone(); let dateindex_to_supply = self - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_supply .bitcoin .dateindex .clone(); - let height_to_realized_cap = self.utxos_vecs.all.1.height_to_realized_cap.clone(); + let height_to_realized_cap = self.utxo_vecs.all.1.height_to_realized_cap.clone(); let dateindex_to_realized_cap = self - .utxos_vecs + .utxo_vecs .all .1 .indexes_to_realized_cap .as_ref() .map(|v| v.dateindex.unwrap_last().clone()); - self.utxos_vecs + // thread::scope(|scope| { + // scope.spawn(|| { + self.utxo_vecs .as_mut_vecs() .par_iter_mut() .try_for_each(|(_, v)| { @@ -752,6 +868,27 @@ impl Vecs { exit, ) })?; + // }); + // scope.spawn(|| { + self.address_vecs + .as_mut_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| { + v.compute_rest_part2( + indexer, + indexes, + fetched, + starting_indexes, + market, + &height_to_supply, + dateindex_to_supply.as_ref().unwrap(), + height_to_realized_cap.as_ref(), + dateindex_to_realized_cap.as_ref(), + exit, + ) + })?; + // }); + // }); self.indexes_to_unspendable_supply.compute_rest( indexer, indexes, @@ -782,12 +919,24 @@ impl Vecs { chain_state: &[BlockState], exit: &Exit, ) -> Result<()> { - self.utxos_vecs + self.utxo_vecs + .as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; + self.address_vecs .as_mut_separate_vecs() .par_iter_mut() .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?; self.height_to_unspendable_supply.safe_flush(exit)?; self.height_to_opreturn_supply.safe_flush(exit)?; + self.addresstype_to_height_to_address_count + .as_mut_vec() + .into_iter() + .try_for_each(|v| v.safe_flush(exit))?; + self.addresstype_to_height_to_empty_address_count + .as_mut_vec() + .into_iter() + .try_for_each(|v| v.safe_flush(exit))?; self.chain_state.truncate_if_needed(Height::ZERO)?; chain_state.iter().for_each(|block_state| { @@ -800,16 +949,33 @@ impl Vecs { pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { [ - self.utxos_vecs + self.utxo_vecs + .vecs() + .into_iter() + .flat_map(|v| v.vecs()) + .collect::>(), + self.address_vecs .vecs() .into_iter() .flat_map(|v| v.vecs()) .collect::>(), self.indexes_to_unspendable_supply.vecs(), self.indexes_to_opreturn_supply.vecs(), + self.addresstype_to_height_to_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v as &dyn AnyCollectableVec) + .collect::>(), + self.addresstype_to_height_to_empty_address_count + .as_typed_vec() + .into_iter() + .map(|(_, v)| v as &dyn AnyCollectableVec) + .collect::>(), vec![ &self.height_to_unspendable_supply, &self.height_to_opreturn_supply, + &self.height_to_address_count, + &self.height_to_empty_address_count, ], ] .into_iter() @@ -833,11 +999,17 @@ impl AddressTypeToTypeIndexVec<(Sats, Option> self.into_typed_vec().into_iter().for_each(|(_type, vec)| { vec.into_iter() .for_each(|(type_index, (value, addressdata_opt))| { + let mut is_new = false; + let addressdata_withsource = addresstype_to_typeindex_to_addressdata .get_mut(_type) .unwrap() .entry(type_index) .or_insert_with(|| { + is_new = addressdata_opt + .as_ref() + .is_some_and(|a| matches!(a, WithAddressDataSource::New(_))); + addressdata_opt.unwrap_or_else(|| { addresstype_to_typeindex_to_emptyaddressdata .get_mut(_type) @@ -850,21 +1022,36 @@ impl AddressTypeToTypeIndexVec<(Sats, Option> let addressdata = addressdata_withsource.deref_mut(); - let prev_filter = vecs.by_size_range.get_mut(addressdata.amount()).0.clone(); + let prev_amount = addressdata.amount(); - addressdata.receive(value, price); + let amount = prev_amount + value; - let (filter, vecs) = vecs.by_size_range.get_mut(addressdata.amount()); + if is_new + || vecs.by_size_range.get_mut(amount).0.clone() + != vecs.by_size_range.get_mut(prev_amount).0.clone() + { + if !is_new { + vecs.by_size_range + .get_mut(prev_amount) + .1 + .state + .subtract(addressdata); + } - if *filter != prev_filter { - // vecs.state.decrement(supply_state, price); + addressdata.receive(value, price); + + vecs.by_size_range.get_mut(amount).1.state.add(addressdata); + } else { + addressdata.receive(value, price); + + vecs.by_size_range + .get_mut(amount) + .1 + .state + .receive(value, price); } - // update vecs states if cohort changes groups - // - // empty addresses ? - // - // count change ? + // type count change ? }); }); } @@ -875,6 +1062,9 @@ impl Sats, Option>, Option, + usize, + f64, + bool, )> { fn process_sent( @@ -883,26 +1073,76 @@ impl addresstype_to_typeindex_to_addressdata: &mut AddressTypeToTypeIndexTree< WithAddressDataSource, >, + addresstype_to_typeindex_to_emptyaddressdata: &mut AddressTypeToTypeIndexTree< + WithAddressDataSource, + >, + price: Option, ) -> Result<()> { self.into_typed_vec() .into_iter() .try_for_each(|(_type, vec)| { - vec.into_iter() - .try_for_each(|(type_index, (value, addressdata_opt, price))| { - let addressdata_withsource = addresstype_to_typeindex_to_addressdata + vec.into_iter().try_for_each( + |( + type_index, + (value, addressdata_opt, prev_price, blocks_old, days_old, older_than_hour), + )| { + let typeindex_to_addressdata = addresstype_to_typeindex_to_addressdata .get_mut(_type) - .unwrap() + .unwrap(); + + let addressdata_withsource = typeindex_to_addressdata .entry(type_index) .or_insert(addressdata_opt.unwrap()); - addressdata_withsource.deref_mut().send(value, price)?; + let addressdata = addressdata_withsource.deref_mut(); + + let prev_amount = addressdata.amount(); + + let amount = prev_amount.checked_sub(value).unwrap(); + + let will_be_empty = addressdata.outputs_len - 1 == 0; + + if will_be_empty + || vecs.by_size_range.get_mut(amount).0.clone() + != vecs.by_size_range.get_mut(prev_amount).0.clone() + { + vecs.by_size_range + .get_mut(prev_amount) + .1 + .state + .subtract(addressdata); + + addressdata.send(value, prev_price)?; + + if will_be_empty { + let addressdata = + typeindex_to_addressdata.remove(&type_index).unwrap(); + + addresstype_to_typeindex_to_emptyaddressdata + .get_mut(_type) + .unwrap() + .insert(type_index, addressdata.into()); + } else { + vecs.by_size_range.get_mut(amount).1.state.add(addressdata); + } + } else { + addressdata.send(value, prev_price)?; + + vecs.by_size_range.get_mut(amount).1.state.send( + value, + price, + prev_price, + blocks_old, + days_old, + older_than_hour, + ); + } + + // type count change Ok(()) - - // move to empty if empty - - // update vecs states if cohort changes - }) + }, + ) }) } } diff --git a/crates/brk_computer/src/vecs/stateful/outputs.rs b/crates/brk_computer/src/vecs/stateful/outputs.rs index 684ffb57b..063b52d29 100644 --- a/crates/brk_computer/src/vecs/stateful/outputs.rs +++ b/crates/brk_computer/src/vecs/stateful/outputs.rs @@ -4,7 +4,6 @@ use brk_core::{ CheckedSub, Dollars, GroupFilter, HalvingEpoch, Height, Result, Timestamp, UTXOGroups, }; use brk_exit::Exit; -use brk_state::{BlockState, CohortStateTrait, Transacted}; use brk_vec::StoredIndex; use rayon::prelude::*; diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs index e30a1d0eb..7b9d02480 100644 --- a/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/vecs/stateful/utxo_cohort.rs @@ -3,12 +3,14 @@ use std::{ops::Deref, path::Path}; use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version}; use brk_exit::Exit; use brk_indexer::Indexer; -use brk_state::UTXOCohortState; use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format}; -use crate::vecs::{ - Indexes, fetched, indexes, market, - stateful::{common, r#trait::CohortVecs}, +use crate::{ + UTXOCohortState, + vecs::{ + Indexes, fetched, indexes, market, + stateful::{common, r#trait::CohortVecs}, + }, }; #[derive(Clone)] diff --git a/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs b/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs index e69a0f2ef..f9b848fbc 100644 --- a/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs @@ -1,22 +1,21 @@ use std::{collections::BTreeMap, ops::ControlFlow, path::Path}; -use brk_core::{ - CheckedSub, Dollars, GroupFilter, GroupedByDateRange, GroupedByEpoch, GroupedByFromDate, - GroupedByFromSize, GroupedBySizeRange, GroupedBySpendableType, GroupedByTerm, - GroupedByUpToDate, GroupedByUpToSize, HalvingEpoch, Height, Result, Timestamp, UTXOGroups, - Version, -}; +use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp, Version}; use brk_exit::Exit; -use brk_state::{BlockState, Transacted}; use brk_vec::{Computation, Format, StoredIndex}; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; -use crate::vecs::{ - Indexes, fetched, - stateful::{r#trait::CohortVecs, utxo_cohort}, +use crate::{ + GroupFilter, GroupedByDateRange, GroupedByEpoch, GroupedByFromDate, GroupedByFromSize, + GroupedBySizeRange, GroupedBySpendableType, GroupedByTerm, GroupedByUpToDate, + GroupedByUpToSize, UTXOGroups, + states::{BlockState, Transacted}, + vecs::{Indexes, fetched}, }; +use super::{r#trait::CohortVecs, utxo_cohort}; + const VERSION: Version = Version::new(0); #[derive(Clone, Deref, DerefMut)] @@ -1107,10 +1106,10 @@ impl Vecs { .timestamp .difference_in_days_between_float(last_timestamp); - let older_than_hour = - jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap()) - .as_second() - >= 60 * 60; + let older_than_hour = last_timestamp + .checked_sub(block_state.timestamp) + .unwrap() + .is_more_than_hour(); time_based_vecs .iter_mut() @@ -1289,4 +1288,10 @@ impl Vecs { vecs.compute_from_stateful(starting_indexes, &stateful, exit) }) } + + pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.as_mut_separate_vecs() + .par_iter_mut() + .try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit)) + } } diff --git a/crates/brk_core/src/lib.rs b/crates/brk_core/src/lib.rs index ffffbbe39..19850d2d1 100644 --- a/crates/brk_core/src/lib.rs +++ b/crates/brk_core/src/lib.rs @@ -1,13 +1,11 @@ #![doc = include_str!("../README.md")] mod error; -mod groups; mod structs; mod traits; mod utils; pub use error::*; -pub use groups::*; pub use structs::*; pub use traits::*; pub use utils::*; diff --git a/crates/brk_core/src/structs/addressdata.rs b/crates/brk_core/src/structs/addressdata.rs index 2489ae573..88665a5df 100644 --- a/crates/brk_core/src/structs/addressdata.rs +++ b/crates/brk_core/src/structs/addressdata.rs @@ -19,16 +19,13 @@ impl AddressData { } #[inline(always)] - pub fn is_empty(&self) -> bool { - if self.amount() == Sats::ZERO { - if self.outputs_len != 0 { - unreachable!(); - } + pub fn has_0_sats(&self) -> bool { + self.amount() == Sats::ZERO + } - true - } else { - false - } + #[inline(always)] + pub fn has_0_utxos(&self) -> bool { + self.outputs_len == 0 } pub fn receive(&mut self, amount: Sats, price: Option) { diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index b5b733eea..e74b90e69 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -75,6 +75,10 @@ impl Timestamp { } } } + + pub fn is_more_than_hour(&self) -> bool { + jiff::Timestamp::from(*self).as_second() >= 60 * 60 + } } impl From for Timestamp { diff --git a/crates/brk_state/Cargo.toml b/crates/brk_state/Cargo.toml deleted file mode 100644 index fc30b22fd..000000000 --- a/crates/brk_state/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "brk_state" -description = "Various states used in the Bitcoin Research Kit" -version.workspace = true -edition.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true - -[dependencies] -bincode = { workspace = true } -brk_core = { workspace = true } -derive_deref = { workspace = true } -serde = { workspace = true } -zerocopy = { workspace = true } -zerocopy-derive = { workspace = true } - -[package.metadata.cargo-machete] -ignored = ["zerocopy"] diff --git a/crates/brk_state/README.md b/crates/brk_state/README.md deleted file mode 100644 index 56e84e077..000000000 --- a/crates/brk_state/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# BRK State - -Various states used in the Bitcoin Research Kit diff --git a/crates/brk_state/src/cohorts/trait.rs b/crates/brk_state/src/cohorts/trait.rs deleted file mode 100644 index 4ac507e0e..000000000 --- a/crates/brk_state/src/cohorts/trait.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::path::Path; - -use brk_core::{Dollars, Height, Result}; - -use crate::{SupplyState, UnrealizedState}; - -pub trait CohortStateTrait: Sized { - fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result; - fn reset_single_iteration_values(&mut self); - fn increment(&mut self, supply_state: &SupplyState, price: Option); - fn decrement(&mut self, supply_state: &SupplyState, price: Option); - fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars); - fn receive(&mut self, supply_state: &SupplyState, price: Option); - fn send( - &mut self, - supply_state: &SupplyState, - current_price: Option, - prev_price: Option, - blocks_old: usize, - days_old: f64, - older_than_hour: bool, - ); - fn compute_unrealized_states( - &self, - height_price: Dollars, - date_price: Option, - ) -> (UnrealizedState, Option); - fn commit(&mut self, height: Height) -> Result<()>; -} diff --git a/crates/brk_state/src/cohorts/utxo.rs b/crates/brk_state/src/cohorts/utxo.rs deleted file mode 100644 index bfe580d0d..000000000 --- a/crates/brk_state/src/cohorts/utxo.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::path::Path; - -use brk_core::{Dollars, Height, Result}; -use derive_deref::{Deref, DerefMut}; - -use crate::{SupplyState, UnrealizedState}; - -use super::CohortState; - -#[derive(Clone, Deref, DerefMut)] -pub struct UTXOCohortState(CohortState); - -impl UTXOCohortState { - pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { - Ok(Self(CohortState::default_and_import( - path, - name, - compute_dollars, - )?)) - } -} - -// fn reset_single_iteration_values(&mut self) { -// self.0.reset_single_iteration_values(); -// } - -// fn increment(&mut self, supply_state: &SupplyState, price: Option) { -// self.0.increment(supply_state, price); -// } - -// fn decrement(&mut self, supply_state: &SupplyState, price: Option) { -// self.0.decrement(supply_state, price); -// } - -// fn decrement_price_to_amount(&mut self, supply_state: &SupplyState, price: Dollars) { -// self.0.decrement_price_to_amount(supply_state, price); -// } - -// fn receive(&mut self, supply_state: &SupplyState, price: Option) { -// self.0.receive(supply_state, price); -// } - -// fn send( -// &mut self, -// supply_state: &SupplyState, -// current_price: Option, -// prev_price: Option, -// blocks_old: usize, -// days_old: f64, -// older_than_hour: bool, -// ) { -// self.0.send( -// supply_state, -// current_price, -// prev_price, -// blocks_old, -// days_old, -// older_than_hour, -// ); -// } - -// fn compute_unrealized_states( -// &self, -// height_price: Dollars, -// date_price: Option, -// ) -> (UnrealizedState, Option) { -// self.0.compute_unrealized_states(height_price, date_price) -// } - -// fn commit(&mut self, height: Height) -> Result<()> { -// self.0.commit(height) -// } -// }