From e002a61a19f814cd6505cfc0c0c747ca7bdf742f Mon Sep 17 00:00:00 2001 From: nym21 Date: Thu, 22 May 2025 19:04:55 +0200 Subject: [PATCH] global: utxos part 5 --- Cargo.lock | 101 +- Cargo.toml | 2 +- crates/brk_computer/src/states/block.rs | 64 +- crates/brk_computer/src/states/cohort.rs | 166 +-- crates/brk_computer/src/states/hot.rs | 142 ++ crates/brk_computer/src/states/mod.rs | 12 +- .../src/states/outputs/by_epoch.rs | 47 +- .../src/states/outputs/by_from.rs | 82 +- .../src/states/outputs/by_range.rs | 50 +- .../src/states/outputs/by_size.rs | 93 +- .../src/states/outputs/by_spendable_type.rs | 134 ++ .../src/states/outputs/by_term.rs | 21 +- .../src/states/outputs/by_type.rs | 119 +- .../src/states/outputs/by_unspendable_type.rs | 43 + .../src/states/outputs/by_up_to.rs | 62 +- .../src/states/outputs/by_value.rs | 2 +- .../brk_computer/src/states/outputs/filter.rs | 14 + crates/brk_computer/src/states/outputs/mod.rs | 275 +++- crates/brk_computer/src/states/received.rs | 8 - crates/brk_computer/src/states/sent.rs | 7 - crates/brk_computer/src/states/supply.rs | 35 + .../brk_computer/src/vecs/grouped/builder.rs | 28 +- .../src/vecs/grouped/from_txindex.rs | 437 +++++- .../src/vecs/grouped/value_from_txindex.rs | 8 +- crates/brk_computer/src/vecs/transactions.rs | 1 + crates/brk_computer/src/vecs/utxos.rs | 1214 ----------------- crates/brk_computer/src/vecs/utxos/cohort.rs | 266 ++++ crates/brk_computer/src/vecs/utxos/mod.rs | 1149 ++++++++++++++++ crates/brk_core/src/structs/halvingepoch.rs | 6 + crates/brk_core/src/structs/sats.rs | 4 + crates/brk_core/src/structs/timestamp.rs | 25 +- crates/brk_fetcher/src/fetchers/binance.rs | 5 + crates/brk_fetcher/src/fetchers/kibo.rs | 5 + crates/brk_fetcher/src/fetchers/kraken.rs | 5 + crates/brk_fetcher/src/lib.rs | 10 +- crates/brk_query/src/vec_trees.rs | 8 +- crates/brk_server/Cargo.toml | 2 +- .../kibo.money/scripts/vecid-to-indexes.js | 18 + 38 files changed, 3047 insertions(+), 1623 deletions(-) create mode 100644 crates/brk_computer/src/states/hot.rs create mode 100644 crates/brk_computer/src/states/outputs/by_spendable_type.rs create mode 100644 crates/brk_computer/src/states/outputs/by_unspendable_type.rs create mode 100644 crates/brk_computer/src/states/outputs/filter.rs delete mode 100644 crates/brk_computer/src/states/received.rs delete mode 100644 crates/brk_computer/src/states/sent.rs create mode 100644 crates/brk_computer/src/states/supply.rs delete mode 100644 crates/brk_computer/src/vecs/utxos.rs create mode 100644 crates/brk_computer/src/vecs/utxos/cohort.rs create mode 100644 crates/brk_computer/src/vecs/utxos/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ca39f1ece..99c7cab43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1113,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1512,9 +1512,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1522,14 +1522,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] name = "jiff-static" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" dependencies = [ "proc-macro2", "quote", @@ -1819,9 +1819,9 @@ checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec" [[package]] name = "oxc" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e161476bfc4a98f1b451c11e29ba8bded86013387f619a635c9d201cb07664" +checksum = "9dd47d686dbfad71f2986f7baa019877d1ca6537d6c4b5de35c504bef34fd241" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1862,9 +1862,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffa6f20cba9bfb3486abc83c438f6a9278e4e030b6e9a16d2b5880132f96a1c" +checksum = "2c8248980c6d9db21f8ad42e0c85c172ef4dd20335522fc81e4ac72b6b70f806" dependencies = [ "allocator-api2", "bumpalo", @@ -1876,9 +1876,9 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406a3454475f817e71a4b8fc0a92f04e149730bc4c07d0d1803d5fc9ef75c357" +checksum = "a05110cb2af185324857a9a5d1a1986196e2cf3c5127cd90a7694c6b326e97c9" dependencies = [ "bitflags", "cow-utils", @@ -1893,10 +1893,11 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d894148693dad702ad668945908f8712fe260c23aaf69bbf9d63a8213a350cdd" +checksum = "24598056bb57788599997bbdb6ebf21a24b3331805aa9190c5b1204c973e636e" dependencies = [ + "phf", "proc-macro2", "quote", "syn 2.0.101", @@ -1904,9 +1905,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba3ad9293c9eed98116a01ef008229895d9640d8a1abca12aa54fdc588a62f3" +checksum = "a263a5d0fcb1fd60696b54cc5fd8fba020266522effdb48ae3c74744602e0116" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1916,9 +1917,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf77762b883cd93185b9b132c9bb4ad35084bbf3bc75cb8bcb6242c1eb6363c" +checksum = "e85ddc41b0e6af5aaa2e0936f4394f37b3e9890a78cd7b09f7b945a2b4ce6902" dependencies = [ "bitflags", "itertools", @@ -1931,9 +1932,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3add299d3a1b4148e4ab85e59bb5c855fbfb2405a4719aad2a199a802495ba0" +checksum = "1ec27365d210ed97419c0e6b31b0b30c9a3f85ca45087b23619e02477b4873ff" dependencies = [ "bitflags", "cow-utils", @@ -1952,15 +1953,18 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc6d1eb979f77be6685a7a67ee5d5124c66ef611c601a84327e7d339db69c41" +checksum = "112fcb78e9c0f3dda6beb1d93865f319d1c0165b0bf067fafda7f6529118328d" +dependencies = [ + "rustversion", +] [[package]] name = "oxc_diagnostics" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ba161cb61925de34f40b11c1d0d2f20e1894d5333d12f7c455a66244453512" +checksum = "f7b801ff7bbda76e11e0d5a43c06a80b3c105219e564c05e0b2fdb56d87b832a" dependencies = [ "cow-utils", "oxc-miette", @@ -1968,9 +1972,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c1827f0741fae82618b6129c7a3248e8334336879f4968cfce231dd65a9ebf" +checksum = "ea29dc44ee5ed0b63adb6d377dd07bf5870b6ae76d986965a92af1fac101dfd4" dependencies = [ "cow-utils", "num-bigint", @@ -1982,9 +1986,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c671fd76e9990c90762b7f7f7dd5c3038bf72e3295989b2a71ba11870a193b07" +checksum = "19ffa7908363d884399956c8971e924f1abba00ebe9995853bc70f536c0ccaaf" [[package]] name = "oxc_index" @@ -1994,9 +1998,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db19decb6e6513cd16f607b7e0ae9c91a9ebeb29b56b182a75c351d062d78102" +checksum = "5b53c9553ca1c89d65db3faa19cacc17cdb8b349ee94dd1e5443aafa395eb4ff" dependencies = [ "fixedbitset", "itertools", @@ -2011,9 +2015,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3555895501f211ca317d254e11b68ab6b8d27d65160338c5f573d462e68919" +checksum = "97b22dec40158a43da6954d6cc148f3d9f03cf57084ba4ce761a6ca6baa7f2d5" dependencies = [ "cow-utils", "oxc_allocator", @@ -2033,9 +2037,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "959f68446d66542753f2fe081189b729ed89f8ed5302de1a522640ff42eba31e" +checksum = "b0d4e22dc3630700e32320bbbc9f04396f109e8be2bb18791a00950485b049a5" dependencies = [ "bitflags", "cow-utils", @@ -2056,10 +2060,11 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7f5710da3fea0f40aaba14d547c61ac30c2840fb5d6a1ea9887766b72310c9" +checksum = "4691c62894d99f689da38419894e6e38c33862ac414206a65f0539d80c3ef252" dependencies = [ + "bitflags", "oxc_allocator", "oxc_ast_macros", "oxc_diagnostics", @@ -2071,9 +2076,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "425bec6c2ac20ff88573b8fbd87fc67aa97e49de9539979a98bcf79c37011077" +checksum = "8d6d869c2d2727114a9fd4b51459e7ce5025c5b7122195c227c6b02cc6500388" dependencies = [ "itertools", "oxc_allocator", @@ -2093,9 +2098,9 @@ dependencies = [ [[package]] name = "oxc_sourcemap" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd7bb37974a2684a080d05b9c28460e1610c5ac5ef13f481a45179f458239cb" +checksum = "24015d93ed1d8f0c2a0d9f534ca85690888990658a8fc4a87ff0c92640e73300" dependencies = [ "base64-simd", "cfg-if", @@ -2107,9 +2112,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc66dc0868f4492562d37733754ef147073410004a44551acb102cf2562f66b" +checksum = "b3d915cbabe501873b16236ae0e8ddfd5c001fd764b1dd073128858b5a0641a4" dependencies = [ "compact_str", "oxc-miette", @@ -2120,9 +2125,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bfa728cbbf2161b9afc3325addde64feeb39e8167a0ef1472ad1f0efbc9c48" +checksum = "9b6318faa445653106b56da5497a080e4fcf9231d5121035639d8c003f992378" dependencies = [ "bitflags", "cow-utils", @@ -2141,9 +2146,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528815f5909c4d1237f634f8482600dd08ac9293b3cc7839099f0f7abf327402" +checksum = "5c6c80fce6487edd19c71034f47ef039d7f736078d66f682e23ce3f3f5ab8d91" dependencies = [ "itoa", "oxc_allocator", @@ -2522,7 +2527,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2877,7 +2882,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2d1f60301..4ece442e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ clap_derive = "4.5.32" color-eyre = "0.6.4" derive_deref = "1.1.1" fjall = "2.10.0" -jiff = "0.2.13" +jiff = "0.2.14" log = { version = "0.4.27" } minreq = { version = "2.13.4", features = ["https", "serde_json"] } rayon = "1.10.0" diff --git a/crates/brk_computer/src/states/block.rs b/crates/brk_computer/src/states/block.rs index c6b3fc0d7..b0e00cb29 100644 --- a/crates/brk_computer/src/states/block.rs +++ b/crates/brk_computer/src/states/block.rs @@ -1,40 +1,64 @@ -#![allow(unused)] +use std::ops::{Add, AddAssign, SubAssign}; -use std::{ - iter::Sum, - ops::{Add, AddAssign, SubAssign}, -}; +use brk_core::{Dollars, Sats, Timestamp}; -use brk_core::{CheckedSub, Sats, StoredU32}; -use serde::Serialize; -use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use super::{OutputsByType, SupplyState}; -#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[derive(Debug, Clone)] pub struct BlockState { - pub utxos: usize, - pub value: Sats, + pub supply: SupplyState, + pub price: Option, + pub timestamp: Timestamp, } impl Add for BlockState { type Output = Self; - fn add(self, rhs: BlockState) -> Self::Output { - Self { - utxos: self.utxos + rhs.utxos, - value: self.value + rhs.value, - } + fn add(mut self, rhs: BlockState) -> Self::Output { + self.supply += &rhs.supply; + self } } impl AddAssign<&BlockState> for BlockState { fn add_assign(&mut self, rhs: &BlockState) { - self.utxos += rhs.utxos; - self.value += rhs.value; + self.supply += &rhs.supply; } } impl SubAssign for BlockState { fn sub_assign(&mut self, rhs: Self) { - self.utxos = self.utxos.checked_sub(rhs.utxos).unwrap(); - self.value = self.value.checked_sub(rhs.value).unwrap(); + self.supply -= rhs.supply; + } +} + +pub struct ReceivedBlockStateData<'a> { + pub received: &'a OutputsByType<(SupplyState, Vec)>, + pub timestamp: Timestamp, + pub price: Option, +} +impl<'a> From> for BlockState { + fn from( + ReceivedBlockStateData { + received, + timestamp, + price, + }: ReceivedBlockStateData<'a>, + ) -> Self { + let mut block_state = BlockState { + supply: SupplyState::default(), + price, + timestamp, + }; + received + .spendable + .as_vec() + .into_iter() + .for_each(|spendable_block_state| { + block_state.supply += &spendable_block_state.0; + }); + block_state.supply.value += received.unspendable.unknown.0.value; + block_state.supply.utxos += + received.unspendable.empty.0.utxos + received.unspendable.unknown.0.utxos; + block_state } } diff --git a/crates/brk_computer/src/states/cohort.rs b/crates/brk_computer/src/states/cohort.rs index e0c195ec6..5f3bca861 100644 --- a/crates/brk_computer/src/states/cohort.rs +++ b/crates/brk_computer/src/states/cohort.rs @@ -1,155 +1,29 @@ -#![allow(unused)] +use brk_core::{Bitcoin, CheckedSub, Dollars}; -use brk_core::{Dollars, Sats, StoredUsize}; +use super::SupplyState; // Vecs ? probably #[derive(Default, Clone)] pub struct CohortState { - pub realized_cap: Dollars, - pub supply: Sats, - pub utxo_count: StoredUsize, + pub supply: SupplyState, + pub realized_cap: Option, // pub price_to_amount: PriceToValue, save it not rounded in fjall } -pub struct OneShotSats { - pub price_paid_state: PricePaidState, - pub unrealized_block_state: UnrealizedState, - pub unrealized_date_state: Option, -} - -pub struct UnrealizedState { - supply_in_profit: Sats, - // supply_in_loss: Sats, - unrealized_profit: Dollars, - unrealized_loss: Dollars, -} - -// Why option ? -#[derive(Default, Debug)] -pub struct PricePaidState { - pp_p5: Option, - pp_p10: Option, - pp_p15: Option, - pp_p20: Option, - pp_p25: Option, - pp_p30: Option, - pp_p35: Option, - pp_p40: Option, - pp_p45: Option, - pp_median: Option, - pp_p55: Option, - pp_p60: Option, - pp_p65: Option, - pp_p70: Option, - pp_p75: Option, - pp_p80: Option, - pp_p85: Option, - pp_p90: Option, - pp_p95: Option, - - processed_amount: Sats, -} - -pub struct PricePaidStateFull { - pp_p1: Option, - pp_p2: Option, - pp_p3: Option, - pp_p4: Option, - pp_p5: Option, - pp_p6: Option, - pp_p7: Option, - pp_p8: Option, - pp_p9: Option, - pp_p10: Option, - pp_p11: Option, - pp_p12: Option, - pp_p13: Option, - pp_p14: Option, - pp_p15: Option, - pp_p16: Option, - pp_p17: Option, - pp_p18: Option, - pp_p19: Option, - pp_p20: Option, - pp_p21: Option, - pp_p22: Option, - pp_p23: Option, - pp_p24: Option, - pp_p25: Option, - pp_p26: Option, - pp_p27: Option, - pp_p28: Option, - pp_p29: Option, - pp_p30: Option, - pp_p31: Option, - pp_p32: Option, - pp_p33: Option, - pp_p34: Option, - pp_p35: Option, - pp_p36: Option, - pp_p37: Option, - pp_p38: Option, - pp_p39: Option, - pp_p40: Option, - pp_p41: Option, - pp_p42: Option, - pp_p43: Option, - pp_p44: Option, - pp_p45: Option, - pp_p46: Option, - pp_p47: Option, - pp_p48: Option, - pp_p49: Option, - pp_p50: Option, - pp_p51: Option, - pp_p52: Option, - pp_p53: Option, - pp_p54: Option, - pp_p55: Option, - pp_p56: Option, - pp_p57: Option, - pp_p58: Option, - pp_p59: Option, - pp_p60: Option, - pp_p61: Option, - pp_p62: Option, - pp_p63: Option, - pp_p64: Option, - pp_p65: Option, - pp_p66: Option, - pp_p67: Option, - pp_p68: Option, - pp_p69: Option, - pp_p70: Option, - pp_p71: Option, - pp_p72: Option, - pp_p73: Option, - pp_p74: Option, - pp_p75: Option, - pp_p76: Option, - pp_p77: Option, - pp_p78: Option, - pp_p79: Option, - pp_p80: Option, - pp_p81: Option, - pp_p82: Option, - pp_p83: Option, - pp_p84: Option, - pp_p85: Option, - pp_p86: Option, - pp_p87: Option, - pp_p88: Option, - pp_p89: Option, - pp_p90: Option, - pp_p91: Option, - pp_p92: Option, - pp_p93: Option, - pp_p94: Option, - pp_p95: Option, - pp_p96: Option, - pp_p97: Option, - pp_p98: Option, - pp_p99: Option, - - processed_amount: Sats, +impl CohortState { + pub fn increment(&mut self, supply_state: &SupplyState, price: Option) { + self.supply += supply_state; + if let Some(realized_cap) = self.realized_cap.as_mut() { + *realized_cap += price.unwrap() * Bitcoin::from(supply_state.value); + } + } + + pub fn decrement(&mut self, supply_state: SupplyState, price: Option) { + if let Some(realized_cap) = self.realized_cap.as_mut() { + *realized_cap = realized_cap + .checked_sub(price.unwrap() * Bitcoin::from(supply_state.value)) + .unwrap(); + } + self.supply -= supply_state; + } } diff --git a/crates/brk_computer/src/states/hot.rs b/crates/brk_computer/src/states/hot.rs new file mode 100644 index 000000000..ce7c8861f --- /dev/null +++ b/crates/brk_computer/src/states/hot.rs @@ -0,0 +1,142 @@ +pub struct OneShotSats { + pub price_paid_state: PricePaidState, + pub unrealized_block_state: UnrealizedState, + pub unrealized_date_state: Option, +} + +pub struct UnrealizedState { + supply_in_profit: Sats, + // supply_in_loss: Sats, + unrealized_profit: Dollars, + unrealized_loss: Dollars, +} + +// Why option ? +#[derive(Default, Debug)] +pub struct PricePaidState { + pp_p5: Option, + pp_p10: Option, + pp_p15: Option, + pp_p20: Option, + pp_p25: Option, + pp_p30: Option, + pp_p35: Option, + pp_p40: Option, + pp_p45: Option, + pp_median: Option, + pp_p55: Option, + pp_p60: Option, + pp_p65: Option, + pp_p70: Option, + pp_p75: Option, + pp_p80: Option, + pp_p85: Option, + pp_p90: Option, + pp_p95: Option, + + processed_amount: Sats, +} + +pub struct PricePaidStateFull { + pp_p1: Option, + pp_p2: Option, + pp_p3: Option, + pp_p4: Option, + pp_p5: Option, + pp_p6: Option, + pp_p7: Option, + pp_p8: Option, + pp_p9: Option, + pp_p10: Option, + pp_p11: Option, + pp_p12: Option, + pp_p13: Option, + pp_p14: Option, + pp_p15: Option, + pp_p16: Option, + pp_p17: Option, + pp_p18: Option, + pp_p19: Option, + pp_p20: Option, + pp_p21: Option, + pp_p22: Option, + pp_p23: Option, + pp_p24: Option, + pp_p25: Option, + pp_p26: Option, + pp_p27: Option, + pp_p28: Option, + pp_p29: Option, + pp_p30: Option, + pp_p31: Option, + pp_p32: Option, + pp_p33: Option, + pp_p34: Option, + pp_p35: Option, + pp_p36: Option, + pp_p37: Option, + pp_p38: Option, + pp_p39: Option, + pp_p40: Option, + pp_p41: Option, + pp_p42: Option, + pp_p43: Option, + pp_p44: Option, + pp_p45: Option, + pp_p46: Option, + pp_p47: Option, + pp_p48: Option, + pp_p49: Option, + pp_p50: Option, + pp_p51: Option, + pp_p52: Option, + pp_p53: Option, + pp_p54: Option, + pp_p55: Option, + pp_p56: Option, + pp_p57: Option, + pp_p58: Option, + pp_p59: Option, + pp_p60: Option, + pp_p61: Option, + pp_p62: Option, + pp_p63: Option, + pp_p64: Option, + pp_p65: Option, + pp_p66: Option, + pp_p67: Option, + pp_p68: Option, + pp_p69: Option, + pp_p70: Option, + pp_p71: Option, + pp_p72: Option, + pp_p73: Option, + pp_p74: Option, + pp_p75: Option, + pp_p76: Option, + pp_p77: Option, + pp_p78: Option, + pp_p79: Option, + pp_p80: Option, + pp_p81: Option, + pp_p82: Option, + pp_p83: Option, + pp_p84: Option, + pp_p85: Option, + pp_p86: Option, + pp_p87: Option, + pp_p88: Option, + pp_p89: Option, + pp_p90: Option, + pp_p91: Option, + pp_p92: Option, + pp_p93: Option, + pp_p94: Option, + pp_p95: Option, + pp_p96: Option, + pp_p97: Option, + pp_p98: Option, + pp_p99: Option, + + processed_amount: Sats, +} diff --git a/crates/brk_computer/src/states/mod.rs b/crates/brk_computer/src/states/mod.rs index 743422ef2..3405b3cc6 100644 --- a/crates/brk_computer/src/states/mod.rs +++ b/crates/brk_computer/src/states/mod.rs @@ -1,13 +1,13 @@ mod block; mod cohort; mod outputs; -mod realized; -mod received; -mod sent; +// mod realized; +// mod hot; +mod supply; pub use block::*; pub use cohort::*; pub use outputs::*; -pub use realized::*; -pub use received::*; -pub use sent::*; +// pub use realized::*; +// pub use hot::*; +pub use supply::*; diff --git a/crates/brk_computer/src/states/outputs/by_epoch.rs b/crates/brk_computer/src/states/outputs/by_epoch.rs index 5fc5d82b9..46e511bc9 100644 --- a/crates/brk_computer/src/states/outputs/by_epoch.rs +++ b/crates/brk_computer/src/states/outputs/by_epoch.rs @@ -1,20 +1,59 @@ +use brk_core::{HalvingEpoch, Height}; + +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsByEpoch { + pub _0: T, pub _1: T, pub _2: T, pub _3: T, pub _4: T, - pub _5: T, +} + +impl From> for OutputsByEpoch<(OutputFilter, T)> { + fn from(value: OutputsByEpoch) -> Self { + Self { + _0: (OutputFilter::Epoch(HalvingEpoch::new(0)), value._0), + _1: (OutputFilter::Epoch(HalvingEpoch::new(1)), value._1), + _2: (OutputFilter::Epoch(HalvingEpoch::new(2)), value._2), + _3: (OutputFilter::Epoch(HalvingEpoch::new(3)), value._3), + _4: (OutputFilter::Epoch(HalvingEpoch::new(4)), value._4), + } + } } impl OutputsByEpoch { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![ + pub fn as_mut_vec(&mut self) -> [&mut T; 5] { + [ + &mut self._0, &mut self._1, &mut self._2, &mut self._3, &mut self._4, - &mut self._5, ] } + + pub fn mut_vec_from_height(&mut self, height: Height) -> &mut T { + let epoch = HalvingEpoch::from(height); + if epoch == HalvingEpoch::new(0) { + &mut self._0 + } else if epoch == HalvingEpoch::new(1) { + &mut self._1 + } else if epoch == HalvingEpoch::new(2) { + &mut self._2 + } else if epoch == HalvingEpoch::new(3) { + &mut self._3 + } else if epoch == HalvingEpoch::new(4) { + &mut self._4 + } else { + todo!("") + } + } +} + +impl OutputsByEpoch<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 5] { + [&self._0.1, &self._1.1, &self._2.1, &self._3.1, &self._4.1] + } } diff --git a/crates/brk_computer/src/states/outputs/by_from.rs b/crates/brk_computer/src/states/outputs/by_from.rs index 27ad15ee8..929eb43ca 100644 --- a/crates/brk_computer/src/states/outputs/by_from.rs +++ b/crates/brk_computer/src/states/outputs/by_from.rs @@ -1,20 +1,98 @@ +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsByFrom { + pub _1d: T, + pub _1w: T, + pub _1m: T, + pub _2m: T, + pub _3m: T, + pub _4m: T, + pub _5m: T, + pub _6m: T, pub _1y: T, pub _2y: T, + pub _3y: T, pub _4y: T, + pub _5y: T, + pub _6y: T, + pub _7y: T, + pub _8y: T, pub _10y: T, pub _15y: T, } impl OutputsByFrom { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![ + pub fn as_mut_vec(&mut self) -> [&mut T; 18] { + [ + &mut self._1d, + &mut self._1w, + &mut self._1m, + &mut self._2m, + &mut self._3m, + &mut self._4m, + &mut self._5m, + &mut self._6m, &mut self._1y, &mut self._2y, + &mut self._3y, &mut self._4y, + &mut self._5y, + &mut self._6y, + &mut self._7y, + &mut self._8y, &mut self._10y, &mut self._15y, ] } } + +impl OutputsByFrom<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 18] { + [ + &self._1d.1, + &self._1w.1, + &self._1m.1, + &self._2m.1, + &self._3m.1, + &self._4m.1, + &self._5m.1, + &self._6m.1, + &self._1y.1, + &self._2y.1, + &self._3y.1, + &self._4y.1, + &self._5y.1, + &self._6y.1, + &self._7y.1, + &self._8y.1, + &self._10y.1, + &self._15y.1, + ] + } +} + +impl From> for OutputsByFrom<(OutputFilter, T)> { + fn from(value: OutputsByFrom) -> Self { + Self { + _1d: (OutputFilter::From(1), value._1d), + _1w: (OutputFilter::From(7), value._1w), + _1m: (OutputFilter::From(30), value._1m), + _2m: (OutputFilter::From(2 * 30), value._2m), + _3m: (OutputFilter::From(3 * 30), value._3m), + _4m: (OutputFilter::From(4 * 30), value._4m), + _5m: (OutputFilter::From(5 * 30), value._5m), + _6m: (OutputFilter::From(6 * 30), value._6m), + _1y: (OutputFilter::From(365), value._1y), + _2y: (OutputFilter::From(2 * 365), value._2y), + _3y: (OutputFilter::From(3 * 365), value._3y), + _4y: (OutputFilter::From(4 * 365), value._4y), + _5y: (OutputFilter::From(5 * 365), value._5y), + _6y: (OutputFilter::From(6 * 365), value._6y), + _7y: (OutputFilter::From(7 * 365), value._7y), + _8y: (OutputFilter::From(8 * 365), value._8y), + _10y: (OutputFilter::From(10 * 365), value._10y), + _15y: (OutputFilter::From(15 * 365), value._15y), + } + } +} diff --git a/crates/brk_computer/src/states/outputs/by_range.rs b/crates/brk_computer/src/states/outputs/by_range.rs index a96e6b850..a616b24af 100644 --- a/crates/brk_computer/src/states/outputs/by_range.rs +++ b/crates/brk_computer/src/states/outputs/by_range.rs @@ -1,3 +1,5 @@ +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsByRange { pub _1d_to_1w: T, @@ -7,15 +9,35 @@ pub struct OutputsByRange { pub _6m_to_1y: T, pub _1y_to_2y: T, pub _2y_to_3y: T, - pub _3y_to_5y: T, + pub _3y_to_4y: T, + pub _4y_to_5y: T, pub _5y_to_7y: T, pub _7y_to_10y: T, pub _10y_to_15y: T, } +impl From> for OutputsByRange<(OutputFilter, T)> { + fn from(value: OutputsByRange) -> Self { + Self { + _1d_to_1w: (OutputFilter::Range(1..7), value._1d_to_1w), + _1w_to_1m: (OutputFilter::Range(7..30), value._1w_to_1m), + _1m_to_3m: (OutputFilter::Range(30..3 * 30), value._1m_to_3m), + _3m_to_6m: (OutputFilter::Range(3 * 30..6 * 30), value._3m_to_6m), + _6m_to_1y: (OutputFilter::Range(6 * 30..365), value._6m_to_1y), + _1y_to_2y: (OutputFilter::Range(365..2 * 365), value._1y_to_2y), + _2y_to_3y: (OutputFilter::Range(2 * 365..3 * 365), value._2y_to_3y), + _3y_to_4y: (OutputFilter::Range(3 * 365..4 * 365), value._3y_to_4y), + _4y_to_5y: (OutputFilter::Range(4 * 365..5 * 365), value._4y_to_5y), + _5y_to_7y: (OutputFilter::Range(5 * 365..7 * 365), value._5y_to_7y), + _7y_to_10y: (OutputFilter::Range(7 * 365..10 * 365), value._7y_to_10y), + _10y_to_15y: (OutputFilter::Range(10 * 365..15 * 365), value._10y_to_15y), + } + } +} + impl OutputsByRange { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![ + pub fn as_mut_vec(&mut self) -> [&mut T; 12] { + [ &mut self._1d_to_1w, &mut self._1w_to_1m, &mut self._1m_to_3m, @@ -23,10 +45,30 @@ impl OutputsByRange { &mut self._6m_to_1y, &mut self._1y_to_2y, &mut self._2y_to_3y, - &mut self._3y_to_5y, + &mut self._3y_to_4y, + &mut self._4y_to_5y, &mut self._5y_to_7y, &mut self._7y_to_10y, &mut self._10y_to_15y, ] } } + +impl OutputsByRange<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 12] { + [ + &self._1d_to_1w.1, + &self._1w_to_1m.1, + &self._1m_to_3m.1, + &self._3m_to_6m.1, + &self._6m_to_1y.1, + &self._1y_to_2y.1, + &self._2y_to_3y.1, + &self._3y_to_4y.1, + &self._4y_to_5y.1, + &self._5y_to_7y.1, + &self._7y_to_10y.1, + &self._10y_to_15y.1, + ] + } +} diff --git a/crates/brk_computer/src/states/outputs/by_size.rs b/crates/brk_computer/src/states/outputs/by_size.rs index 6df9985f7..96f98800f 100644 --- a/crates/brk_computer/src/states/outputs/by_size.rs +++ b/crates/brk_computer/src/states/outputs/by_size.rs @@ -1,3 +1,7 @@ +use brk_core::Sats; + +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsBySize { pub from_1sat_to_10sats: T, @@ -16,9 +20,73 @@ pub struct OutputsBySize { pub from_100_000btc: T, } +impl From> for OutputsBySize<(OutputFilter, T)> { + fn from(value: OutputsBySize) -> Self { + #[allow(clippy::inconsistent_digit_grouping)] + Self { + from_1sat_to_10sats: ( + OutputFilter::Size(Sats::new(1)..Sats::new(10)), + value.from_1sat_to_10sats, + ), + from_10sats_to_100sats: ( + OutputFilter::Size(Sats::new(10)..Sats::new(100)), + value.from_10sats_to_100sats, + ), + from_100sats_to_1_000sats: ( + OutputFilter::Size(Sats::new(100)..Sats::new(1_000)), + value.from_100sats_to_1_000sats, + ), + from_1_000sats_to_10_000sats: ( + OutputFilter::Size(Sats::new(1_000)..Sats::new(10_000)), + value.from_1_000sats_to_10_000sats, + ), + from_10_000sats_to_100_000sats: ( + OutputFilter::Size(Sats::new(10_000)..Sats::new(100_000)), + value.from_10_000sats_to_100_000sats, + ), + from_100_000sats_to_1_000_000sats: ( + OutputFilter::Size(Sats::new(100_000)..Sats::new(1_000_000)), + value.from_100_000sats_to_1_000_000sats, + ), + from_1_000_000sats_to_10_000_000sats: ( + OutputFilter::Size(Sats::new(1_000_000)..Sats::new(10_000_000)), + value.from_1_000_000sats_to_10_000_000sats, + ), + from_10_000_000sats_to_1btc: ( + OutputFilter::Size(Sats::new(10_000_000)..Sats::new(1_00_000_000)), + value.from_10_000_000sats_to_1btc, + ), + from_1btc_to_10btc: ( + OutputFilter::Size(Sats::new(1_00_000_000)..Sats::new(10_00_000_000)), + value.from_1btc_to_10btc, + ), + from_10btc_to_100btc: ( + OutputFilter::Size(Sats::new(10_00_000_000)..Sats::new(100_00_000_000)), + value.from_10btc_to_100btc, + ), + from_100btc_to_1_000btc: ( + OutputFilter::Size(Sats::new(100_00_000_000)..Sats::new(1_000_00_000_000)), + value.from_100btc_to_1_000btc, + ), + from_1_000btc_to_10_000btc: ( + OutputFilter::Size(Sats::new(1_000_00_000_000)..Sats::new(10_000_00_000_000)), + value.from_1_000btc_to_10_000btc, + ), + from_10_000btc_to_100_000btc: ( + OutputFilter::Size(Sats::new(10_000_00_000_000)..Sats::new(100_000_00_000_000)), + value.from_10_000btc_to_100_000btc, + ), + from_100_000btc: ( + OutputFilter::Size(Sats::new(100_000_00_000_000)..Sats::MAX), + value.from_100_000btc, + ), + } + } +} + impl OutputsBySize { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![ + pub fn as_mut_vec(&mut self) -> [&mut T; 14] { + [ &mut self.from_1sat_to_10sats, &mut self.from_10sats_to_100sats, &mut self.from_100sats_to_1_000sats, @@ -36,3 +104,24 @@ impl OutputsBySize { ] } } + +impl OutputsBySize<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 14] { + [ + &self.from_1sat_to_10sats.1, + &self.from_10sats_to_100sats.1, + &self.from_100sats_to_1_000sats.1, + &self.from_1_000sats_to_10_000sats.1, + &self.from_10_000sats_to_100_000sats.1, + &self.from_100_000sats_to_1_000_000sats.1, + &self.from_1_000_000sats_to_10_000_000sats.1, + &self.from_10_000_000sats_to_1btc.1, + &self.from_1btc_to_10btc.1, + &self.from_10btc_to_100btc.1, + &self.from_100btc_to_1_000btc.1, + &self.from_1_000btc_to_10_000btc.1, + &self.from_10_000btc_to_100_000btc.1, + &self.from_100_000btc.1, + ] + } +} diff --git a/crates/brk_computer/src/states/outputs/by_spendable_type.rs b/crates/brk_computer/src/states/outputs/by_spendable_type.rs new file mode 100644 index 000000000..3bce18d3a --- /dev/null +++ b/crates/brk_computer/src/states/outputs/by_spendable_type.rs @@ -0,0 +1,134 @@ +use brk_core::{OutputType, Sats}; + +use crate::states::SupplyState; + +use super::OutputFilter; + +#[derive(Default, Clone)] +pub struct OutputsBySpendableType { + pub p2pk65: T, + pub p2pk33: T, + pub p2pkh: T, + pub p2ms: T, + pub p2sh: T, + pub p2wpkh: T, + pub p2wsh: T, + pub p2tr: T, + pub p2a: T, +} + +impl OutputsBySpendableType { + pub fn get(&self, output_type: OutputType) -> &T { + match output_type { + OutputType::P2PK65 => &self.p2pk65, + OutputType::P2PK33 => &self.p2pk33, + OutputType::P2PKH => &self.p2pkh, + OutputType::P2MS => &self.p2ms, + OutputType::P2SH => &self.p2sh, + OutputType::P2WPKH => &self.p2wpkh, + OutputType::P2WSH => &self.p2wsh, + OutputType::P2TR => &self.p2tr, + OutputType::P2A => &self.p2a, + _ => unreachable!(), + } + } + + pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { + match output_type { + OutputType::P2PK65 => &mut self.p2pk65, + OutputType::P2PK33 => &mut self.p2pk33, + OutputType::P2PKH => &mut self.p2pkh, + OutputType::P2MS => &mut self.p2ms, + OutputType::P2SH => &mut self.p2sh, + OutputType::P2WPKH => &mut self.p2wpkh, + OutputType::P2WSH => &mut self.p2wsh, + OutputType::P2TR => &mut self.p2tr, + OutputType::P2A => &mut self.p2a, + _ => unreachable!(), + } + } + + pub fn as_vec(&self) -> [&T; 9] { + [ + &self.p2pk65, + &self.p2pk33, + &self.p2pkh, + &self.p2ms, + &self.p2sh, + &self.p2wpkh, + &self.p2wsh, + &self.p2tr, + &self.p2a, + ] + } + + pub fn as_mut_vec(&mut self) -> [&mut T; 9] { + [ + &mut self.p2pk65, + &mut self.p2pk33, + &mut self.p2pkh, + &mut self.p2ms, + &mut self.p2sh, + &mut self.p2wpkh, + &mut self.p2wsh, + &mut self.p2tr, + &mut self.p2a, + ] + } + + pub fn as_typed_vec(&self) -> [(OutputType, &T); 9] { + [ + (OutputType::P2PK65, &self.p2pk65), + (OutputType::P2PK33, &self.p2pk33), + (OutputType::P2PKH, &self.p2pkh), + (OutputType::P2MS, &self.p2ms), + (OutputType::P2SH, &self.p2sh), + (OutputType::P2WPKH, &self.p2wpkh), + (OutputType::P2WSH, &self.p2wsh), + (OutputType::P2TR, &self.p2tr), + (OutputType::P2A, &self.p2a), + ] + } +} + +impl OutputsBySpendableType<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 9] { + [ + &self.p2pk65.1, + &self.p2pk33.1, + &self.p2pkh.1, + &self.p2ms.1, + &self.p2sh.1, + &self.p2wpkh.1, + &self.p2wsh.1, + &self.p2tr.1, + &self.p2a.1, + ] + } +} + +impl From> for OutputsBySpendableType<(OutputFilter, T)> { + fn from(value: OutputsBySpendableType) -> Self { + Self { + p2pk65: (OutputFilter::Type(OutputType::P2PK65), value.p2pk65), + p2pk33: (OutputFilter::Type(OutputType::P2PK33), value.p2pk33), + p2pkh: (OutputFilter::Type(OutputType::P2PKH), value.p2pkh), + p2ms: (OutputFilter::Type(OutputType::P2MS), value.p2ms), + p2sh: (OutputFilter::Type(OutputType::P2SH), value.p2sh), + p2wpkh: (OutputFilter::Type(OutputType::P2WPKH), value.p2wpkh), + p2wsh: (OutputFilter::Type(OutputType::P2WSH), value.p2wsh), + p2tr: (OutputFilter::Type(OutputType::P2TR), value.p2tr), + p2a: (OutputFilter::Type(OutputType::P2A), value.p2a), + } + } +} + +impl OutputsBySpendableType<(SupplyState, Vec)> { + pub fn reduce(&self) -> SupplyState { + let mut supply_state = SupplyState::default(); + self.as_vec().iter().for_each(|(supply_state_, _)| { + supply_state += supply_state_; + }); + supply_state + } +} diff --git a/crates/brk_computer/src/states/outputs/by_term.rs b/crates/brk_computer/src/states/outputs/by_term.rs index ad59dba99..72dbdd2ad 100644 --- a/crates/brk_computer/src/states/outputs/by_term.rs +++ b/crates/brk_computer/src/states/outputs/by_term.rs @@ -1,3 +1,5 @@ +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsByTerm { pub short: T, @@ -5,7 +7,22 @@ pub struct OutputsByTerm { } impl OutputsByTerm { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![&mut self.short, &mut self.long] + pub fn as_mut_vec(&mut self) -> [&mut T; 2] { + [&mut self.short, &mut self.long] + } +} + +impl OutputsByTerm<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 2] { + [&self.short.1, &self.long.1] + } +} + +impl From> for OutputsByTerm<(OutputFilter, T)> { + fn from(value: OutputsByTerm) -> Self { + Self { + long: (OutputFilter::From(155), value.long), + short: (OutputFilter::To(155), value.short), + } } } diff --git a/crates/brk_computer/src/states/outputs/by_type.rs b/crates/brk_computer/src/states/outputs/by_type.rs index 9ec4c9d7a..f2544a556 100644 --- a/crates/brk_computer/src/states/outputs/by_type.rs +++ b/crates/brk_computer/src/states/outputs/by_type.rs @@ -1,94 +1,61 @@ use brk_core::OutputType; +use super::{OutputsBySpendableType, OutputsByUnspendableType}; + #[derive(Default, Clone)] pub struct OutputsByType { - pub p2pk65: T, - pub p2pk33: T, - pub p2pkh: T, - pub p2ms: T, - pub p2sh: T, - pub op_return: T, - pub p2wpkh: T, - pub p2wsh: T, - pub p2tr: T, - pub p2a: T, - pub empty: T, - pub unknown: T, + pub spendable: OutputsBySpendableType, + pub unspendable: OutputsByUnspendableType, } impl OutputsByType { - pub fn get(&self, output_type: OutputType) -> &T { - match output_type { - OutputType::P2PK65 => &self.p2pk65, - OutputType::P2PK33 => &self.p2pk33, - OutputType::P2PKH => &self.p2pkh, - OutputType::P2MS => &self.p2ms, - OutputType::P2SH => &self.p2sh, - OutputType::OpReturn => &self.op_return, - OutputType::P2WPKH => &self.p2wpkh, - OutputType::P2WSH => &self.p2wsh, - OutputType::P2TR => &self.p2tr, - OutputType::P2A => &self.p2a, - OutputType::Empty => &self.empty, - OutputType::Unknown => &self.unknown, - } - } + // pub fn get(&self, output_type: OutputType) -> &T { + // match output_type { + // OutputType::P2PK65 => &self.spendable.p2pk65, + // OutputType::P2PK33 => &self.spendable.p2pk33, + // OutputType::P2PKH => &self.spendable.p2pkh, + // OutputType::P2MS => &self.spendable.p2ms, + // OutputType::P2SH => &self.spendable.p2sh, + // OutputType::P2WPKH => &self.spendable.p2wpkh, + // OutputType::P2WSH => &self.spendable.p2wsh, + // OutputType::P2TR => &self.spendable.p2tr, + // OutputType::P2A => &self.spendable.p2a, + // OutputType::OpReturn => &self.unspendable.op_return, + // OutputType::Empty => &self.unspendable.empty, + // OutputType::Unknown => &self.unspendable.unknown, + // } + // } pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { match output_type { - OutputType::P2PK65 => &mut self.p2pk65, - OutputType::P2PK33 => &mut self.p2pk33, - OutputType::P2PKH => &mut self.p2pkh, - OutputType::P2MS => &mut self.p2ms, - OutputType::P2SH => &mut self.p2sh, - OutputType::OpReturn => &mut self.op_return, - OutputType::P2WPKH => &mut self.p2wpkh, - OutputType::P2WSH => &mut self.p2wsh, - OutputType::P2TR => &mut self.p2tr, - OutputType::P2A => &mut self.p2a, - OutputType::Empty => &mut self.empty, - OutputType::Unknown => &mut self.unknown, + OutputType::P2PK65 => &mut self.spendable.p2pk65, + OutputType::P2PK33 => &mut self.spendable.p2pk33, + OutputType::P2PKH => &mut self.spendable.p2pkh, + OutputType::P2MS => &mut self.spendable.p2ms, + OutputType::P2SH => &mut self.spendable.p2sh, + OutputType::P2WPKH => &mut self.spendable.p2wpkh, + OutputType::P2WSH => &mut self.spendable.p2wsh, + OutputType::P2TR => &mut self.spendable.p2tr, + OutputType::P2A => &mut self.spendable.p2a, + OutputType::OpReturn => &mut self.unspendable.op_return, + OutputType::Empty => &mut self.unspendable.empty, + OutputType::Unknown => &mut self.unspendable.unknown, } } - pub fn to_spendable_vec(&self) -> Vec<&T> { - OutputType::as_vec() + pub fn as_vec(&self) -> Vec<&T> { + self.spendable + .as_vec() .into_iter() - .map(|t| (self.get(t))) + .chain(self.unspendable.as_vec()) .collect::>() } - pub fn as_vec(&mut self) -> Vec<&T> { - vec![ - &self.p2pk65, - &self.p2pk33, - &self.p2pkh, - &self.p2ms, - &self.p2sh, - &self.op_return, - &self.p2wpkh, - &self.p2wsh, - &self.p2tr, - &self.p2a, - &self.empty, - &self.unknown, - ] - } - - pub fn as_mut_vec(&mut self) -> Vec<&mut T> { - vec![ - &mut self.p2pk65, - &mut self.p2pk33, - &mut self.p2pkh, - &mut self.p2ms, - &mut self.p2sh, - &mut self.op_return, - &mut self.p2wpkh, - &mut self.p2wsh, - &mut self.p2tr, - &mut self.p2a, - &mut self.empty, - &mut self.unknown, - ] - } + // pub fn as_mut_vec(&mut self) -> Vec<&mut T> { + // self.spendable + // .as_mut_vec() + // .into_iter() + // .chain(self.unspendable.as_mut_vec()) + // .collect::>() + // } } diff --git a/crates/brk_computer/src/states/outputs/by_unspendable_type.rs b/crates/brk_computer/src/states/outputs/by_unspendable_type.rs new file mode 100644 index 000000000..38d44f6d1 --- /dev/null +++ b/crates/brk_computer/src/states/outputs/by_unspendable_type.rs @@ -0,0 +1,43 @@ +// use brk_core::OutputType; + +#[derive(Default, Clone)] +pub struct OutputsByUnspendableType { + pub op_return: T, + pub empty: T, + pub unknown: T, +} + +impl OutputsByUnspendableType { + // pub fn get(&self, output_type: OutputType) -> &T { + // match output_type { + // OutputType::OpReturn => &self.op_return, + // OutputType::Empty => &self.empty, + // OutputType::Unknown => &self.unknown, + // _ => unreachable!(), + // } + // } + + // pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { + // match output_type { + // OutputType::OpReturn => &mut self.op_return, + // OutputType::Empty => &mut self.empty, + // OutputType::Unknown => &mut self.unknown, + // _ => unreachable!(), + // } + // } + + // pub fn to_unspendable_vec(&self) -> Vec<&T> { + // OutputType::as_vec() + // .into_iter() + // .map(|t| (self.get(t))) + // .collect::>() + // } + + pub fn as_vec(&self) -> [&T; 3] { + [&self.op_return, &self.empty, &self.unknown] + } + + // pub fn as_mut_vec(&mut self) -> [&mut T; 3] { + // [&mut self.op_return, &mut self.empty, &mut self.unknown] + // } +} diff --git a/crates/brk_computer/src/states/outputs/by_up_to.rs b/crates/brk_computer/src/states/outputs/by_up_to.rs index 23ff5c492..9deefdc6c 100644 --- a/crates/brk_computer/src/states/outputs/by_up_to.rs +++ b/crates/brk_computer/src/states/outputs/by_up_to.rs @@ -1,3 +1,5 @@ +use super::OutputFilter; + #[derive(Default, Clone)] pub struct OutputsByUpTo { pub _1d: T, @@ -11,15 +13,18 @@ pub struct OutputsByUpTo { pub _1y: T, pub _2y: T, pub _3y: T, + pub _4y: T, pub _5y: T, + pub _6y: T, pub _7y: T, + pub _8y: T, pub _10y: T, pub _15y: T, } impl OutputsByUpTo { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - vec![ + pub fn as_mut_vec(&mut self) -> [&mut T; 18] { + [ &mut self._1d, &mut self._1w, &mut self._1m, @@ -31,10 +36,63 @@ impl OutputsByUpTo { &mut self._1y, &mut self._2y, &mut self._3y, + &mut self._4y, &mut self._5y, + &mut self._6y, &mut self._7y, + &mut self._8y, &mut self._10y, &mut self._15y, ] } } + +impl OutputsByUpTo<(OutputFilter, T)> { + pub fn vecs(&self) -> [&T; 18] { + [ + &self._1d.1, + &self._1w.1, + &self._1m.1, + &self._2m.1, + &self._3m.1, + &self._4m.1, + &self._5m.1, + &self._6m.1, + &self._1y.1, + &self._2y.1, + &self._3y.1, + &self._4y.1, + &self._5y.1, + &self._6y.1, + &self._7y.1, + &self._8y.1, + &self._10y.1, + &self._15y.1, + ] + } +} + +impl From> for OutputsByUpTo<(OutputFilter, T)> { + fn from(value: OutputsByUpTo) -> Self { + Self { + _1d: (OutputFilter::To(1), value._1d), + _1w: (OutputFilter::To(7), value._1w), + _1m: (OutputFilter::To(30), value._1m), + _2m: (OutputFilter::To(2 * 30), value._2m), + _3m: (OutputFilter::To(3 * 30), value._3m), + _4m: (OutputFilter::To(4 * 30), value._4m), + _5m: (OutputFilter::To(5 * 30), value._5m), + _6m: (OutputFilter::To(6 * 30), value._6m), + _1y: (OutputFilter::To(365), value._1y), + _2y: (OutputFilter::To(2 * 365), value._2y), + _3y: (OutputFilter::To(3 * 365), value._3y), + _4y: (OutputFilter::To(4 * 365), value._4y), + _5y: (OutputFilter::To(5 * 365), value._5y), + _6y: (OutputFilter::To(6 * 365), value._6y), + _7y: (OutputFilter::To(7 * 365), value._7y), + _8y: (OutputFilter::To(8 * 365), value._8y), + _10y: (OutputFilter::To(10 * 365), value._10y), + _15y: (OutputFilter::To(15 * 365), value._15y), + } + } +} diff --git a/crates/brk_computer/src/states/outputs/by_value.rs b/crates/brk_computer/src/states/outputs/by_value.rs index f4bdd98c1..8aa4e90fb 100644 --- a/crates/brk_computer/src/states/outputs/by_value.rs +++ b/crates/brk_computer/src/states/outputs/by_value.rs @@ -17,7 +17,7 @@ pub struct OutputsByValue { } impl OutputsByValue { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { + pub fn as_mut_vec(&mut self) -> Vec<&mut T> { vec![ &mut self.up_to_1cent, &mut self.from_1c_to_10c, diff --git a/crates/brk_computer/src/states/outputs/filter.rs b/crates/brk_computer/src/states/outputs/filter.rs new file mode 100644 index 000000000..834b7b79d --- /dev/null +++ b/crates/brk_computer/src/states/outputs/filter.rs @@ -0,0 +1,14 @@ +use std::ops::Range; + +use brk_core::{HalvingEpoch, OutputType, Sats}; + +#[derive(Clone)] +pub enum OutputFilter { + All, + To(usize), + Range(Range), + From(usize), + Size(Range), + Epoch(HalvingEpoch), + Type(OutputType), +} diff --git a/crates/brk_computer/src/states/outputs/mod.rs b/crates/brk_computer/src/states/outputs/mod.rs index 29eda42d5..0166c1488 100644 --- a/crates/brk_computer/src/states/outputs/mod.rs +++ b/crates/brk_computer/src/states/outputs/mod.rs @@ -1,45 +1,288 @@ -#![allow(unused)] +use brk_vec::StoredIndex; +use rayon::prelude::*; +use std::{collections::BTreeMap, ops::ControlFlow}; -use brk_core::OutputType; - -use crate::vecs::utxos::Vecs_; +use brk_core::{Dollars, HalvingEpoch, Height, Sats, Timestamp}; mod by_epoch; mod by_from; mod by_range; mod by_size; +mod by_spendable_type; mod by_term; mod by_type; +mod by_unspendable_type; mod by_up_to; -mod by_value; +// mod by_value; +mod filter; pub use by_epoch::*; pub use by_from::*; pub use by_range::*; pub use by_size::*; +pub use by_spendable_type::*; pub use by_term::*; pub use by_type::*; +pub use by_unspendable_type::*; pub use by_up_to::*; -pub use by_value::*; +// pub use by_value::*; +pub use filter::*; + +use crate::vecs; + +use super::{BlockState, SupplyState}; #[derive(Default, Clone)] pub struct Outputs { pub all: T, pub by_term: OutputsByTerm, - pub by_up_to: OutputsByUpTo, - pub by_from: OutputsByFrom, - pub by_range: OutputsByRange, - pub by_epoch: OutputsByEpoch, - pub by_size: OutputsBySize, - pub by_value: OutputsByValue, - pub by_type: OutputsByType, + // pub by_up_to: OutputsByUpTo, + // pub by_from: OutputsByFrom, + // pub by_range: OutputsByRange, + // pub by_epoch: OutputsByEpoch, + // pub by_size: OutputsBySize, + // // Needs whole UTXO set, TODO later + // // pub by_value: OutputsByValue, + // pub by_spendable_type: OutputsBySpendableType, } impl Outputs { - pub fn mut_flatten(&mut self) -> Vec<&mut T> { - [vec![&mut self.all], self.by_term.mut_flatten()] + pub fn as_mut_vec(&mut self) -> Vec<&mut T> { + [&mut self.all] .into_iter() - .flatten() + .chain(self.by_term.as_mut_vec()) + // .chain(self.by_up_to.as_mut_vec()) + // .chain(self.by_from.as_mut_vec()) + // .chain(self.by_range.as_mut_vec()) + // .chain(self.by_epoch.as_mut_vec()) + // .chain(self.by_size.as_mut_vec()) + // // .chain(self.by_value.as_mut_vec()) + // .chain(self.by_spendable_type.as_mut_vec()) .collect::>() } } + +impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> { + pub fn tick_tock_next_block(&mut self, chain_state: &[BlockState], timestamp: Timestamp) { + if chain_state.is_empty() { + return; + } + + let prev_timestamp = chain_state.last().unwrap().timestamp; + + self.by_term + .as_mut_vec() + .into_par_iter() + // .chain(self.by_up_to.as_mut_vec()) + // .chain(self.by_from.as_mut_vec()) + // .chain(self.by_range.as_mut_vec()) + .for_each(|(filter, v)| { + let state = &mut v.state; + + let mut check_days_old = |days_old: usize| -> bool { + match filter { + OutputFilter::From(from) => *from <= days_old, + OutputFilter::To(to) => *to > days_old, + OutputFilter::Range(range) => range.contains(&days_old), + OutputFilter::All + | OutputFilter::Epoch(_) + | OutputFilter::Size(_) + | OutputFilter::Type(_) => unreachable!(), + } + }; + + let _ = chain_state + .iter() + .try_for_each(|block_state| -> ControlFlow<()> { + let prev_days_old = block_state + .timestamp + .difference_in_days_between(prev_timestamp); + let days_old = block_state.timestamp.difference_in_days_between(timestamp); + if prev_days_old == days_old { + return ControlFlow::Continue(()); + } + + let is = check_days_old(days_old); + let was = check_days_old(prev_days_old); + if is && !was { + state.increment(&block_state.supply, block_state.price); + } else if was && !is { + state.decrement(block_state.supply.clone(), block_state.price); + } + + ControlFlow::Continue(()) + }); + }); + } + + pub fn send( + &mut self, + height_to_sent: BTreeMap)>>, + chain_state: &[BlockState], + ) { + let mut time_based_vecs = self + .by_term + .as_mut_vec() + .into_iter() + // .chain(self.by_up_to.as_mut_vec()) + // .chain(self.by_from.as_mut_vec()) + // .chain(self.by_range.as_mut_vec()) + // .chain(self.by_epoch.as_mut_vec()) + .collect::>(); + + let last_timestamp = chain_state.last().unwrap().timestamp; + + height_to_sent.into_iter().for_each(|(height, by_type)| { + let by_spendable_type = by_type.spendable; + + let block_state = chain_state.get(height.unwrap_to_usize()).unwrap(); + let price = block_state.price; + let supply_state = by_spendable_type.reduce(); + + let days_old = block_state + .timestamp + .difference_in_days_between(last_timestamp); + + self.all.1.state.decrement(supply_state.clone(), price); + + // by_spendable_type.as_typed_vec().into_iter().for_each( + // |(output_type, (supply_state, _))| { + // self.by_spendable_type + // .get_mut(output_type) + // .1 + // .state + // .decrement(supply_state.clone(), price) + // }, + // ); + + // by_spendable_type + // .as_vec() + // .into_iter() + // .flat_map(|(_, sats_received)| sats_received.iter()) + // .for_each(|sats| { + // let sats = *sats; + // self.by_size + // .as_mut_vec() + // .iter_mut() + // .filter(|(filter, _)| match filter { + // OutputFilter::Size(range) => range.contains(&sats), + // _ => unreachable!(), + // }) + // .for_each(|(_, vec)| { + // vec.state.decrement( + // SupplyState { + // utxos: 1, + // value: sats, + // }, + // price, + // ); + // }); + // }); + + time_based_vecs + .iter_mut() + .filter(|(filter, _)| match filter { + OutputFilter::From(from) => *from <= days_old, + OutputFilter::To(to) => *to > days_old, + OutputFilter::Range(range) => range.contains(&days_old), + OutputFilter::Epoch(epoch) => *epoch == HalvingEpoch::from(height), + _ => unreachable!(), + }) + .for_each(|(_, vecs)| { + vecs.state.decrement(supply_state.clone(), price); + }); + }); + } + + pub fn receive( + &mut self, + received: OutputsByType<(SupplyState, Vec)>, + height: Height, + price: Option, + ) { + let supply_state = received.spendable.reduce(); + + [ + &mut self.all.1, + &mut self.by_term.short.1, + // &mut self.by_epoch.mut_vec_from_height(height).1, + // Skip from and range as can't receive in the past + ] + .into_iter() + // .chain(self.by_up_to.as_mut_vec().map(|(_, v)| v)) + .for_each(|v| { + v.state.increment(&supply_state, price); + }); + + // let mut by_size = self.by_size.as_mut_vec(); + + // received + // .spendable + // .as_vec() + // .into_iter() + // .flat_map(|(_, sats_received)| sats_received.iter()) + // .for_each(|sats| { + // let sats = *sats; + // by_size + // .iter_mut() + // .filter(|(filter, _)| match filter { + // OutputFilter::Size(range) => range.contains(&sats), + // _ => unreachable!(), + // }) + // .for_each(|(_, vec)| { + // vec.state.increment( + // &SupplyState { + // utxos: 1, + // value: sats, + // }, + // price, + // ); + // }); + // }); + + // self.by_spendable_type + // .as_mut_vec() + // .into_iter() + // .for_each(|(filter, vecs)| { + // let output_type = match filter { + // OutputFilter::Type(output_type) => *output_type, + // _ => unreachable!(), + // }; + // vecs.state + // .increment(&received.spendable.get(output_type).0, price) + // }); + } +} + +impl Outputs<(OutputFilter, T)> { + pub fn vecs(&self) -> Vec<&T> { + [&self.all.1] + .into_iter() + .chain(self.by_term.vecs()) + // .chain(self.by_up_to.vecs()) + // .chain(self.by_from.vecs()) + // .chain(self.by_range.vecs()) + // .chain(self.by_epoch.vecs()) + // .chain(self.by_size.vecs()) + // // .chain(self.by_value.vecs()) + // .chain(self.by_spendable_type.vecs()) + .collect::>() + } +} + +impl From> for Outputs<(OutputFilter, T)> { + fn from(value: Outputs) -> Self { + Self { + all: (OutputFilter::All, value.all), + by_term: OutputsByTerm::from(value.by_term), + // by_up_to: OutputsByUpTo::from(value.by_up_to), + // by_from: OutputsByFrom::from(value.by_from), + // by_range: OutputsByRange::from(value.by_range), + // by_epoch: OutputsByEpoch::from(value.by_epoch), + // by_size: OutputsBySize::from(value.by_size), + // // Needs whole UTXO set, TODO later + // // by_value: OutputsByValue, + // by_spendable_type: OutputsBySpendableType::from(value.by_spendable_type), + } + } +} diff --git a/crates/brk_computer/src/states/received.rs b/crates/brk_computer/src/states/received.rs deleted file mode 100644 index a3f8266c1..000000000 --- a/crates/brk_computer/src/states/received.rs +++ /dev/null @@ -1,8 +0,0 @@ -use brk_core::{Sats, StoredUsize}; - -#[derive(Debug, Default)] -pub struct ReceivedState { - utxos: StoredUsize, - sats: Sats, - unspendable: Sats, -} diff --git a/crates/brk_computer/src/states/sent.rs b/crates/brk_computer/src/states/sent.rs deleted file mode 100644 index c351cfd50..000000000 --- a/crates/brk_computer/src/states/sent.rs +++ /dev/null @@ -1,7 +0,0 @@ -use brk_core::{Sats, StoredUsize}; - -#[derive(Debug, Default)] -pub struct SentState { - utxos: StoredUsize, - sats: Sats, -} diff --git a/crates/brk_computer/src/states/supply.rs b/crates/brk_computer/src/states/supply.rs new file mode 100644 index 000000000..3428d438c --- /dev/null +++ b/crates/brk_computer/src/states/supply.rs @@ -0,0 +1,35 @@ +use std::ops::{Add, AddAssign, SubAssign}; + +use brk_core::{CheckedSub, Sats}; +use serde::Serialize; +use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +pub struct SupplyState { + pub utxos: usize, + pub value: Sats, +} + +impl Add for SupplyState { + type Output = Self; + fn add(self, rhs: SupplyState) -> Self::Output { + Self { + utxos: self.utxos + rhs.utxos, + value: self.value + rhs.value, + } + } +} + +impl AddAssign<&SupplyState> for SupplyState { + fn add_assign(&mut self, rhs: &Self) { + self.utxos += rhs.utxos; + self.value += rhs.value; + } +} + +impl SubAssign for SupplyState { + fn sub_assign(&mut self, rhs: Self) { + self.utxos = self.utxos.checked_sub(rhs.utxos).unwrap(); + self.value = self.value.checked_sub(rhs.value).unwrap(); + } +} diff --git a/crates/brk_computer/src/vecs/grouped/builder.rs b/crates/brk_computer/src/vecs/grouped/builder.rs index 051ffd93d..bfcf7fb4f 100644 --- a/crates/brk_computer/src/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/vecs/grouped/builder.rs @@ -18,18 +18,18 @@ where I: StoredIndex, T: ComputedType, { - first: Option>>, - average: Option>>, - sum: Option>>, - max: Option>>, - _90p: Option>>, - _75p: Option>>, - median: Option>>, - _25p: Option>>, - _10p: Option>>, - min: Option>>, - last: Option>>, - total: Option>>, + pub first: Option>>, + pub average: Option>>, + pub sum: Option>>, + pub max: Option>>, + pub _90p: Option>>, + pub _75p: Option>>, + pub median: Option>>, + pub _25p: Option>>, + pub _10p: Option>>, + pub min: Option>>, + pub last: Option>>, + pub total: Option>>, } const VERSION: Version = Version::ZERO; @@ -537,7 +537,7 @@ where Ok(()) } - fn starting_index(&self, max_from: I) -> I { + pub fn starting_index(&self, max_from: I) -> I { max_from.min(I::from( self.vecs().into_iter().map(|v| v.len()).min().unwrap(), )) @@ -671,7 +671,7 @@ where Ok(()) } - fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { + pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> { if let Some(first) = self.first.as_mut() { first.validate_computed_version_or_reset_file(Version::ZERO + version)?; } diff --git a/crates/brk_computer/src/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/vecs/grouped/from_txindex.rs index 6d5a82d95..de37a9ff1 100644 --- a/crates/brk_computer/src/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/from_txindex.rs @@ -1,16 +1,17 @@ use std::path::Path; use brk_core::{ - DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, TxIndex, WeekIndex, - YearIndex, + Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex, + Sats, TxIndex, WeekIndex, YearIndex, }; use brk_exit::Exit; use brk_indexer::Indexer; use brk_vec::{ - AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version, + AnyCollectableVec, AnyVec, CollectableVec, Compressed, EagerVec, Result, StoredIndex, + VecIterator, Version, }; -use crate::vecs::{Indexes, indexes}; +use crate::vecs::{Indexes, fetched, indexes}; use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions}; @@ -77,37 +78,37 @@ where }) } - #[allow(unused)] - pub fn compute_all( - &mut self, - indexer: &Indexer, - indexes: &indexes::Vecs, - starting_indexes: &Indexes, - exit: &Exit, - mut compute: F, - ) -> color_eyre::Result<()> - where - F: FnMut( - &mut EagerVec, - &Indexer, - &indexes::Vecs, - &Indexes, - &Exit, - ) -> Result<()>, - { - compute( - self.txindex.as_mut().unwrap(), - indexer, - indexes, - starting_indexes, - exit, - )?; + // #[allow(unused)] + // pub fn compute_all( + // &mut self, + // indexer: &Indexer, + // indexes: &indexes::Vecs, + // starting_indexes: &Indexes, + // exit: &Exit, + // mut compute: F, + // ) -> color_eyre::Result<()> + // where + // F: FnMut( + // &mut EagerVec, + // &Indexer, + // &indexes::Vecs, + // &Indexes, + // &Exit, + // ) -> Result<()>, + // { + // compute( + // self.txindex.as_mut().unwrap(), + // indexer, + // indexes, + // starting_indexes, + // exit, + // )?; - let txindex: Option<&StoredVec> = None; - self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?; + // let txindex: Option<&StoredVec> = None; + // self.compute_rest(indexer, indexes, starting_indexes, exit, txindex)?; - Ok(()) - } + // Ok(()) + // } pub fn compute_rest( &mut self, @@ -116,7 +117,7 @@ where starting_indexes: &Indexes, exit: &Exit, txindex: Option<&impl CollectableVec>, - ) -> color_eyre::Result<()> { + ) -> Result<()> { if let Some(txindex) = txindex { self.height.compute( starting_indexes.height, @@ -137,6 +138,15 @@ where )?; } + self.compute_after_height(indexes, starting_indexes, exit) + } + + fn compute_after_height( + &mut self, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + ) -> Result<()> { self.dateindex.from_aligned( starting_indexes.dateindex, &self.height, @@ -216,3 +226,362 @@ where .collect::>() } } + +impl ComputedVecsFromTxindex { + pub fn compute_rest_from_sats( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + sats: &ComputedVecsFromTxindex, + txindex: Option<&impl CollectableVec>, + ) -> Result<()> { + let txindex_version = if let Some(txindex) = txindex { + txindex.version() + } else { + self.txindex.as_ref().unwrap().as_ref().version() + }; + + self.height + .validate_computed_version_or_reset_file(txindex_version)?; + + let starting_index = self.height.starting_index(starting_indexes.height); + + (starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len()) + .map(Height::from) + .try_for_each(|height| -> Result<()> { + if let Some(first) = self.height.first.as_mut() { + first.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_first() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(average) = self.height.average.as_mut() { + average.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_average() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(sum) = self.height.sum.as_mut() { + sum.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_sum() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(max) = self.height.max.as_mut() { + max.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_max() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(_90p) = self.height._90p.as_mut() { + _90p.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_90p() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(_75p) = self.height._75p.as_mut() { + _75p.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_75p() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(median) = self.height.median.as_mut() { + median.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_median() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(_25p) = self.height._25p.as_mut() { + _25p.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_25p() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(_10p) = self.height._10p.as_mut() { + _10p.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_10p() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(min) = self.height.min.as_mut() { + min.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_min() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(last) = self.height.last.as_mut() { + last.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_last() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + if let Some(total) = self.height.total.as_mut() { + total.forced_push_at( + height, + Bitcoin::from( + sats.height + .unwrap_total() + .into_iter() + .unwrap_get_inner(height), + ), + exit, + )?; + } + Ok(()) + })?; + + self.height.safe_flush(exit)?; + + self.compute_after_height(indexes, starting_indexes, exit) + } +} + +impl ComputedVecsFromTxindex { + pub fn compute_rest_from_bitcoin( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + starting_indexes: &Indexes, + exit: &Exit, + bitcoin: &ComputedVecsFromTxindex, + txindex: Option<&impl CollectableVec>, + fetched: &fetched::Vecs, + ) -> Result<()> { + let txindex_version = if let Some(txindex) = txindex { + txindex.version() + } else { + self.txindex.as_ref().unwrap().as_ref().version() + }; + + self.height + .validate_computed_version_or_reset_file(txindex_version)?; + + let starting_index = self.height.starting_index(starting_indexes.height); + + let mut close_iter = fetched.chainindexes_to_close.height.into_iter(); + + (starting_index.unwrap_to_usize()..indexer.vecs().height_to_weight.len()) + .map(Height::from) + .try_for_each(|height| -> Result<()> { + let price = *close_iter.unwrap_get_inner(height); + + if let Some(first) = self.height.first.as_mut() { + first.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_first() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(average) = self.height.average.as_mut() { + average.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_average() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(sum) = self.height.sum.as_mut() { + sum.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_sum() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(max) = self.height.max.as_mut() { + max.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_max() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(_90p) = self.height._90p.as_mut() { + _90p.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_90p() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(_75p) = self.height._75p.as_mut() { + _75p.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_75p() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(median) = self.height.median.as_mut() { + median.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_median() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(_25p) = self.height._25p.as_mut() { + _25p.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_25p() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(_10p) = self.height._10p.as_mut() { + _10p.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_10p() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(min) = self.height.min.as_mut() { + min.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_min() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(last) = self.height.last.as_mut() { + last.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_last() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + if let Some(total) = self.height.total.as_mut() { + total.forced_push_at( + height, + price + * bitcoin + .height + .unwrap_total() + .into_iter() + .unwrap_get_inner(height), + exit, + )?; + } + Ok(()) + })?; + + self.height.safe_flush(exit)?; + + self.compute_after_height(indexes, starting_indexes, exit) + } +} diff --git a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs index 5ccaea444..cc3b8b344 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs @@ -180,6 +180,7 @@ impl ComputedValueVecsFromTxindex { starting_indexes: &Indexes, exit: &Exit, txindex: Option<&impl CollectableVec>, + fetched: Option<&fetched::Vecs>, ) -> color_eyre::Result<()> { if let Some(txindex) = txindex { self.sats @@ -190,11 +191,12 @@ impl ComputedValueVecsFromTxindex { .compute_rest(indexer, indexes, starting_indexes, exit, txindex)?; } - self.bitcoin.compute_rest( + self.bitcoin.compute_rest_from_sats( indexer, indexes, starting_indexes, exit, + &self.sats, Some(&self.bitcoin_txindex), )?; @@ -203,12 +205,14 @@ impl ComputedValueVecsFromTxindex { dollars_txindex.compute_if_necessary(starting_indexes.txindex, exit)?; - dollars.compute_rest( + dollars.compute_rest_from_bitcoin( indexer, indexes, starting_indexes, exit, + &self.bitcoin, Some(dollars_txindex), + fetched.as_ref().unwrap(), )?; } diff --git a/crates/brk_computer/src/vecs/transactions.rs b/crates/brk_computer/src/vecs/transactions.rs index bbbac0228..6080c6bb4 100644 --- a/crates/brk_computer/src/vecs/transactions.rs +++ b/crates/brk_computer/src/vecs/transactions.rs @@ -784,6 +784,7 @@ impl Vecs { starting_indexes, exit, Some(&self.txindex_to_fee), + fetched, )?; self.indexes_to_feerate.compute_rest( diff --git a/crates/brk_computer/src/vecs/utxos.rs b/crates/brk_computer/src/vecs/utxos.rs deleted file mode 100644 index 94f3deaab..000000000 --- a/crates/brk_computer/src/vecs/utxos.rs +++ /dev/null @@ -1,1214 +0,0 @@ -use std::{cmp::Ordering, collections::BTreeMap, fs, mem, path::Path, thread}; - -use brk_core::{ - Bitcoin, CheckedSub, Dollars, Height, InputIndex, OutputIndex, OutputType, Sats, StoredUsize, -}; -use brk_exit::Exit; -use brk_indexer::Indexer; -use brk_vec::{ - AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Compressed, Computation, EagerVec, - GenericStoredVec, IndexedVec, Result, StoredIndex, StoredVec, VecIterator, Version, -}; -use derive_deref::{Deref, DerefMut}; -use rayon::prelude::*; - -use crate::states::{ - BlockState, CohortState, Outputs, OutputsByEpoch, OutputsByFrom, OutputsByRange, OutputsBySize, - OutputsByTerm, OutputsByType, OutputsByUpTo, OutputsByValue, RealizedState, ReceivedState, - SentState, -}; - -use super::{ - Indexes, fetched, - grouped::{ComputedValueVecsFromHeight, ComputedVecsFromHeight, StorableVecGeneatorOptions}, - indexes, transactions, -}; - -const VERSION: Version = Version::new(3); - -#[derive(Clone)] -pub struct Vecs { - chain_state: Vec, - saved_chain_state: StoredVec, - - // unspendable - // cointime ? - pub height_to_unspendable_supply: EagerVec, - pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, - utxos_vecs: Outputs, -} - -impl Vecs { - pub fn forced_import( - path: &Path, - _computation: Computation, - compressed: Compressed, - fetched: Option<&fetched::Vecs>, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); - - let mut states_path = path.to_owned(); - states_path.pop(); - states_path = states_path.join("states"); - - Ok(Self { - chain_state: vec![], - saved_chain_state: StoredVec::forced_import( - &states_path, - "chain", - Version::ZERO, - Compressed::NO, - )?, - - height_to_unspendable_supply: EagerVec::forced_import( - path, - "unspendable_supply", - VERSION + Version::ZERO, - compressed, - )?, - indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( - path, - "unspendable_supply", - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - compute_dollars, - )?, - utxos_vecs: { - Outputs { - all: Vecs_::forced_import(path, None, _computation, compressed, fetched)?, - by_term: OutputsByTerm { - short: Vecs_::forced_import( - path, - Some("sth"), - _computation, - compressed, - fetched, - )?, - long: Vecs_::forced_import( - path, - Some("lth"), - _computation, - compressed, - fetched, - )?, - }, - by_up_to: OutputsByUpTo { - _1d: Vecs_::forced_import( - path, - Some("up_to_1d"), - _computation, - compressed, - fetched, - )?, - _1w: Vecs_::forced_import( - path, - Some("up_to_1w"), - _computation, - compressed, - fetched, - )?, - _1m: Vecs_::forced_import( - path, - Some("up_to_1m"), - _computation, - compressed, - fetched, - )?, - _2m: Vecs_::forced_import( - path, - Some("up_to_2m"), - _computation, - compressed, - fetched, - )?, - _3m: Vecs_::forced_import( - path, - Some("up_to_3m"), - _computation, - compressed, - fetched, - )?, - _4m: Vecs_::forced_import( - path, - Some("up_to_4m"), - _computation, - compressed, - fetched, - )?, - _5m: Vecs_::forced_import( - path, - Some("up_to_5m"), - _computation, - compressed, - fetched, - )?, - _6m: Vecs_::forced_import( - path, - Some("up_to_6m"), - _computation, - compressed, - fetched, - )?, - _1y: Vecs_::forced_import( - path, - Some("up_to_1y"), - _computation, - compressed, - fetched, - )?, - _2y: Vecs_::forced_import( - path, - Some("up_to_2y"), - _computation, - compressed, - fetched, - )?, - _3y: Vecs_::forced_import( - path, - Some("up_to_3y"), - _computation, - compressed, - fetched, - )?, - _5y: Vecs_::forced_import( - path, - Some("up_to_5y"), - _computation, - compressed, - fetched, - )?, - _7y: Vecs_::forced_import( - path, - Some("up_to_7y"), - _computation, - compressed, - fetched, - )?, - _10y: Vecs_::forced_import( - path, - Some("up_to_10y"), - _computation, - compressed, - fetched, - )?, - _15y: Vecs_::forced_import( - path, - Some("up_to_15y"), - _computation, - compressed, - fetched, - )?, - }, - by_from: OutputsByFrom { - _1y: Vecs_::forced_import( - path, - Some("from_1y"), - _computation, - compressed, - fetched, - )?, - _2y: Vecs_::forced_import( - path, - Some("from_2y"), - _computation, - compressed, - fetched, - )?, - _4y: Vecs_::forced_import( - path, - Some("from_4y"), - _computation, - compressed, - fetched, - )?, - _10y: Vecs_::forced_import( - path, - Some("from_10y"), - _computation, - compressed, - fetched, - )?, - _15y: Vecs_::forced_import( - path, - Some("from_15y"), - _computation, - compressed, - fetched, - )?, - }, - by_range: OutputsByRange { - _1d_to_1w: Vecs_::forced_import( - path, - Some("from_1d_to_1w"), - _computation, - compressed, - fetched, - )?, - _1w_to_1m: Vecs_::forced_import( - path, - Some("from_1w_to_1m"), - _computation, - compressed, - fetched, - )?, - _1m_to_3m: Vecs_::forced_import( - path, - Some("from_1m_to_3m"), - _computation, - compressed, - fetched, - )?, - _3m_to_6m: Vecs_::forced_import( - path, - Some("from_3m_to_6m"), - _computation, - compressed, - fetched, - )?, - _6m_to_1y: Vecs_::forced_import( - path, - Some("from_6m_to_1y"), - _computation, - compressed, - fetched, - )?, - _1y_to_2y: Vecs_::forced_import( - path, - Some("from_1y_to_2y"), - _computation, - compressed, - fetched, - )?, - _2y_to_3y: Vecs_::forced_import( - path, - Some("from_2y_to_3y"), - _computation, - compressed, - fetched, - )?, - _3y_to_5y: Vecs_::forced_import( - path, - Some("from_3y_to_5y"), - _computation, - compressed, - fetched, - )?, - _5y_to_7y: Vecs_::forced_import( - path, - Some("from_5y_to_7y"), - _computation, - compressed, - fetched, - )?, - _7y_to_10y: Vecs_::forced_import( - path, - Some("from_7y_to_10y"), - _computation, - compressed, - fetched, - )?, - _10y_to_15y: Vecs_::forced_import( - path, - Some("from_10y_to_15y"), - _computation, - compressed, - fetched, - )?, - }, - by_epoch: OutputsByEpoch { - _1: Vecs_::forced_import( - path, - Some("epoch_1"), - _computation, - compressed, - fetched, - )?, - _2: Vecs_::forced_import( - path, - Some("epoch_2"), - _computation, - compressed, - fetched, - )?, - _3: Vecs_::forced_import( - path, - Some("epoch_3"), - _computation, - compressed, - fetched, - )?, - _4: Vecs_::forced_import( - path, - Some("epoch_4"), - _computation, - compressed, - fetched, - )?, - _5: Vecs_::forced_import( - path, - Some("epoch_5"), - _computation, - compressed, - fetched, - )?, - }, - by_size: OutputsBySize { - from_1sat_to_10sats: Vecs_::forced_import( - path, - Some("from_1sat_to_10sats"), - _computation, - compressed, - fetched, - )?, - from_10sats_to_100sats: Vecs_::forced_import( - path, - Some("from_10sats_to_100sats"), - _computation, - compressed, - fetched, - )?, - from_100sats_to_1_000sats: Vecs_::forced_import( - path, - Some("from_100sats_to_1_000sats"), - _computation, - compressed, - fetched, - )?, - from_1_000sats_to_10_000sats: Vecs_::forced_import( - path, - Some("from_1_000sats_to_10_000sats"), - _computation, - compressed, - fetched, - )?, - from_10_000sats_to_100_000sats: Vecs_::forced_import( - path, - Some("from_10_000sats_to_100_000sats"), - _computation, - compressed, - fetched, - )?, - from_100_000sats_to_1_000_000sats: Vecs_::forced_import( - path, - Some("from_100_000sats_to_1_000_000sats"), - _computation, - compressed, - fetched, - )?, - from_1_000_000sats_to_10_000_000sats: Vecs_::forced_import( - path, - Some("from_1_000_000sats_to_10_000_000sats"), - _computation, - compressed, - fetched, - )?, - from_10_000_000sats_to_1btc: Vecs_::forced_import( - path, - Some("from_10_000_000sats_to_1btc"), - _computation, - compressed, - fetched, - )?, - from_1btc_to_10btc: Vecs_::forced_import( - path, - Some("from_1btc_to_10btc"), - _computation, - compressed, - fetched, - )?, - from_10btc_to_100btc: Vecs_::forced_import( - path, - Some("from_10btc_to_100btc"), - _computation, - compressed, - fetched, - )?, - from_100btc_to_1_000btc: Vecs_::forced_import( - path, - Some("from_100btc_to_1_000btc"), - _computation, - compressed, - fetched, - )?, - from_1_000btc_to_10_000btc: Vecs_::forced_import( - path, - Some("from_1_000btc_to_10_000btc"), - _computation, - compressed, - fetched, - )?, - from_10_000btc_to_100_000btc: Vecs_::forced_import( - path, - Some("from_10_000btc_to_100_000btc"), - _computation, - compressed, - fetched, - )?, - from_100_000btc: Vecs_::forced_import( - path, - Some("from_100_000btc"), - _computation, - compressed, - fetched, - )?, - }, - by_value: OutputsByValue { - up_to_1cent: Vecs_::forced_import( - path, - Some("up_to_1cent"), - _computation, - compressed, - fetched, - )?, - from_1c_to_10c: Vecs_::forced_import( - path, - Some("from_1c_to_10c"), - _computation, - compressed, - fetched, - )?, - from_10c_to_1d: Vecs_::forced_import( - path, - Some("from_10c_to_1d"), - _computation, - compressed, - fetched, - )?, - from_1d_to_10d: Vecs_::forced_import( - path, - Some("from_1d_to_10d"), - _computation, - compressed, - fetched, - )?, - from_10usd_to_100usd: Vecs_::forced_import( - path, - Some("from_10usd_to_100usd"), - _computation, - compressed, - fetched, - )?, - from_100usd_to_1_000usd: Vecs_::forced_import( - path, - Some("from_100usd_to_1_000usd"), - _computation, - compressed, - fetched, - )?, - from_1_000usd_to_10_000usd: Vecs_::forced_import( - path, - Some("from_1_000usd_to_10_000usd"), - _computation, - compressed, - fetched, - )?, - from_10_000usd_to_100_000usd: Vecs_::forced_import( - path, - Some("from_10_000usd_to_100_000usd"), - _computation, - compressed, - fetched, - )?, - from_100_000usd_to_1_000_000usd: Vecs_::forced_import( - path, - Some("from_100_000usd_to_1_000_000usd"), - _computation, - compressed, - fetched, - )?, - from_1_000_000usd_to_10_000_000usd: Vecs_::forced_import( - path, - Some("from_1_000_000usd_to_10_000_000usd"), - _computation, - compressed, - fetched, - )?, - from_10_000_000usd_to_100_000_000usd: Vecs_::forced_import( - path, - Some("from_10_000_000usd_to_100_000_000usd"), - _computation, - compressed, - fetched, - )?, - from_100_000_000usd_to_1_000_000_000usd: Vecs_::forced_import( - path, - Some("from_100_000_000usd_to_1_000_000_000usd"), - _computation, - compressed, - fetched, - )?, - from_1_000_000_000usd: Vecs_::forced_import( - path, - Some("from_1_000_000_000usd"), - _computation, - compressed, - fetched, - )?, - }, - by_type: OutputsByType { - p2pk65: Vecs_::forced_import( - path, - Some("p2pk65"), - _computation, - compressed, - fetched, - )?, - p2pk33: Vecs_::forced_import( - path, - Some("p2pk33"), - _computation, - compressed, - fetched, - )?, - p2pkh: Vecs_::forced_import( - path, - Some("p2pkh"), - _computation, - compressed, - fetched, - )?, - p2ms: Vecs_::forced_import( - path, - Some("p2ms"), - _computation, - compressed, - fetched, - )?, - p2sh: Vecs_::forced_import( - path, - Some("p2sh"), - _computation, - compressed, - fetched, - )?, - op_return: Vecs_::forced_import( - path, - Some("op_return"), - _computation, - compressed, - fetched, - )?, - p2wpkh: Vecs_::forced_import( - path, - Some("p2wpkh"), - _computation, - compressed, - fetched, - )?, - p2wsh: Vecs_::forced_import( - path, - Some("p2wsh"), - _computation, - compressed, - fetched, - )?, - p2tr: Vecs_::forced_import( - path, - Some("p2tr"), - _computation, - compressed, - fetched, - )?, - p2a: Vecs_::forced_import( - path, - Some("p2a"), - _computation, - compressed, - fetched, - )?, - empty: Vecs_::forced_import( - path, - Some("empty"), - _computation, - compressed, - fetched, - )?, - unknown: Vecs_::forced_import( - path, - Some("unknown"), - _computation, - compressed, - fetched, - )?, - }, - } - }, - }) - } - - pub fn compute( - &mut self, - indexer: &Indexer, - indexes: &indexes::Vecs, - transactions: &transactions::Vecs, - fetched: Option<&fetched::Vecs>, - starting_indexes: &Indexes, - exit: &Exit, - ) -> color_eyre::Result<()> { - let indexer_vecs = indexer.vecs(); - - let height_to_first_outputindex = &indexer_vecs.height_to_first_outputindex; - let height_to_first_inputindex = &indexer_vecs.height_to_first_inputindex; - let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_sum(); - let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_sum(); - let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex; - let outputindex_to_value = &indexer_vecs.outputindex_to_value; - let txindex_to_height = &indexes.txindex_to_height; - let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; - let outputindex_to_txindex = &indexes.outputindex_to_txindex; - let outputindex_to_outputtype = &indexer_vecs.outputindex_to_outputtype; - let height_to_unclaimed_rewards = transactions - .indexes_to_unclaimed_rewards - .sats - .height - .as_ref() - .unwrap() - .as_ref(); - let height_to_close = &fetched - .as_ref() - .map(|fetched| &fetched.chainindexes_to_close.height); - - let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); - let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); - let mut height_to_output_count_iter = height_to_output_count.into_iter(); - let mut height_to_input_count_iter = height_to_input_count.into_iter(); - let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter(); - let mut outputindex_to_value_iter = outputindex_to_value.into_iter(); - let mut outputindex_to_value_iter_2 = outputindex_to_value.into_iter(); - let mut txindex_to_height_iter = txindex_to_height.into_iter(); - let mut outputindex_to_txindex_iter = outputindex_to_txindex.into_iter(); - let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter()); - let mut outputindex_to_outputtype_iter = outputindex_to_outputtype.into_iter(); - let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter(); - let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); - let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); - - let mut flat_vecs_ = self.utxos_vecs.mut_flatten(); - - let base_version = Version::ZERO - + height_to_first_outputindex.version() - + height_to_first_inputindex.version() - + height_to_timestamp_fixed.version() - + height_to_output_count.version() - + height_to_input_count.version() - + inputindex_to_outputindex.version() - + outputindex_to_value.version() - + txindex_to_height.version() - + outputindex_to_txindex.version() - + outputindex_to_outputtype.version() - + height_to_unclaimed_rewards.version() - + height_to_close - .as_ref() - .map_or(Version::ZERO, |v| v.version()); - - flat_vecs_ - .iter_mut() - .try_for_each(|v| v.validate_computed_versions(base_version))?; - self.height_to_unspendable_supply - .validate_computed_version_or_reset_file( - base_version + self.height_to_unspendable_supply.inner_version(), - )?; - - let chain_state_starting_height = Height::from(self.saved_chain_state.len()); - - let starting_height = flat_vecs_ - .iter_mut() - .map(|v| v.init(starting_indexes)) - .min() - .unwrap_or_default() - .min(chain_state_starting_height); - - // self.state.unspendable_supply = self - // .height_to_unspendable_supply - // .into_iter() - // .unwrap_get_inner(prev_height); - // - // / self.state.unspendable_supply = self - // .height_to_unspendable_supply - // . - // starting_height - - match starting_height.cmp(&chain_state_starting_height) { - Ordering::Greater => unreachable!(), - Ordering::Equal => { - self.chain_state = self.saved_chain_state.collect_range(None, None)?; - } - Ordering::Less => { - todo!("rollback instead"); - // self.chain_state = vec![]; - // starting_height = Height::ZERO; - } - } - - let mut height = Height::ZERO; - (starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len()) - .map(Height::from) - .try_for_each(|_height| -> color_eyre::Result<()> { - height = _height; - - // let sent_state = SentState::default(); - // let received_state = ReceivedState::default(); - // let realized_state = RealizedState::default(); - - let first_outputindex = height_to_first_outputindex_iter - .unwrap_get_inner(height) - .unwrap_to_usize(); - let first_inputindex = height_to_first_inputindex_iter - .unwrap_get_inner(height) - .unwrap_to_usize(); - let output_count = height_to_output_count_iter.unwrap_get_inner(height); - let input_count = height_to_input_count_iter.unwrap_get_inner(height); - // let opreturn_count = height_to_opreturn_count_iter.unwrap_get_inner(height); - - let ((height_to_sent_data, block_state_to_subtract), mut received) = - thread::scope(|s| { - let sent = s.spawn(|| { - let mut txindex_to_height = BTreeMap::new(); - let mut height_to_sent_data = BTreeMap::new(); - let mut block_state_to_subtract = BlockState::default(); - - // Skip coinbase - (first_inputindex + 1..first_inputindex + *input_count) - .map(InputIndex::from) - .map(|inputindex| { - inputindex_to_outputindex_iter.unwrap_get_inner(inputindex) - }) - .for_each(|outputindex| { - let value = - outputindex_to_value_iter.unwrap_get_inner(outputindex); - - let _type = outputindex_to_outputtype_iter - .unwrap_get_inner(outputindex); - - let txindex = - outputindex_to_txindex_iter.unwrap_get_inner(outputindex); - - let input_height = - *txindex_to_height.entry(txindex).or_insert_with(|| { - txindex_to_height_iter.unwrap_get_inner(txindex) - }); - - match input_height.cmp(&height) { - Ordering::Equal => { - block_state_to_subtract.utxos += 1; - block_state_to_subtract.value += value; - } - Ordering::Greater => unreachable!(), - Ordering::Less => { - let block_state = self - .chain_state - .get_mut(input_height.unwrap_to_usize()) - .unwrap(); - block_state.utxos -= 1; - block_state.value -= value; - } - } - - let input_data = height_to_sent_data - .entry(input_height) - .or_insert_with(|| { - let timestamp = height_to_timestamp_fixed_iter - .unwrap_get_inner(input_height); - - if let Some(height_to_close_iter) = - height_to_close_iter.as_mut() - { - let dollars = *height_to_close_iter - .unwrap_get_inner(input_height); - - (timestamp, dollars, Sats::ZERO, _type) - } else { - (timestamp, Dollars::ZERO, Sats::ZERO, _type) - } - }); - - input_data.2 += value; - }); - - (height_to_sent_data, block_state_to_subtract) - }); - - let received = s.spawn(|| { - let mut by_type: OutputsByType = OutputsByType::default(); - - (first_outputindex..first_outputindex + *output_count) - .map(OutputIndex::from) - .for_each(|outputindex| { - let value = - outputindex_to_value_iter_2.unwrap_get_inner(outputindex); - - let outputtype = outputindex_to_outputtype_iter_2 - .unwrap_get_inner(outputindex); - - by_type.get_mut(outputtype).value += value; - by_type.get_mut(outputtype).utxos += 1; - }); - - by_type - }); - - (sent.join().unwrap(), received.join().unwrap()) - }); - - // Compute then push current block state - let mut block_state = BlockState::default(); - received - .to_spendable_vec() - .into_iter() - .for_each(|spendable_block_state| { - block_state += spendable_block_state; - }); - block_state -= block_state_to_subtract; - self.chain_state.push(block_state); - - let (sent, realized_cap_destroyed) = height_to_sent_data - .par_iter() - .map(|(_, (_, dollars, sats, _))| { - let dollars = *dollars; - let sats = *sats; - (sats, dollars * Bitcoin::from(sats)) - }) - .reduce( - || (Sats::ZERO, Dollars::ZERO), - |acc, (sats, dollars)| (acc.0 + sats, acc.1 + dollars), - ); - - let utxos_created = *output_count - received.op_return.utxos; - - let mut received_unspendable = OutputType::as_vec() - .into_iter() - .filter(|t| t.is_unspendable()) - .map(|t| received.get_mut(t).value) - .sum::() - + height_to_unclaimed_rewards_iter.unwrap_get_inner(height); - - // Three invalid coinbases which all have 1 output - let utxos_destroyed = if height == Height::new(0) - || height == Height::new(91_842) - || height == Height::new(91_880) - { - // They're all p2pk65 - received.p2pk65.utxos -= 1; - received.p2pk65.value -= Sats::FIFTY_BTC; - received_unspendable += Sats::FIFTY_BTC; - *input_count - } else { - *input_count - 1 - }; - - // state.supply -= sent; - - // state.supply += received_spendable; - // state.unspendable_supply += received_unspendable; - - // *state.utxo_count += utxos_created; - // *state.utxo_count -= utxos_destroyed; - - // if let Some(height_to_close_iter) = height_to_close_iter.as_mut() { - // let received = received_spendable + received_unspendable; - // let price = *height_to_close_iter.unwrap_get_inner(height); - // let realized_cap_created = price * Bitcoin::from(received); - // state.realized_cap = (state.realized_cap + realized_cap_created) - // .checked_sub(realized_cap_destroyed) - // .unwrap(); - // } - - self.utxos_vecs - .mut_flatten() - .iter_mut() - .try_for_each(|v| v.forced_pushed_at(height, exit))?; - - // // self.height_to_unspendable_supply.forced_push_at( - // height, - // self.state.unspendable_supply, - // exit, - // )?; - - Ok(()) - })?; - - exit.block(); - - let mut flat_vecs_ = self.utxos_vecs.mut_flatten(); - - // Flush rest of values - flat_vecs_ - .iter_mut() - .try_for_each(|v| v.safe_flush_height_vecs(exit))?; - self.height_to_unspendable_supply.safe_flush(exit)?; - - // Compute other vecs from height vecs - flat_vecs_ - .iter_mut() - .try_for_each(|v| v.compute_rest(indexer, indexes, fetched, starting_indexes, exit))?; - self.indexes_to_unspendable_supply.compute_rest( - indexer, - indexes, - fetched, - starting_indexes, - exit, - Some(&self.height_to_unspendable_supply), - )?; - - // Save chain state - self.saved_chain_state.truncate_if_needed(Height::ZERO)?; - mem::take(&mut self.chain_state) - .into_iter() - .for_each(|block_state| { - self.saved_chain_state.push(block_state); - }); - self.saved_chain_state.flush()?; - - exit.release(); - - Ok(()) - } - - pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { - [ - self.utxos_vecs.all.vecs(), - self.indexes_to_unspendable_supply.vecs(), - vec![&self.height_to_unspendable_supply], - ] - .into_iter() - .flatten() - .collect::>() - } -} - -pub struct Vecs_ { - starting_height: Height, - state: CohortState, - - pub height_to_realized_cap: Option>, - pub indexes_to_realized_cap: Option>, - pub height_to_supply: EagerVec, - pub indexes_to_supply: ComputedValueVecsFromHeight, - pub height_to_utxo_count: EagerVec, - pub indexes_to_utxo_count: ComputedVecsFromHeight, -} - -impl Vecs_ { - pub fn forced_import( - path: &Path, - cohort_name: Option<&str>, - _computation: Computation, - compressed: Compressed, - fetched: Option<&fetched::Vecs>, - ) -> color_eyre::Result { - let compute_dollars = fetched.is_some(); - - fs::create_dir_all(path)?; - - // let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}")); - - let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); - - Ok(Self { - starting_height: Height::ZERO, - state: CohortState::default(), - - height_to_realized_cap: compute_dollars.then(|| { - EagerVec::forced_import( - path, - &suffix("realized_cap"), - VERSION + Version::ZERO, - compressed, - ) - .unwrap() - }), - indexes_to_realized_cap: compute_dollars.then(|| { - ComputedVecsFromHeight::forced_import( - path, - &suffix("realized_cap"), - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - ) - .unwrap() - }), - height_to_supply: EagerVec::forced_import( - path, - &suffix("supply"), - VERSION + Version::ZERO, - compressed, - )?, - indexes_to_supply: ComputedValueVecsFromHeight::forced_import( - path, - &suffix("supply"), - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - compute_dollars, - )?, - height_to_utxo_count: EagerVec::forced_import( - path, - &suffix("utxo_count"), - VERSION + Version::ZERO, - compressed, - )?, - indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( - path, - &suffix("utxo_count"), - false, - VERSION + Version::ZERO, - compressed, - StorableVecGeneatorOptions::default().add_last(), - )?, - }) - } - - pub fn init(&mut self, starting_indexes: &Indexes) -> Height { - self.starting_height = [ - self.height_to_supply.len(), - // self.height_to_unspendable_supply.len(), - self.height_to_utxo_count.len(), - self.height_to_realized_cap - .as_ref() - .map_or(usize::MAX, |v| v.len()), - ] - .into_iter() - .map(Height::from) - .min() - .unwrap() - .min(starting_indexes.height); - - if let Some(prev_height) = self.starting_height.checked_sub(Height::new(1)) { - self.state.supply = self - .height_to_supply - .into_iter() - .unwrap_get_inner(prev_height); - self.state.utxo_count = self - .height_to_utxo_count - .into_iter() - .unwrap_get_inner(prev_height); - if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { - self.state.realized_cap = height_to_realized_cap - .into_iter() - .unwrap_get_inner(prev_height); - } - } - - self.starting_height - } - - pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { - self.height_to_supply - .validate_computed_version_or_reset_file( - base_version + self.height_to_supply.inner_version(), - )?; - - self.height_to_utxo_count - .validate_computed_version_or_reset_file( - base_version + self.height_to_utxo_count.inner_version(), - )?; - - if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() { - height_to_realized_cap.validate_computed_version_or_reset_file( - base_version + height_to_realized_cap.inner_version(), - )?; - } - Ok(()) - } - - pub fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { - self.height_to_supply - .forced_push_at(height, self.state.supply, exit)?; - - self.height_to_utxo_count - .forced_push_at(height, self.state.utxo_count, exit)?; - - if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { - height_to_realized_cap.forced_push_at(height, self.state.realized_cap, exit)?; - } - Ok(()) - } - - pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> { - self.height_to_supply.safe_flush(exit)?; - - self.height_to_utxo_count.safe_flush(exit)?; - - if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { - height_to_realized_cap.safe_flush(exit)?; - } - - Ok(()) - } - - pub fn compute_rest( - &mut self, - indexer: &Indexer, - indexes: &indexes::Vecs, - fetched: Option<&fetched::Vecs>, - starting_indexes: &Indexes, - exit: &Exit, - ) -> color_eyre::Result<()> { - self.indexes_to_supply.compute_rest( - indexer, - indexes, - fetched, - starting_indexes, - exit, - Some(&self.height_to_supply), - )?; - - self.indexes_to_utxo_count.compute_rest( - indexes, - starting_indexes, - exit, - Some(&self.height_to_utxo_count), - )?; - - if let Some(indexes_to_realized_cap) = self.indexes_to_realized_cap.as_mut() { - indexes_to_realized_cap.compute_rest( - indexes, - starting_indexes, - exit, - Some(self.height_to_realized_cap.as_ref().unwrap()), - )?; - } - - Ok(()) - } - - pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { - [ - vec![ - &self.height_to_supply as &dyn AnyCollectableVec, - &self.height_to_utxo_count, - ], - self.height_to_realized_cap - .as_ref() - .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]), - self.indexes_to_supply.vecs(), - self.indexes_to_utxo_count.vecs(), - self.indexes_to_realized_cap - .as_ref() - .map_or(vec![], |v| v.vecs()), - ] - .into_iter() - .flatten() - .collect::>() - } -} - -impl Clone for Vecs_ { - fn clone(&self) -> Self { - Self { - starting_height: self.starting_height, - state: CohortState::default(), - - height_to_realized_cap: self.height_to_realized_cap.clone(), - indexes_to_realized_cap: self.indexes_to_realized_cap.clone(), - height_to_supply: self.height_to_supply.clone(), - indexes_to_supply: self.indexes_to_supply.clone(), - height_to_utxo_count: self.height_to_utxo_count.clone(), - indexes_to_utxo_count: self.indexes_to_utxo_count.clone(), - } - } -} diff --git a/crates/brk_computer/src/vecs/utxos/cohort.rs b/crates/brk_computer/src/vecs/utxos/cohort.rs new file mode 100644 index 000000000..cf5a7c778 --- /dev/null +++ b/crates/brk_computer/src/vecs/utxos/cohort.rs @@ -0,0 +1,266 @@ +use std::{fs, path::Path}; + +use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{ + AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, Result, VecIterator, Version, +}; + +use crate::{ + states::CohortState, + vecs::{ + Indexes, fetched, + grouped::{ + ComputedValueVecsFromHeight, ComputedVecsFromHeight, StorableVecGeneatorOptions, + }, + indexes, + }, +}; + +const VERSION: Version = Version::ZERO; + +pub struct Vecs { + starting_height: Height, + pub state: CohortState, + + pub height_to_realized_cap: Option>, + pub indexes_to_realized_cap: Option>, + pub height_to_supply: EagerVec, + pub indexes_to_supply: ComputedValueVecsFromHeight, + pub height_to_utxo_count: EagerVec, + pub indexes_to_utxo_count: ComputedVecsFromHeight, +} + +impl Vecs { + pub fn forced_import( + path: &Path, + cohort_name: Option<&str>, + _computation: Computation, + compressed: Compressed, + fetched: Option<&fetched::Vecs>, + ) -> color_eyre::Result { + let compute_dollars = fetched.is_some(); + + fs::create_dir_all(path)?; + + // let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}")); + + let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); + + Ok(Self { + starting_height: Height::ZERO, + state: CohortState::default(), + + height_to_realized_cap: compute_dollars.then(|| { + EagerVec::forced_import( + path, + &suffix("realized_cap"), + VERSION + Version::ZERO, + compressed, + ) + .unwrap() + }), + indexes_to_realized_cap: compute_dollars.then(|| { + ComputedVecsFromHeight::forced_import( + path, + &suffix("realized_cap"), + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + ) + .unwrap() + }), + height_to_supply: EagerVec::forced_import( + path, + &suffix("supply"), + VERSION + Version::ZERO, + compressed, + )?, + indexes_to_supply: ComputedValueVecsFromHeight::forced_import( + path, + &suffix("supply"), + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + compute_dollars, + )?, + height_to_utxo_count: EagerVec::forced_import( + path, + &suffix("utxo_count"), + VERSION + Version::ZERO, + compressed, + )?, + indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( + path, + &suffix("utxo_count"), + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + )?, + }) + } + + pub fn init(&mut self, starting_indexes: &Indexes) -> Height { + self.starting_height = [ + self.height_to_supply.len(), + self.height_to_utxo_count.len(), + self.height_to_realized_cap + .as_ref() + .map_or(usize::MAX, |v| v.len()), + ] + .into_iter() + .map(Height::from) + .min() + .unwrap() + .min(starting_indexes.height); + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + if let Some(prev_height) = self.starting_height.checked_sub(Height::new(1)) { + self.state.supply.value = self + .height_to_supply + .into_iter() + .unwrap_get_inner(prev_height); + self.state.supply.utxos = *self + .height_to_utxo_count + .into_iter() + .unwrap_get_inner(prev_height); + self.state.realized_cap = Some( + height_to_realized_cap + .into_iter() + .unwrap_get_inner(prev_height), + ); + } else { + self.state.realized_cap = Some(Dollars::ZERO); + } + } + + self.starting_height + } + + pub fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { + self.height_to_supply + .validate_computed_version_or_reset_file( + base_version + self.height_to_supply.inner_version(), + )?; + + self.height_to_utxo_count + .validate_computed_version_or_reset_file( + base_version + self.height_to_utxo_count.inner_version(), + )?; + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut().as_mut() { + height_to_realized_cap.validate_computed_version_or_reset_file( + base_version + height_to_realized_cap.inner_version(), + )?; + } + Ok(()) + } + + pub fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { + self.height_to_supply + .forced_push_at(height, self.state.supply.value, exit)?; + + self.height_to_utxo_count.forced_push_at( + height, + StoredUsize::from(self.state.supply.utxos), + exit, + )?; + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + height_to_realized_cap.forced_push_at( + height, + self.state.realized_cap.unwrap(), + exit, + )?; + } + Ok(()) + } + + pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> { + self.height_to_supply.safe_flush(exit)?; + + self.height_to_utxo_count.safe_flush(exit)?; + + if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() { + height_to_realized_cap.safe_flush(exit)?; + } + + Ok(()) + } + + pub fn compute_rest( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + self.indexes_to_supply.compute_rest( + indexer, + indexes, + fetched, + starting_indexes, + exit, + Some(&self.height_to_supply), + )?; + + self.indexes_to_utxo_count.compute_rest( + indexes, + starting_indexes, + exit, + Some(&self.height_to_utxo_count), + )?; + + if let Some(indexes_to_realized_cap) = self.indexes_to_realized_cap.as_mut() { + indexes_to_realized_cap.compute_rest( + indexes, + starting_indexes, + exit, + Some(self.height_to_realized_cap.as_ref().unwrap()), + )?; + } + + Ok(()) + } + + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + [ + vec![ + &self.height_to_supply as &dyn AnyCollectableVec, + &self.height_to_utxo_count, + ], + self.height_to_realized_cap + .as_ref() + .map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]), + self.indexes_to_supply.vecs(), + self.indexes_to_utxo_count.vecs(), + self.indexes_to_realized_cap + .as_ref() + .map_or(vec![], |v| v.vecs()), + ] + .into_iter() + .flatten() + .collect::>() + } +} + +impl Clone for Vecs { + fn clone(&self) -> Self { + Self { + starting_height: self.starting_height, + state: CohortState::default(), + + height_to_realized_cap: self.height_to_realized_cap.clone(), + indexes_to_realized_cap: self.indexes_to_realized_cap.clone(), + height_to_supply: self.height_to_supply.clone(), + indexes_to_supply: self.indexes_to_supply.clone(), + height_to_utxo_count: self.height_to_utxo_count.clone(), + indexes_to_utxo_count: self.indexes_to_utxo_count.clone(), + } + } +} diff --git a/crates/brk_computer/src/vecs/utxos/mod.rs b/crates/brk_computer/src/vecs/utxos/mod.rs new file mode 100644 index 000000000..5260b244c --- /dev/null +++ b/crates/brk_computer/src/vecs/utxos/mod.rs @@ -0,0 +1,1149 @@ +use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread}; + +use brk_core::{Height, InputIndex, OutputIndex, Sats}; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_vec::{ + AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Compressed, Computation, EagerVec, + GenericStoredVec, StoredIndex, StoredVec, VecIterator, Version, +}; +use log::info; + +use crate::states::{ + BlockState, OutputFilter, Outputs, OutputsByEpoch, OutputsByFrom, OutputsByRange, + OutputsBySize, OutputsBySpendableType, OutputsByTerm, OutputsByType, OutputsByUpTo, + ReceivedBlockStateData, SupplyState, +}; + +use super::{ + Indexes, fetched, + grouped::{ComputedValueVecsFromHeight, StorableVecGeneatorOptions}, + indexes, transactions, +}; + +pub mod cohort; + +const VERSION: Version = Version::new(50); + +#[derive(Clone)] +pub struct Vecs { + chain_state: StoredVec, + + // cointime,... + pub height_to_unspendable_supply: EagerVec, + pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, + utxos_vecs: Outputs<(OutputFilter, cohort::Vecs)>, +} + +impl Vecs { + pub fn forced_import( + path: &Path, + _computation: Computation, + compressed: Compressed, + fetched: Option<&fetched::Vecs>, + ) -> color_eyre::Result { + let compute_dollars = fetched.is_some(); + + let mut states_path = path.to_owned(); + states_path.pop(); + states_path = states_path.join("states"); + + Ok(Self { + chain_state: StoredVec::forced_import( + &states_path, + "chain", + Version::ZERO, + Compressed::NO, + )?, + + height_to_unspendable_supply: EagerVec::forced_import( + path, + "unspendable_supply", + VERSION + Version::ZERO, + compressed, + )?, + indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( + path, + "unspendable_supply", + false, + VERSION + Version::ZERO, + compressed, + StorableVecGeneatorOptions::default().add_last(), + compute_dollars, + )?, + utxos_vecs: { + Outputs::<(OutputFilter, cohort::Vecs)>::from(Outputs { + all: cohort::Vecs::forced_import( + path, + None, + _computation, + compressed, + fetched, + )?, + by_term: OutputsByTerm { + short: cohort::Vecs::forced_import( + path, + Some("sth"), + _computation, + compressed, + fetched, + )?, + long: cohort::Vecs::forced_import( + path, + Some("lth"), + _computation, + compressed, + fetched, + )?, + }, + // by_up_to: OutputsByUpTo { + // _1d: cohort::Vecs::forced_import( + // path, + // Some("up_to_1d"), + // _computation, + // compressed, + // fetched, + // )?, + // _1w: cohort::Vecs::forced_import( + // path, + // Some("up_to_1w"), + // _computation, + // compressed, + // fetched, + // )?, + // _1m: cohort::Vecs::forced_import( + // path, + // Some("up_to_1m"), + // _computation, + // compressed, + // fetched, + // )?, + // _2m: cohort::Vecs::forced_import( + // path, + // Some("up_to_2m"), + // _computation, + // compressed, + // fetched, + // )?, + // _3m: cohort::Vecs::forced_import( + // path, + // Some("up_to_3m"), + // _computation, + // compressed, + // fetched, + // )?, + // _4m: cohort::Vecs::forced_import( + // path, + // Some("up_to_4m"), + // _computation, + // compressed, + // fetched, + // )?, + // _5m: cohort::Vecs::forced_import( + // path, + // Some("up_to_5m"), + // _computation, + // compressed, + // fetched, + // )?, + // _6m: cohort::Vecs::forced_import( + // path, + // Some("up_to_6m"), + // _computation, + // compressed, + // fetched, + // )?, + // _1y: cohort::Vecs::forced_import( + // path, + // Some("up_to_1y"), + // _computation, + // compressed, + // fetched, + // )?, + // _2y: cohort::Vecs::forced_import( + // path, + // Some("up_to_2y"), + // _computation, + // compressed, + // fetched, + // )?, + // _3y: cohort::Vecs::forced_import( + // path, + // Some("up_to_3y"), + // _computation, + // compressed, + // fetched, + // )?, + // _4y: cohort::Vecs::forced_import( + // path, + // Some("up_to_4y"), + // _computation, + // compressed, + // fetched, + // )?, + // _5y: cohort::Vecs::forced_import( + // path, + // Some("up_to_5y"), + // _computation, + // compressed, + // fetched, + // )?, + // _6y: cohort::Vecs::forced_import( + // path, + // Some("up_to_6y"), + // _computation, + // compressed, + // fetched, + // )?, + // _7y: cohort::Vecs::forced_import( + // path, + // Some("up_to_7y"), + // _computation, + // compressed, + // fetched, + // )?, + // _8y: cohort::Vecs::forced_import( + // path, + // Some("up_to_8y"), + // _computation, + // compressed, + // fetched, + // )?, + // _10y: cohort::Vecs::forced_import( + // path, + // Some("up_to_10y"), + // _computation, + // compressed, + // fetched, + // )?, + // _15y: cohort::Vecs::forced_import( + // path, + // Some("up_to_15y"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_from: OutputsByFrom { + // _1d: cohort::Vecs::forced_import( + // path, + // Some("from_1d"), + // _computation, + // compressed, + // fetched, + // )?, + // _1w: cohort::Vecs::forced_import( + // path, + // Some("from_1w"), + // _computation, + // compressed, + // fetched, + // )?, + // _1m: cohort::Vecs::forced_import( + // path, + // Some("from_1m"), + // _computation, + // compressed, + // fetched, + // )?, + // _2m: cohort::Vecs::forced_import( + // path, + // Some("from_2m"), + // _computation, + // compressed, + // fetched, + // )?, + // _3m: cohort::Vecs::forced_import( + // path, + // Some("from_3m"), + // _computation, + // compressed, + // fetched, + // )?, + // _4m: cohort::Vecs::forced_import( + // path, + // Some("from_4m"), + // _computation, + // compressed, + // fetched, + // )?, + // _5m: cohort::Vecs::forced_import( + // path, + // Some("from_5m"), + // _computation, + // compressed, + // fetched, + // )?, + // _6m: cohort::Vecs::forced_import( + // path, + // Some("from_6m"), + // _computation, + // compressed, + // fetched, + // )?, + // _1y: cohort::Vecs::forced_import( + // path, + // Some("from_1y"), + // _computation, + // compressed, + // fetched, + // )?, + // _2y: cohort::Vecs::forced_import( + // path, + // Some("from_2y"), + // _computation, + // compressed, + // fetched, + // )?, + // _3y: cohort::Vecs::forced_import( + // path, + // Some("from_3y"), + // _computation, + // compressed, + // fetched, + // )?, + // _4y: cohort::Vecs::forced_import( + // path, + // Some("from_4y"), + // _computation, + // compressed, + // fetched, + // )?, + // _5y: cohort::Vecs::forced_import( + // path, + // Some("from_5y"), + // _computation, + // compressed, + // fetched, + // )?, + // _6y: cohort::Vecs::forced_import( + // path, + // Some("from_6y"), + // _computation, + // compressed, + // fetched, + // )?, + // _7y: cohort::Vecs::forced_import( + // path, + // Some("from_7y"), + // _computation, + // compressed, + // fetched, + // )?, + // _8y: cohort::Vecs::forced_import( + // path, + // Some("from_8y"), + // _computation, + // compressed, + // fetched, + // )?, + // _10y: cohort::Vecs::forced_import( + // path, + // Some("from_10y"), + // _computation, + // compressed, + // fetched, + // )?, + // _15y: cohort::Vecs::forced_import( + // path, + // Some("from_15y"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_range: OutputsByRange { + // _1d_to_1w: cohort::Vecs::forced_import( + // path, + // Some("from_1d_to_1w"), + // _computation, + // compressed, + // fetched, + // )?, + // _1w_to_1m: cohort::Vecs::forced_import( + // path, + // Some("from_1w_to_1m"), + // _computation, + // compressed, + // fetched, + // )?, + // _1m_to_3m: cohort::Vecs::forced_import( + // path, + // Some("from_1m_to_3m"), + // _computation, + // compressed, + // fetched, + // )?, + // _3m_to_6m: cohort::Vecs::forced_import( + // path, + // Some("from_3m_to_6m"), + // _computation, + // compressed, + // fetched, + // )?, + // _6m_to_1y: cohort::Vecs::forced_import( + // path, + // Some("from_6m_to_1y"), + // _computation, + // compressed, + // fetched, + // )?, + // _1y_to_2y: cohort::Vecs::forced_import( + // path, + // Some("from_1y_to_2y"), + // _computation, + // compressed, + // fetched, + // )?, + // _2y_to_3y: cohort::Vecs::forced_import( + // path, + // Some("from_2y_to_3y"), + // _computation, + // compressed, + // fetched, + // )?, + // _3y_to_4y: cohort::Vecs::forced_import( + // path, + // Some("from_3y_to_4y"), + // _computation, + // compressed, + // fetched, + // )?, + // _4y_to_5y: cohort::Vecs::forced_import( + // path, + // Some("from_4y_to_5y"), + // _computation, + // compressed, + // fetched, + // )?, + // _5y_to_7y: cohort::Vecs::forced_import( + // path, + // Some("from_5y_to_7y"), + // _computation, + // compressed, + // fetched, + // )?, + // _7y_to_10y: cohort::Vecs::forced_import( + // path, + // Some("from_7y_to_10y"), + // _computation, + // compressed, + // fetched, + // )?, + // _10y_to_15y: cohort::Vecs::forced_import( + // path, + // Some("from_10y_to_15y"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_epoch: OutputsByEpoch { + // _0: cohort::Vecs::forced_import( + // path, + // Some("epoch_0"), + // _computation, + // compressed, + // fetched, + // )?, + // _1: cohort::Vecs::forced_import( + // path, + // Some("epoch_1"), + // _computation, + // compressed, + // fetched, + // )?, + // _2: cohort::Vecs::forced_import( + // path, + // Some("epoch_2"), + // _computation, + // compressed, + // fetched, + // )?, + // _3: cohort::Vecs::forced_import( + // path, + // Some("epoch_3"), + // _computation, + // compressed, + // fetched, + // )?, + // _4: cohort::Vecs::forced_import( + // path, + // Some("epoch_4"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_size: OutputsBySize { + // from_1sat_to_10sats: cohort::Vecs::forced_import( + // path, + // Some("from_1sat_to_10sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10sats_to_100sats: cohort::Vecs::forced_import( + // path, + // Some("from_10sats_to_100sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100sats_to_1_000sats: cohort::Vecs::forced_import( + // path, + // Some("from_100sats_to_1_000sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000sats_to_10_000sats: cohort::Vecs::forced_import( + // path, + // Some("from_1_000sats_to_10_000sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10_000sats_to_100_000sats: cohort::Vecs::forced_import( + // path, + // Some("from_10_000sats_to_100_000sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100_000sats_to_1_000_000sats: cohort::Vecs::forced_import( + // path, + // Some("from_100_000sats_to_1_000_000sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000_000sats_to_10_000_000sats: cohort::Vecs::forced_import( + // path, + // Some("from_1_000_000sats_to_10_000_000sats"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10_000_000sats_to_1btc: cohort::Vecs::forced_import( + // path, + // Some("from_10_000_000sats_to_1btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1btc_to_10btc: cohort::Vecs::forced_import( + // path, + // Some("from_1btc_to_10btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10btc_to_100btc: cohort::Vecs::forced_import( + // path, + // Some("from_10btc_to_100btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100btc_to_1_000btc: cohort::Vecs::forced_import( + // path, + // Some("from_100btc_to_1_000btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000btc_to_10_000btc: cohort::Vecs::forced_import( + // path, + // Some("from_1_000btc_to_10_000btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10_000btc_to_100_000btc: cohort::Vecs::forced_import( + // path, + // Some("from_10_000btc_to_100_000btc"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100_000btc: cohort::Vecs::forced_import( + // path, + // Some("from_100_000btc"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_value: OutputsByValue { + // up_to_1cent: cohort::Vecs::forced_import( + // path, + // Some("up_to_1cent"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1c_to_10c: cohort::Vecs::forced_import( + // path, + // Some("from_1c_to_10c"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10c_to_1d: cohort::Vecs::forced_import( + // path, + // Some("from_10c_to_1d"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1d_to_10d: cohort::Vecs::forced_import( + // path, + // Some("from_1d_to_10d"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10usd_to_100usd: cohort::Vecs::forced_import( + // path, + // Some("from_10usd_to_100usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100usd_to_1_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_100usd_to_1_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000usd_to_10_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_1_000usd_to_10_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10_000usd_to_100_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_10_000usd_to_100_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100_000usd_to_1_000_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_100_000usd_to_1_000_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000_000usd_to_10_000_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_1_000_000usd_to_10_000_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_10_000_000usd_to_100_000_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_10_000_000usd_to_100_000_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_100_000_000usd_to_1_000_000_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_100_000_000usd_to_1_000_000_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // from_1_000_000_000usd: cohort::Vecs::forced_import( + // path, + // Some("from_1_000_000_000usd"), + // _computation, + // compressed, + // fetched, + // )?, + // }, + // by_spendable_type: OutputsBySpendableType { + // p2pk65: cohort::Vecs::forced_import( + // path, + // Some("p2pk65"), + // _computation, + // compressed, + // fetched, + // )?, + // p2pk33: cohort::Vecs::forced_import( + // path, + // Some("p2pk33"), + // _computation, + // compressed, + // fetched, + // )?, + // p2pkh: cohort::Vecs::forced_import( + // path, + // Some("p2pkh"), + // _computation, + // compressed, + // fetched, + // )?, + // p2ms: cohort::Vecs::forced_import( + // path, + // Some("p2ms"), + // _computation, + // compressed, + // fetched, + // )?, + // p2sh: cohort::Vecs::forced_import( + // path, + // Some("p2sh"), + // _computation, + // compressed, + // fetched, + // )?, + // // op_return: cohort::Vecs::forced_import( + // // path, + // // Some("op_return"), + // // _computation, + // // compressed, + // // fetched, + // // )?, + // p2wpkh: cohort::Vecs::forced_import( + // path, + // Some("p2wpkh"), + // _computation, + // compressed, + // fetched, + // )?, + // p2wsh: cohort::Vecs::forced_import( + // path, + // Some("p2wsh"), + // _computation, + // compressed, + // fetched, + // )?, + // p2tr: cohort::Vecs::forced_import( + // path, + // Some("p2tr"), + // _computation, + // compressed, + // fetched, + // )?, + // p2a: cohort::Vecs::forced_import( + // path, + // Some("p2a"), + // _computation, + // compressed, + // fetched, + // )?, + // // empty: cohort::Vecs::forced_import( + // // path, + // // Some("empty"), + // // _computation, + // // compressed, + // // fetched, + // // )?, + // // unknown: cohort::Vecs::forced_import( + // // path, + // // Some("unknown"), + // // _computation, + // // compressed, + // // fetched, + // // )?, + // }, + }) + }, + }) + } + + pub fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + transactions: &transactions::Vecs, + fetched: Option<&fetched::Vecs>, + starting_indexes: &Indexes, + exit: &Exit, + ) -> color_eyre::Result<()> { + let indexer_vecs = indexer.vecs(); + + let height_to_first_outputindex = &indexer_vecs.height_to_first_outputindex; + let height_to_first_inputindex = &indexer_vecs.height_to_first_inputindex; + let height_to_output_count = transactions.indexes_to_output_count.height.unwrap_sum(); + let height_to_input_count = transactions.indexes_to_input_count.height.unwrap_sum(); + let inputindex_to_outputindex = &indexer_vecs.inputindex_to_outputindex; + let outputindex_to_value = &indexer_vecs.outputindex_to_value; + let txindex_to_height = &indexes.txindex_to_height; + let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed; + let outputindex_to_txindex = &indexes.outputindex_to_txindex; + let outputindex_to_outputtype = &indexer_vecs.outputindex_to_outputtype; + let height_to_unclaimed_rewards = transactions + .indexes_to_unclaimed_rewards + .sats + .height + .as_ref() + .unwrap() + .as_ref(); + let height_to_close = &fetched + .as_ref() + .map(|fetched| &fetched.chainindexes_to_close.height); + + let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter(); + let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter(); + let mut height_to_output_count_iter = height_to_output_count.into_iter(); + let mut height_to_input_count_iter = height_to_input_count.into_iter(); + let mut inputindex_to_outputindex_iter = inputindex_to_outputindex.into_iter(); + let mut outputindex_to_value_iter = outputindex_to_value.into_iter(); + let mut outputindex_to_value_iter_2 = outputindex_to_value.into_iter(); + let mut txindex_to_height_iter = txindex_to_height.into_iter(); + let mut outputindex_to_txindex_iter = outputindex_to_txindex.into_iter(); + let mut height_to_close_iter = height_to_close.as_ref().map(|v| v.into_iter()); + let mut outputindex_to_outputtype_iter = outputindex_to_outputtype.into_iter(); + let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter(); + let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter(); + let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter(); + + let mut flat_vecs_ = self.utxos_vecs.as_mut_vec(); + + let base_version = Version::ZERO + + height_to_first_outputindex.version() + + height_to_first_inputindex.version() + + height_to_timestamp_fixed.version() + + height_to_output_count.version() + + height_to_input_count.version() + + inputindex_to_outputindex.version() + + outputindex_to_value.version() + + txindex_to_height.version() + + outputindex_to_txindex.version() + + outputindex_to_outputtype.version() + + height_to_unclaimed_rewards.version() + + height_to_close + .as_ref() + .map_or(Version::ZERO, |v| v.version()); + + flat_vecs_ + .iter_mut() + .try_for_each(|(_, v)| v.validate_computed_versions(base_version))?; + self.height_to_unspendable_supply + .validate_computed_version_or_reset_file( + base_version + self.height_to_unspendable_supply.inner_version(), + )?; + + let mut chain_state: Vec; + let mut chain_state_starting_height = Height::from(self.chain_state.len()); + + let stateful_starting_height = match flat_vecs_ + .iter_mut() + .map(|(_, v)| v.init(starting_indexes)) + .min() + .unwrap_or_default() + .min(chain_state_starting_height) + .cmp(&chain_state_starting_height) + { + Ordering::Greater => unreachable!(), + Ordering::Equal => { + chain_state = self + .chain_state + .collect_range(None, None)? + .into_iter() + .enumerate() + .map(|(height, supply)| { + let height = Height::from(height); + let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height); + let price = height_to_close_iter + .as_mut() + .map(|i| *i.unwrap_get_inner(height)); + BlockState { + timestamp, + price, + supply, + } + }) + .collect::>(); + chain_state_starting_height + } + Ordering::Less => { + // todo!("rollback instead"); + chain_state = vec![]; + chain_state_starting_height = Height::ZERO; + Height::ZERO + } + }; + if stateful_starting_height.is_zero() { + info!("Starting processing utxos from the start"); + } + let starting_height = + stateful_starting_height.min(Height::from(self.height_to_unspendable_supply.len())); + + let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() { + self.height_to_unspendable_supply + .into_iter() + .unwrap_get_inner(prev_height) + } else { + Sats::ZERO + }; + + let mut height = Height::ZERO; + (starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len()) + .map(Height::from) + .try_for_each(|_height| -> color_eyre::Result<()> { + height = _height; + + info!("Processing utxo set at {height}..."); + + let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height); + let price = height_to_close_iter + .as_mut() + .map(|i| *i.unwrap_get_inner(height)); + let first_outputindex = height_to_first_outputindex_iter + .unwrap_get_inner(height) + .unwrap_to_usize(); + let first_inputindex = height_to_first_inputindex_iter + .unwrap_get_inner(height) + .unwrap_to_usize(); + let output_count = height_to_output_count_iter.unwrap_get_inner(height); + let input_count = height_to_input_count_iter.unwrap_get_inner(height); + + // let sent_state = SentState::default(); + // let received_state = ReceivedState::default(); + // let realized_state = RealizedState::default(); + + let (mut height_to_sent, mut received) = thread::scope(|s| { + if chain_state_starting_height <= height { + s.spawn(|| { + self.utxos_vecs + .tick_tock_next_block(&chain_state, timestamp); + }); + } + + let sent_handle = s.spawn(|| { + let mut txindex_to_height = BTreeMap::new(); + let mut height_to_sent: BTreeMap< + Height, + OutputsByType<(SupplyState, Vec)>, + > = BTreeMap::new(); + + // Skip coinbase + (first_inputindex + 1..first_inputindex + *input_count) + .map(InputIndex::from) + .for_each(|inputindex| { + let outputindex = + inputindex_to_outputindex_iter.unwrap_get_inner(inputindex); + + let value = outputindex_to_value_iter.unwrap_get_inner(outputindex); + + let input_type = + outputindex_to_outputtype_iter.unwrap_get_inner(outputindex); + + let input_txindex = + outputindex_to_txindex_iter.unwrap_get_inner(outputindex); + + let input_height = + *txindex_to_height.entry(input_txindex).or_insert_with(|| { + txindex_to_height_iter.unwrap_get_inner(input_txindex) + }); + + let input_data = height_to_sent.entry(input_height).or_default(); + + let (sent_supply, sats_vec) = input_data.get_mut(input_type); + + sent_supply.utxos += 1; + sent_supply.value += value; + sats_vec.push(value); + }); + + height_to_sent + }); + + let received_handle = s.spawn(|| { + let mut by_type: OutputsByType<(SupplyState, Vec)> = + OutputsByType::default(); + + (first_outputindex..first_outputindex + *output_count) + .map(OutputIndex::from) + .for_each(|outputindex| { + let value = + outputindex_to_value_iter_2.unwrap_get_inner(outputindex); + + let output_type = + outputindex_to_outputtype_iter_2.unwrap_get_inner(outputindex); + + let (sent_supply, sats_vec) = by_type.get_mut(output_type); + + sent_supply.value += value; + sent_supply.utxos += 1; + sats_vec.push(value); + }); + + by_type + }); + + (sent_handle.join().unwrap(), received_handle.join().unwrap()) + }); + + unspendable_supply += received + .unspendable + .as_vec() + .into_iter() + .map(|state| state.0.value) + .sum::() + + height_to_unclaimed_rewards_iter.unwrap_get_inner(height); + + if height == Height::new(0) { + received.spendable.p2pk65.1.remove(0); + received.spendable.p2pk65.0.utxos -= 1; + received.spendable.p2pk65.0.value -= Sats::FIFTY_BTC; + unspendable_supply += Sats::FIFTY_BTC; + } else { + // Need to destroy invalid coinbases due to duplicate txids + if height == Height::new(91_842) || height == Height::new(91_880) { + let entry = if height == Height::new(91_842) { + height_to_sent.entry(Height::new(91_812)).or_default() + } else { + height_to_sent.entry(Height::new(91_722)).or_default() + }; + entry.spendable.p2pk65.0.value += Sats::FIFTY_BTC; + entry.spendable.p2pk65.0.utxos += 1; + entry.spendable.p2pk65.1.push(Sats::FIFTY_BTC); + } + }; + + if chain_state_starting_height <= height { + // RECEIVE + + // Push current block state before processing sends and receives + chain_state.push(BlockState::from(ReceivedBlockStateData { + received: &received, + price, + timestamp, + })); + + self.utxos_vecs.receive(received, height, price); + + // --- + + // SEND + + // Apply sent to + height_to_sent + .iter() + .for_each(|(height, sent_data_by_type)| { + let block_state = + chain_state.get_mut(height.unwrap_to_usize()).unwrap(); + sent_data_by_type + .as_vec() + .into_iter() + .for_each(|(supply, _)| { + block_state.supply -= supply.clone(); + }); + }); + + self.utxos_vecs.send(height_to_sent, chain_state.as_slice()); + } else { + panic!("temp, just making sure") + } + + // received. + // 2. push received + // 3. subtract sent + + // 4. check what's the point with wth is after this message + + // let (sent, realized_cap_destroyed) = height_to_sent + // .par_iter() + // .map(|(_, (_, dollars, sats, _))| { + // let dollars = *dollars; + // let sats = *sats; + // (sats, dollars * Bitcoin::from(sats)) + // }) + // .reduce( + // || (Sats::ZERO, Dollars::ZERO), + // |acc, (sats, dollars)| (acc.0 + sats, acc.1 + dollars), + // ); + + // state.supply -= sent; + + // state.supply += received_spendable; + // state.unspendable_supply += received_unspendable; + + // *state.utxo_count += utxos_created; + // *state.utxo_count -= utxos_destroyed; + + // if let Some(height_to_close_iter) = height_to_close_iter.as_mut() { + // let received = received_spendable + received_unspendable; + // let price = *height_to_close_iter.unwrap_get_inner(height); + // let realized_cap_created = price * Bitcoin::from(received); + // state.realized_cap = (state.realized_cap + realized_cap_created) + // .checked_sub(realized_cap_destroyed) + // .unwrap(); + // } + + self.utxos_vecs + .as_mut_vec() + .iter_mut() + .try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?; + + self.height_to_unspendable_supply.forced_push_at( + height, + unspendable_supply, + exit, + )?; + + Ok(()) + })?; + + exit.block(); + + let mut flat_vecs_ = self.utxos_vecs.as_mut_vec(); + + // Flush rest of values + flat_vecs_ + .iter_mut() + .try_for_each(|(_, v)| v.safe_flush_height_vecs(exit))?; + self.height_to_unspendable_supply.safe_flush(exit)?; + + // Compute other vecs from height vecs + flat_vecs_.iter_mut().try_for_each(|(_, v)| { + v.compute_rest(indexer, indexes, fetched, starting_indexes, exit) + })?; + self.indexes_to_unspendable_supply.compute_rest( + indexer, + indexes, + fetched, + starting_indexes, + exit, + Some(&self.height_to_unspendable_supply), + )?; + + // Save chain state + self.chain_state.truncate_if_needed(Height::ZERO)?; + mem::take(&mut chain_state) + .into_iter() + .for_each(|block_state| { + self.chain_state.push(block_state.supply); + }); + self.chain_state.flush()?; + + exit.release(); + + Ok(()) + } + + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { + [ + self.utxos_vecs + .vecs() + .into_iter() + .flat_map(|v| v.vecs()) + .collect::>(), + self.indexes_to_unspendable_supply.vecs(), + vec![&self.height_to_unspendable_supply], + ] + .into_iter() + .flatten() + .collect::>() + } +} diff --git a/crates/brk_core/src/structs/halvingepoch.rs b/crates/brk_core/src/structs/halvingepoch.rs index dfe31e4d4..f9097684d 100644 --- a/crates/brk_core/src/structs/halvingepoch.rs +++ b/crates/brk_core/src/structs/halvingepoch.rs @@ -28,6 +28,12 @@ use super::Height; )] pub struct HalvingEpoch(u8); +impl HalvingEpoch { + pub fn new(value: u8) -> Self { + Self(value) + } +} + impl From for HalvingEpoch { fn from(value: u8) -> Self { Self(value) diff --git a/crates/brk_core/src/structs/sats.rs b/crates/brk_core/src/structs/sats.rs index e0b905230..32a54d561 100644 --- a/crates/brk_core/src/structs/sats.rs +++ b/crates/brk_core/src/structs/sats.rs @@ -35,6 +35,10 @@ impl Sats { pub const ONE_BTC: Self = Self(1_00_000_000); pub const FIFTY_BTC: Self = Self(50_00_000_000); + pub fn new(sats: u64) -> Self { + Self(sats) + } + pub fn is_zero(&self) -> bool { *self == Self::ZERO } diff --git a/crates/brk_core/src/structs/timestamp.rs b/crates/brk_core/src/structs/timestamp.rs index e3fc3d26e..5d1bc400a 100644 --- a/crates/brk_core/src/structs/timestamp.rs +++ b/crates/brk_core/src/structs/timestamp.rs @@ -39,19 +39,24 @@ impl Timestamp { } pub fn floor_seconds(self) -> Self { - let t = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC); - let d = jiff::civil::DateTime::from(t); - let d = date(d.year(), d.month(), d.day()).at(d.hour(), d.minute(), 0, 0); - Self::from(d.to_zoned(TimeZone::UTC).unwrap().timestamp()) + let zoned = jiff::Timestamp::from(self).to_zoned(TimeZone::UTC); + let date_time = jiff::civil::DateTime::from(zoned); + let trunc_date_time = date(date_time.year(), date_time.month(), date_time.day()).at( + date_time.hour(), + date_time.minute(), + 0, + 0, + ); + Self::from(trunc_date_time.to_zoned(TimeZone::UTC).unwrap().timestamp()) } - pub fn difference_in_days_between(earlier: Self, later: Self) -> usize { - match later.cmp(&earlier) { - Ordering::Less => panic!("Shouldn't be used with inverted"), + pub fn difference_in_days_between(&self, other: Self) -> usize { + match self.cmp(&other) { Ordering::Equal => 0, - Ordering::Greater => { - (jiff::Timestamp::from(earlier) - .duration_until(jiff::Timestamp::from(later)) + Ordering::Greater => other.difference_in_days_between(*self), + Ordering::Less => { + (jiff::Timestamp::from(*self) + .duration_until(jiff::Timestamp::from(other)) .as_secs() / ONE_DAY_IN_SEC) as usize } diff --git a/crates/brk_fetcher/src/fetchers/binance.rs b/crates/brk_fetcher/src/fetchers/binance.rs index b8ef27c55..37dbf7078 100644 --- a/crates/brk_fetcher/src/fetchers/binance.rs +++ b/crates/brk_fetcher/src/fetchers/binance.rs @@ -238,4 +238,9 @@ impl Binance { fn url(query: &str) -> String { format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}") } + + pub fn clear(&mut self) { + self._1d.take(); + self._1mn.take(); + } } diff --git a/crates/brk_fetcher/src/fetchers/kibo.rs b/crates/brk_fetcher/src/fetchers/kibo.rs index 7dd6cb622..aa3eaafc3 100644 --- a/crates/brk_fetcher/src/fetchers/kibo.rs +++ b/crates/brk_fetcher/src/fetchers/kibo.rs @@ -147,4 +147,9 @@ impl Kibo { Close::new(get_value("close")?), ))) } + + pub fn clear(&mut self) { + self.height_to_ohlc_vec.clear(); + self.year_to_date_to_ohlc.clear(); + } } diff --git a/crates/brk_fetcher/src/fetchers/kraken.rs b/crates/brk_fetcher/src/fetchers/kraken.rs index 1efdfaa58..8cd5c4960 100644 --- a/crates/brk_fetcher/src/fetchers/kraken.rs +++ b/crates/brk_fetcher/src/fetchers/kraken.rs @@ -129,4 +129,9 @@ impl Kraken { fn url(interval: usize) -> String { format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}") } + + pub fn clear(&mut self) { + self._1d.take(); + self._1mn.take(); + } } diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index bee431ca8..42cae2e0d 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -36,7 +36,7 @@ impl Fetcher { self.kraken .get_from_1d(&date) .or_else(|e| { - eprintln!("{e}"); + // eprintln!("{e}"); self.binance.get_from_1d(&date) }) .or_else(|e| { @@ -82,6 +82,8 @@ impl Fetcher { sleep(Duration::from_secs(30)); if tries < 8 * 60 * 2 { + self.clear(); + return self .get_height_(height, timestamp, previous_timestamp, tries + 1) .unwrap(); @@ -156,4 +158,10 @@ How to fix this: Ok(final_ohlc) } + + pub fn clear(&mut self) { + self.binance.clear(); + self.kibo.clear(); + self.kraken.clear(); + } } diff --git a/crates/brk_query/src/vec_trees.rs b/crates/brk_query/src/vec_trees.rs index c4d90099a..f2ae01057 100644 --- a/crates/brk_query/src/vec_trees.rs +++ b/crates/brk_query/src/vec_trees.rs @@ -16,7 +16,11 @@ impl<'a> VecTrees<'a> { pub fn insert(&mut self, vec: &'a dyn AnyCollectableVec) { let name = vec.name(); let split = name.split("_to_").collect::>(); - if split.len() != 2 { + if split.len() != 2 + && !(split.len() == 3 + && (split.get(1) == Some(&"up") + || split.get(1).is_some_and(|s| s.starts_with("from")))) + { dbg!(&name, &split); panic!(); } @@ -35,7 +39,7 @@ impl<'a> VecTrees<'a> { dbg!(&name, split[0], index.to_string()); panic!(); } - let key = split[1].to_string().replace("_", "-"); + let key = split[1..].join("_").to_string().replace("_", "-"); let prev = self .id_to_index_to_vec .entry(key.clone()) diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 484bc8b27..44889c400 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -23,7 +23,7 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.70.0", features = ["codegen", "minifier"] } +oxc = { version = "0.71.0", features = ["codegen", "minifier"] } serde = { workspace = true } tokio = { version = "1.45.0", features = ["full"] } tower-http = { version = "0.6.4", features = ["compression-full", "trace"] } diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index bf28f1b00..fcc497cb8 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -735,6 +735,11 @@ export function createVecIdToIndexes() { "low": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "low-in-cents": [DateIndex, Height], "low-in-sats": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "lth-realized-cap": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "lth-supply": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "lth-supply-in-btc": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "lth-supply-in-usd": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "lth-utxo-count": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "marketcap": [DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "max-days-between-ath": [DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "max-years-between-ath": [DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex], @@ -893,6 +898,12 @@ export function createVecIdToIndexes() { "price-8y-ago": [DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "quarterindex": [MonthIndex, QuarterIndex], "rawlocktime": [TxIndex], + "realized-cap": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "sth-realized-cap": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "sth-supply": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "sth-supply-in-btc": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "sth-supply-in-usd": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "sth-utxo-count": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "subsidy": [Height], "subsidy-10p": [DateIndex], "subsidy-25p": [DateIndex], @@ -923,6 +934,9 @@ export function createVecIdToIndexes() { "subsidy-median": [DateIndex], "subsidy-min": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "subsidy-sum": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "supply": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "supply-in-btc": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "supply-in-usd": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "timestamp": [DateIndex, DecadeIndex, DifficultyEpoch, HalvingEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "timestamp-fixed": [Height], "total-block-count": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], @@ -1013,6 +1027,10 @@ export function createVecIdToIndexes() { "unknownoutput-count-min": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "unknownoutput-count-sum": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "unknownoutputindex": [UnknownOutputIndex], + "unspendable-supply": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "unspendable-supply-in-btc": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "unspendable-supply-in-usd": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "utxo-count": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "value": [InputIndex, OutputIndex], "vbytes": [Height], "vsize": [TxIndex],