diff --git a/CHANGELOG.md b/CHANGELOG.md index df3dda348..3bc96f93d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Added `Realized Profit To Loss Ratio` to all entities - Added `Hash Price Min` - Added `Hash Price Rebound` +- Removed all year datasets (25) in favor for epoch datasets (5), the former was too granular to be really useful - Removed datasets split by liquidity for all datasets already split by any address kind, while fun to have, they took time to compute, ram, and space to store and no one was actually checking them - Fixed a lot of values in split by liquidity datasets @@ -43,7 +44,7 @@ - Added handling of SIGINT and SIGTERM terminal signals which menas you can now safely CTRL+C or kill the parser while it's exporting - Added config print at the start of the program - Compressed `empty_address_data` struct to save space (should shave of between up to 50% of the `address_index_to_empty_address_data` database) -- ~Doubled the number of `txid_to_tx_data` databases from 4096 to 8192~ If you ran with this you need to delete that database +- Doubled the number of `txid_to_tx_data` databases from 4096 to 8192 - Added `--recompute_computed true` argument, to allow recomputation of computed datasets in case of a bug - Fixed not saved arguments, not being processed properly - Fixed bug in `generic_map.multi_insert_simple_average` diff --git a/README.md b/README.md index 120a6204d..954921ceb 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Please open an issue if you want to add another instance ## Structure - `parser`: The backbone of the project, it does most of the work by parsing and then computing datasets from the timechain + - Takes 2 to 4 days to parse the whole chain and create all datasets + - If up to date wil take 1-3mn to compute the last 100 blocks - `website`: A web app which displays the generated datasets in various charts - `server`: A small server which will serve both the website and the computed datasets via an API @@ -70,9 +72,12 @@ Please open an issue if you want to add another instance ### Requirements -- At least 16 GB of RAM +- At least 8 GB of RAM - 1 TB of free space (will use 60-80% of that) -- A running instance of bitcoin-core (>= 28.0 **NOT** supported) with txindex=1 and rpc credentials +- A running instance of bitcoin-core with: + - `-txindex=1` + - `-blocksxor=0` + - RPC credentials - Git ### Docker diff --git a/parser/src/actions/min_height.rs b/parser/src/actions/min_height.rs index 2b68aa2d3..953c1ddf1 100644 --- a/parser/src/actions/min_height.rs +++ b/parser/src/actions/min_height.rs @@ -67,7 +67,7 @@ pub fn find_first_inserted_unsafe_height( let inserted_last_date_is_older_than_saved_state = min_datasets_inserted_last_date.map_or(true, |min_datasets_last_date| min_datasets_last_date < last_safe_date); if inserted_last_date_is_older_than_saved_state { - dbg!(min_datasets_inserted_last_date , *last_safe_date); + // dbg!(min_datasets_inserted_last_date , *last_safe_date); return None; } diff --git a/parser/src/databases/address_index_to_address_data.rs b/parser/src/databases/address_index_to_address_data.rs index b89385502..a8d1c0dd3 100644 --- a/parser/src/databases/address_index_to_address_data.rs +++ b/parser/src/databases/address_index_to_address_data.rs @@ -130,8 +130,9 @@ impl AddressIndexToAddressData { impl AnyDatabaseGroup for AddressIndexToAddressData { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 1), + map: BTreeMap::default(), - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/databases/address_index_to_empty_address_data.rs b/parser/src/databases/address_index_to_empty_address_data.rs index 8f30034e1..f9a284684 100644 --- a/parser/src/databases/address_index_to_empty_address_data.rs +++ b/parser/src/databases/address_index_to_empty_address_data.rs @@ -90,8 +90,9 @@ impl AddressIndexToEmptyAddressData { impl AnyDatabaseGroup for AddressIndexToEmptyAddressData { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 1), + map: BTreeMap::default(), - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/databases/address_to_address_index.rs b/parser/src/databases/address_to_address_index.rs index 7cf5bc2a8..b28c9709d 100644 --- a/parser/src/databases/address_to_address_index.rs +++ b/parser/src/databases/address_to_address_index.rs @@ -235,6 +235,8 @@ impl AddressToAddressIndex { impl AnyDatabaseGroup for AddressToAddressIndex { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 1), + p2pk: BTreeMap::default(), p2pkh: BTreeMap::default(), p2sh: BTreeMap::default(), @@ -246,7 +248,6 @@ impl AnyDatabaseGroup for AddressToAddressIndex { unknown: None, empty: None, multisig: None, - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/databases/metadata.rs b/parser/src/databases/metadata.rs index 6cf56ceb0..1081f69aa 100644 --- a/parser/src/databases/metadata.rs +++ b/parser/src/databases/metadata.rs @@ -1,5 +1,6 @@ use allocative::Allocative; use bincode::{Decode, Encode}; +use color_eyre::eyre::eyre; use serde::{Deserialize, Serialize}; use std::{ fmt::Debug, @@ -34,10 +35,10 @@ impl DerefMut for Metadata { } impl Metadata { - pub fn import(path: &str) -> Self { + pub fn import(path: &str, version: u16) -> Self { Self { path: path.to_owned(), - data: MetadataData::import(path).unwrap_or_default(), + data: MetadataData::import(path, version).unwrap_or_default(), } } @@ -77,6 +78,7 @@ impl Metadata { #[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)] pub struct MetadataData { + version: u16, pub serial: usize, pub len: Counter, pub last_height: Option, @@ -93,10 +95,16 @@ impl MetadataData { format!("{folder_path}/{name}") } - pub fn import(path: &str) -> color_eyre::Result { + pub fn import(path: &str, version: u16) -> color_eyre::Result { fs::create_dir_all(path)?; - Serialization::Binary.import(Path::new(&Self::full_path(path))) + let s: MetadataData = Serialization::Binary.import(Path::new(&Self::full_path(path)))?; + + if s.version != version { + return Err(eyre!("Bad version")); + } + + Ok(s) } pub fn export(&self, path: &str) -> color_eyre::Result<()> { diff --git a/parser/src/databases/txid_to_tx_data.rs b/parser/src/databases/txid_to_tx_data.rs index 696516a22..de77c523a 100644 --- a/parser/src/databases/txid_to_tx_data.rs +++ b/parser/src/databases/txid_to_tx_data.rs @@ -114,15 +114,16 @@ impl TxidToTxData { } fn db_index(txid: &Txid) -> u16 { - ((txid[0] as u16) << 4) + ((txid[1] as u16) >> 4) + ((txid[0] as u16) << 5) + ((txid[1] as u16) >> 3) } } impl AnyDatabaseGroup for TxidToTxData { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 2), + map: BTreeMap::default(), - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/databases/txout_index_to_address_index.rs b/parser/src/databases/txout_index_to_address_index.rs index dc582bd2b..38f1d0c3a 100644 --- a/parser/src/databases/txout_index_to_address_index.rs +++ b/parser/src/databases/txout_index_to_address_index.rs @@ -89,8 +89,9 @@ impl TxoutIndexToAddressIndex { impl AnyDatabaseGroup for TxoutIndexToAddressIndex { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 1), + map: BTreeMap::default(), - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/databases/txout_index_to_amount.rs b/parser/src/databases/txout_index_to_amount.rs index 28319a4d2..6d469f65a 100644 --- a/parser/src/databases/txout_index_to_amount.rs +++ b/parser/src/databases/txout_index_to_amount.rs @@ -89,8 +89,9 @@ impl TxoutIndexToAmount { impl AnyDatabaseGroup for TxoutIndexToAmount { fn import() -> Self { Self { + metadata: Metadata::import(&Self::full_path(), 1), + map: BTreeMap::default(), - metadata: Metadata::import(&Self::full_path()), } } diff --git a/parser/src/datasets/mod.rs b/parser/src/datasets/mod.rs index 2fa4e3db3..43a8c94c1 100644 --- a/parser/src/datasets/mod.rs +++ b/parser/src/datasets/mod.rs @@ -149,7 +149,9 @@ impl AllDatasets { } pub fn insert(&mut self, insert_data: InsertData) { - self.address.insert(&insert_data); + if insert_data.compute_addresses { + self.address.insert(&insert_data); + } self.utxo.insert(&insert_data); diff --git a/parser/src/datasets/price/mod.rs b/parser/src/datasets/price/mod.rs index 058647b83..863d7f8ca 100644 --- a/parser/src/datasets/price/mod.rs +++ b/parser/src/datasets/price/mod.rs @@ -354,14 +354,14 @@ impl PriceDatasets { dates, &mut self.all_time_high.date, |(value, date, _, map)| { - let high = self.high.date.get(date).unwrap(); + let high = self.high.date.get_or_import(date).unwrap(); let is_ath = high == value; if is_ath { *date } else { let previous_date = date.checked_sub(1).unwrap(); - *map.get(&previous_date).as_ref().unwrap_or(date) + *map.get_or_import(&previous_date).as_ref().unwrap_or(date) } }, ); @@ -399,7 +399,7 @@ impl PriceDatasets { pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { if self.ohlc.date.is_key_safe(date) { - Ok(self.ohlc.date.get(&date).unwrap().to_owned()) + Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned()) } else { let ohlc = self .get_from_daily_kraken(&date) @@ -488,7 +488,7 @@ impl PriceDatasets { timestamp: Timestamp, previous_timestamp: Option, ) -> color_eyre::Result { - if let Some(ohlc) = self.ohlc.height.get(&height) { + if let Some(ohlc) = self.ohlc.height.get_or_import(&height) { return Ok(ohlc); } diff --git a/parser/src/io/binary.rs b/parser/src/io/binary.rs index b150d1c3d..202a6bdfb 100644 --- a/parser/src/io/binary.rs +++ b/parser/src/io/binary.rs @@ -60,7 +60,7 @@ impl Binary { let config = config::standard(); - let decoded = decode_from_slice::(&decompressed, config).unwrap().0; + let decoded = decode_from_slice::(&decompressed, config)?.0; Ok(decoded) } diff --git a/parser/src/states/cohorts_states/utxo/cohort_filter.rs b/parser/src/states/cohorts_states/utxo/cohort_filter.rs index 2479080a8..41a3f2d47 100644 --- a/parser/src/states/cohorts_states/utxo/cohort_filter.rs +++ b/parser/src/states/cohorts_states/utxo/cohort_filter.rs @@ -1,17 +1,19 @@ +use crate::structs::{Epoch, Height}; + pub enum UTXOFilter { To(u32), FromTo { from: u32, to: u32 }, From(u32), - Year(u32), + Epoch(Epoch), } impl UTXOCheck for UTXOFilter { - fn check(&self, days_old: &u32, year: &u32) -> bool { + fn check(&self, days_old: &u32, height: &Height) -> bool { match self { UTXOFilter::From(from) => from <= days_old, UTXOFilter::To(to) => to > days_old, UTXOFilter::FromTo { from, to } => from <= days_old && to > days_old, - UTXOFilter::Year(_year) => _year == year, + UTXOFilter::Epoch(epoch) => *epoch == height.into(), } } @@ -20,13 +22,13 @@ impl UTXOCheck for UTXOFilter { UTXOFilter::From(from) => from <= days_old, UTXOFilter::To(to) => to > days_old, UTXOFilter::FromTo { from, to } => from <= days_old && to > days_old, - UTXOFilter::Year(_) => unreachable!(), + UTXOFilter::Epoch(_) => unreachable!(), } } } pub trait UTXOCheck { - fn check(&self, days_old: &u32, year: &u32) -> bool; + fn check(&self, days_old: &u32, height: &Height) -> bool; fn check_days_old(&self, days_old: &u32) -> bool; } diff --git a/parser/src/states/cohorts_states/utxo/cohort_filters.rs b/parser/src/states/cohorts_states/utxo/cohort_filters.rs index 5ce5da040..da9a3f767 100644 --- a/parser/src/states/cohorts_states/utxo/cohort_filters.rs +++ b/parser/src/states/cohorts_states/utxo/cohort_filters.rs @@ -1,3 +1,5 @@ +use crate::structs::Epoch; + use super::{SplitByUTXOCohort, UTXOFilter}; pub const UTXO_FILTERS: SplitByUTXOCohort = SplitByUTXOCohort { @@ -62,22 +64,11 @@ pub const UTXO_FILTERS: SplitByUTXOCohort = SplitByUTXOCohort { from_10y: UTXOFilter::From(10 * 365), from_15y: UTXOFilter::From(15 * 365), - year_2009: UTXOFilter::Year(2009), - year_2010: UTXOFilter::Year(2010), - year_2011: UTXOFilter::Year(2011), - year_2012: UTXOFilter::Year(2012), - year_2013: UTXOFilter::Year(2013), - year_2014: UTXOFilter::Year(2014), - year_2015: UTXOFilter::Year(2015), - year_2016: UTXOFilter::Year(2016), - year_2017: UTXOFilter::Year(2017), - year_2018: UTXOFilter::Year(2018), - year_2019: UTXOFilter::Year(2019), - year_2020: UTXOFilter::Year(2020), - year_2021: UTXOFilter::Year(2021), - year_2022: UTXOFilter::Year(2022), - year_2023: UTXOFilter::Year(2023), - year_2024: UTXOFilter::Year(2024), + epoch_1: UTXOFilter::Epoch(Epoch(1)), + epoch_2: UTXOFilter::Epoch(Epoch(2)), + epoch_3: UTXOFilter::Epoch(Epoch(3)), + epoch_4: UTXOFilter::Epoch(Epoch(4)), + epoch_5: UTXOFilter::Epoch(Epoch(5)), sth: UTXOFilter::To(155), lth: UTXOFilter::From(155), diff --git a/parser/src/states/cohorts_states/utxo/cohort_id.rs b/parser/src/states/cohorts_states/utxo/cohort_id.rs index dbe3dccfa..846708e02 100644 --- a/parser/src/states/cohorts_states/utxo/cohort_id.rs +++ b/parser/src/states/cohorts_states/utxo/cohort_id.rs @@ -37,22 +37,11 @@ pub enum UTXOCohortId { From10y, From15y, - Year2009, - Year2010, - Year2011, - Year2012, - Year2013, - Year2014, - Year2015, - Year2016, - Year2017, - Year2018, - Year2019, - Year2020, - Year2021, - Year2022, - Year2023, - Year2024, + Epoch1, + Epoch2, + Epoch3, + Epoch4, + Epoch5, ShortTermHolders, LongTermHolders, @@ -95,22 +84,11 @@ impl UTXOCohortId { Self::From10y => "from_10y", Self::From15y => "from_15y", - Self::Year2009 => "year_2009", - Self::Year2010 => "year_2010", - Self::Year2011 => "year_2011", - Self::Year2012 => "year_2012", - Self::Year2013 => "year_2013", - Self::Year2014 => "year_2014", - Self::Year2015 => "year_2015", - Self::Year2016 => "year_2016", - Self::Year2017 => "year_2017", - Self::Year2018 => "year_2018", - Self::Year2019 => "year_2019", - Self::Year2020 => "year_2020", - Self::Year2021 => "year_2021", - Self::Year2022 => "year_2022", - Self::Year2023 => "year_2023", - Self::Year2024 => "year_2024", + Self::Epoch1 => "epoch_1", + Self::Epoch2 => "epoch_2", + Self::Epoch3 => "epoch_3", + Self::Epoch4 => "epoch_4", + Self::Epoch5 => "epoch_5", Self::ShortTermHolders => "sth", Self::LongTermHolders => "lth", diff --git a/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs b/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs index 3dab4ee4f..16cafddcf 100644 --- a/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs +++ b/parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs @@ -1,5 +1,4 @@ use allocative::Allocative; -use chrono::Datelike; use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; @@ -19,8 +18,6 @@ impl UTXOCohortsDurableStates { if let Some(last_block_data) = date_data_vec.last_block() { date_data_vec.iter().for_each(|date_data| { - let year = date_data.date.year() as u32; - date_data.blocks.iter().for_each(|block_data| { let amount = block_data.amount; let utxo_count = block_data.utxos as f64; @@ -35,7 +32,7 @@ impl UTXOCohortsDurableStates { last_block_data.timestamp, ); - s.initial_filtered_apply(&increment_days_old, &year, |state| { + s.initial_filtered_apply(&increment_days_old, &block_data.height, |state| { state .increment(amount, utxo_count, block_data.price) .unwrap(); @@ -53,6 +50,7 @@ impl UTXOCohortsDurableStates { last_block_data: &BlockData, previous_last_block_data: Option<&BlockData>, ) { + let height = block_data.height; let amount = block_data.amount; let utxo_count = block_data.utxos as f64; let price = block_data.price; @@ -63,9 +61,7 @@ impl UTXOCohortsDurableStates { } if block_data.height == last_block_data.height { - let year = block_data.timestamp.to_year(); - - self.initial_filtered_apply(&0, &year, |state| { + self.initial_filtered_apply(&0, &height, |state| { state.increment(amount, utxo_count, price).unwrap(); }) } else { @@ -120,9 +116,7 @@ impl UTXOCohortsDurableStates { previous_last_block_data.timestamp, ); - let year = block_data.timestamp.to_year(); - - self.initial_filtered_apply(&days_old, &year, |state| { + self.initial_filtered_apply(&days_old, &block_data.height, |state| { state .decrement(amount, utxo_count, block_data.price) .unwrap_or_else(|report| { diff --git a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs b/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs index 73b037162..9b0da8a05 100644 --- a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs +++ b/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs @@ -1,6 +1,5 @@ use std::{cmp::Ordering, collections::BTreeMap}; -use chrono::Datelike; use derive_deref::{Deref, DerefMut}; use crate::{ @@ -33,8 +32,6 @@ impl UTXOCohortsSentStates { .for_each(|(block_path, sent_data)| { let date_data = date_data_vec.get_date_data(block_path).unwrap(); - let year = date_data.date.year() as u32; - let block_data = date_data.get_block_data(block_path).unwrap(); let days_old = Timestamp::difference_in_days_between( @@ -44,10 +41,11 @@ impl UTXOCohortsSentStates { let previous_timestamp = block_data.timestamp; let previous_price = block_data.price; + let height = block_data.height; let amount_sent = sent_data.volume; - self.initial_filtered_apply(&days_old, &year, |state| { + self.initial_filtered_apply(&days_old, &height, |state| { state.input.iterate(sent_data.count as f64, amount_sent); let previous_value = previous_price * amount_sent; diff --git a/parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs b/parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs index e5054c143..d2d310b0a 100644 --- a/parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs +++ b/parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs @@ -2,6 +2,8 @@ use allocative::Allocative; use super::{UTXOCheck, UTXOCohortId, UTXO_FILTERS}; +use crate::structs::Height; + #[derive(Default, Allocative)] pub struct SplitByUTXOCohort { pub sth: T, @@ -41,22 +43,11 @@ pub struct SplitByUTXOCohort { pub from_10y: T, pub from_15y: T, - pub year_2009: T, - pub year_2010: T, - pub year_2011: T, - pub year_2012: T, - pub year_2013: T, - pub year_2014: T, - pub year_2015: T, - pub year_2016: T, - pub year_2017: T, - pub year_2018: T, - pub year_2019: T, - pub year_2020: T, - pub year_2021: T, - pub year_2022: T, - pub year_2023: T, - pub year_2024: T, + pub epoch_1: T, + pub epoch_2: T, + pub epoch_3: T, + pub epoch_4: T, + pub epoch_5: T, } impl SplitByUTXOCohort { @@ -93,22 +84,11 @@ impl SplitByUTXOCohort { UTXOCohortId::From4y => &self.from_4y, UTXOCohortId::From10y => &self.from_10y, UTXOCohortId::From15y => &self.from_15y, - UTXOCohortId::Year2009 => &self.year_2009, - UTXOCohortId::Year2010 => &self.year_2010, - UTXOCohortId::Year2011 => &self.year_2011, - UTXOCohortId::Year2012 => &self.year_2012, - UTXOCohortId::Year2013 => &self.year_2013, - UTXOCohortId::Year2014 => &self.year_2014, - UTXOCohortId::Year2015 => &self.year_2015, - UTXOCohortId::Year2016 => &self.year_2016, - UTXOCohortId::Year2017 => &self.year_2017, - UTXOCohortId::Year2018 => &self.year_2018, - UTXOCohortId::Year2019 => &self.year_2019, - UTXOCohortId::Year2020 => &self.year_2020, - UTXOCohortId::Year2021 => &self.year_2021, - UTXOCohortId::Year2022 => &self.year_2022, - UTXOCohortId::Year2023 => &self.year_2023, - UTXOCohortId::Year2024 => &self.year_2024, + UTXOCohortId::Epoch1 => &self.epoch_1, + UTXOCohortId::Epoch2 => &self.epoch_2, + UTXOCohortId::Epoch3 => &self.epoch_3, + UTXOCohortId::Epoch4 => &self.epoch_4, + UTXOCohortId::Epoch5 => &self.epoch_5, UTXOCohortId::ShortTermHolders => &self.sth, UTXOCohortId::LongTermHolders => &self.lth, } @@ -147,28 +127,17 @@ impl SplitByUTXOCohort { UTXOCohortId::From4y => &mut self.from_4y, UTXOCohortId::From10y => &mut self.from_10y, UTXOCohortId::From15y => &mut self.from_15y, - UTXOCohortId::Year2009 => &mut self.year_2009, - UTXOCohortId::Year2010 => &mut self.year_2010, - UTXOCohortId::Year2011 => &mut self.year_2011, - UTXOCohortId::Year2012 => &mut self.year_2012, - UTXOCohortId::Year2013 => &mut self.year_2013, - UTXOCohortId::Year2014 => &mut self.year_2014, - UTXOCohortId::Year2015 => &mut self.year_2015, - UTXOCohortId::Year2016 => &mut self.year_2016, - UTXOCohortId::Year2017 => &mut self.year_2017, - UTXOCohortId::Year2018 => &mut self.year_2018, - UTXOCohortId::Year2019 => &mut self.year_2019, - UTXOCohortId::Year2020 => &mut self.year_2020, - UTXOCohortId::Year2021 => &mut self.year_2021, - UTXOCohortId::Year2022 => &mut self.year_2022, - UTXOCohortId::Year2023 => &mut self.year_2023, - UTXOCohortId::Year2024 => &mut self.year_2024, + UTXOCohortId::Epoch1 => &mut self.epoch_1, + UTXOCohortId::Epoch2 => &mut self.epoch_2, + UTXOCohortId::Epoch3 => &mut self.epoch_3, + UTXOCohortId::Epoch4 => &mut self.epoch_4, + UTXOCohortId::Epoch5 => &mut self.epoch_5, UTXOCohortId::ShortTermHolders => &mut self.sth, UTXOCohortId::LongTermHolders => &mut self.lth, } } - /// Excluding years since they're static + /// Excluding epochs since they're static pub fn duo_filtered_apply( &mut self, current_days_old: &u32, @@ -455,175 +424,158 @@ impl SplitByUTXOCohort { } } - /// Includes years since it's the initial apply - pub fn initial_filtered_apply(&mut self, days_old: &u32, year: &u32, apply: impl Fn(&mut T)) { - if UTXO_FILTERS.up_to_1d.check(days_old, year) { + /// Includes epochs since it's the initial apply + pub fn initial_filtered_apply( + &mut self, + days_old: &u32, + height: &Height, + apply: impl Fn(&mut T), + ) { + if UTXO_FILTERS.up_to_1d.check(days_old, height) { apply(&mut self.up_to_1d); - } else if UTXO_FILTERS.from_1d_to_1w.check(days_old, year) { + } else if UTXO_FILTERS.from_1d_to_1w.check(days_old, height) { apply(&mut self.from_1d_to_1w); - } else if UTXO_FILTERS.from_1w_to_1m.check(days_old, year) { + } else if UTXO_FILTERS.from_1w_to_1m.check(days_old, height) { apply(&mut self.from_1w_to_1m); - } else if UTXO_FILTERS.from_1m_to_3m.check(days_old, year) { + } else if UTXO_FILTERS.from_1m_to_3m.check(days_old, height) { apply(&mut self.from_1m_to_3m); - } else if UTXO_FILTERS.from_3m_to_6m.check(days_old, year) { + } else if UTXO_FILTERS.from_3m_to_6m.check(days_old, height) { apply(&mut self.from_3m_to_6m); - } else if UTXO_FILTERS.from_6m_to_1y.check(days_old, year) { + } else if UTXO_FILTERS.from_6m_to_1y.check(days_old, height) { apply(&mut self.from_6m_to_1y); - } else if UTXO_FILTERS.from_1y_to_2y.check(days_old, year) { + } else if UTXO_FILTERS.from_1y_to_2y.check(days_old, height) { apply(&mut self.from_1y_to_2y); - } else if UTXO_FILTERS.from_2y_to_3y.check(days_old, year) { + } else if UTXO_FILTERS.from_2y_to_3y.check(days_old, height) { apply(&mut self.from_2y_to_3y); - } else if UTXO_FILTERS.from_3y_to_5y.check(days_old, year) { + } else if UTXO_FILTERS.from_3y_to_5y.check(days_old, height) { apply(&mut self.from_3y_to_5y); - } else if UTXO_FILTERS.from_5y_to_7y.check(days_old, year) { + } else if UTXO_FILTERS.from_5y_to_7y.check(days_old, height) { apply(&mut self.from_5y_to_7y); - } else if UTXO_FILTERS.from_7y_to_10y.check(days_old, year) { + } else if UTXO_FILTERS.from_7y_to_10y.check(days_old, height) { apply(&mut self.from_7y_to_10y); - } else if UTXO_FILTERS.from_10y_to_15y.check(days_old, year) { + } else if UTXO_FILTERS.from_10y_to_15y.check(days_old, height) { apply(&mut self.from_10y_to_15y); } - if UTXO_FILTERS.year_2009.check(days_old, year) { - apply(&mut self.year_2009); - } else if UTXO_FILTERS.year_2010.check(days_old, year) { - apply(&mut self.year_2010); - } else if UTXO_FILTERS.year_2011.check(days_old, year) { - apply(&mut self.year_2011); - } else if UTXO_FILTERS.year_2012.check(days_old, year) { - apply(&mut self.year_2012); - } else if UTXO_FILTERS.year_2013.check(days_old, year) { - apply(&mut self.year_2013); - } else if UTXO_FILTERS.year_2014.check(days_old, year) { - apply(&mut self.year_2014); - } else if UTXO_FILTERS.year_2015.check(days_old, year) { - apply(&mut self.year_2015); - } else if UTXO_FILTERS.year_2016.check(days_old, year) { - apply(&mut self.year_2016); - } else if UTXO_FILTERS.year_2017.check(days_old, year) { - apply(&mut self.year_2017); - } else if UTXO_FILTERS.year_2018.check(days_old, year) { - apply(&mut self.year_2018); - } else if UTXO_FILTERS.year_2019.check(days_old, year) { - apply(&mut self.year_2019); - } else if UTXO_FILTERS.year_2020.check(days_old, year) { - apply(&mut self.year_2020); - } else if UTXO_FILTERS.year_2021.check(days_old, year) { - apply(&mut self.year_2021); - } else if UTXO_FILTERS.year_2022.check(days_old, year) { - apply(&mut self.year_2022); - } else if UTXO_FILTERS.year_2023.check(days_old, year) { - apply(&mut self.year_2023); - } else if UTXO_FILTERS.year_2024.check(days_old, year) { - apply(&mut self.year_2024); + if UTXO_FILTERS.epoch_1.check(days_old, height) { + apply(&mut self.epoch_1); + } else if UTXO_FILTERS.epoch_2.check(days_old, height) { + apply(&mut self.epoch_2); + } else if UTXO_FILTERS.epoch_3.check(days_old, height) { + apply(&mut self.epoch_3); + } else if UTXO_FILTERS.epoch_4.check(days_old, height) { + apply(&mut self.epoch_4); + } else if UTXO_FILTERS.epoch_5.check(days_old, height) { + apply(&mut self.epoch_5); } - if UTXO_FILTERS.sth.check(days_old, year) { + if UTXO_FILTERS.sth.check(days_old, height) { apply(&mut self.sth); - } else if UTXO_FILTERS.lth.check(days_old, year) { + } else if UTXO_FILTERS.lth.check(days_old, height) { apply(&mut self.lth); } else { unreachable!() } - if UTXO_FILTERS.from_1y.check(days_old, year) { + if UTXO_FILTERS.from_1y.check(days_old, height) { apply(&mut self.from_1y); } - if UTXO_FILTERS.from_2y.check(days_old, year) { + if UTXO_FILTERS.from_2y.check(days_old, height) { apply(&mut self.from_2y); } - if UTXO_FILTERS.from_4y.check(days_old, year) { + if UTXO_FILTERS.from_4y.check(days_old, height) { apply(&mut self.from_4y); } - if UTXO_FILTERS.from_10y.check(days_old, year) { + if UTXO_FILTERS.from_10y.check(days_old, height) { apply(&mut self.from_10y); } - if UTXO_FILTERS.from_15y.check(days_old, year) { + if UTXO_FILTERS.from_15y.check(days_old, height) { apply(&mut self.from_15y); } - if UTXO_FILTERS.up_to_15y.check(days_old, year) { + if UTXO_FILTERS.up_to_15y.check(days_old, height) { apply(&mut self.up_to_15y); } else { return; } - if UTXO_FILTERS.up_to_10y.check(days_old, year) { + if UTXO_FILTERS.up_to_10y.check(days_old, height) { apply(&mut self.up_to_10y); } else { return; } - if UTXO_FILTERS.up_to_7y.check(days_old, year) { + if UTXO_FILTERS.up_to_7y.check(days_old, height) { apply(&mut self.up_to_7y); } else { return; } - if UTXO_FILTERS.up_to_5y.check(days_old, year) { + if UTXO_FILTERS.up_to_5y.check(days_old, height) { apply(&mut self.up_to_5y); } else { return; } - if UTXO_FILTERS.up_to_3y.check(days_old, year) { + if UTXO_FILTERS.up_to_3y.check(days_old, height) { apply(&mut self.up_to_3y); } else { return; } - if UTXO_FILTERS.up_to_2y.check(days_old, year) { + if UTXO_FILTERS.up_to_2y.check(days_old, height) { apply(&mut self.up_to_2y); } else { return; } - if UTXO_FILTERS.up_to_1y.check(days_old, year) { + if UTXO_FILTERS.up_to_1y.check(days_old, height) { apply(&mut self.up_to_1y); } else { return; } - if UTXO_FILTERS.up_to_6m.check(days_old, year) { + if UTXO_FILTERS.up_to_6m.check(days_old, height) { apply(&mut self.up_to_6m); } else { return; } - if UTXO_FILTERS.up_to_5m.check(days_old, year) { + if UTXO_FILTERS.up_to_5m.check(days_old, height) { apply(&mut self.up_to_5m); } else { return; } - if UTXO_FILTERS.up_to_4m.check(days_old, year) { + if UTXO_FILTERS.up_to_4m.check(days_old, height) { apply(&mut self.up_to_4m); } else { return; } - if UTXO_FILTERS.up_to_3m.check(days_old, year) { + if UTXO_FILTERS.up_to_3m.check(days_old, height) { apply(&mut self.up_to_3m); } else { return; } - if UTXO_FILTERS.up_to_2m.check(days_old, year) { + if UTXO_FILTERS.up_to_2m.check(days_old, height) { apply(&mut self.up_to_2m); } else { return; } - if UTXO_FILTERS.up_to_1m.check(days_old, year) { + if UTXO_FILTERS.up_to_1m.check(days_old, height) { apply(&mut self.up_to_1m); } else { return; } - if UTXO_FILTERS.up_to_1w.check(days_old, year) { + if UTXO_FILTERS.up_to_1w.check(days_old, height) { apply(&mut self.up_to_1w); } } @@ -662,22 +614,11 @@ impl SplitByUTXOCohort { (&self.from_4y, UTXOCohortId::From4y), (&self.from_10y, UTXOCohortId::From10y), (&self.from_15y, UTXOCohortId::From15y), - (&self.year_2009, UTXOCohortId::Year2009), - (&self.year_2010, UTXOCohortId::Year2010), - (&self.year_2011, UTXOCohortId::Year2011), - (&self.year_2012, UTXOCohortId::Year2012), - (&self.year_2013, UTXOCohortId::Year2013), - (&self.year_2014, UTXOCohortId::Year2014), - (&self.year_2015, UTXOCohortId::Year2015), - (&self.year_2016, UTXOCohortId::Year2016), - (&self.year_2017, UTXOCohortId::Year2017), - (&self.year_2018, UTXOCohortId::Year2018), - (&self.year_2019, UTXOCohortId::Year2019), - (&self.year_2020, UTXOCohortId::Year2020), - (&self.year_2021, UTXOCohortId::Year2021), - (&self.year_2022, UTXOCohortId::Year2022), - (&self.year_2023, UTXOCohortId::Year2023), - (&self.year_2024, UTXOCohortId::Year2024), + (&self.epoch_1, UTXOCohortId::Epoch1), + (&self.epoch_2, UTXOCohortId::Epoch2), + (&self.epoch_3, UTXOCohortId::Epoch3), + (&self.epoch_4, UTXOCohortId::Epoch4), + (&self.epoch_5, UTXOCohortId::Epoch5), (&self.sth, UTXOCohortId::ShortTermHolders), (&self.lth, UTXOCohortId::LongTermHolders), ] @@ -717,22 +658,11 @@ impl SplitByUTXOCohort { (&mut self.from_4y, UTXOCohortId::From4y), (&mut self.from_10y, UTXOCohortId::From10y), (&mut self.from_15y, UTXOCohortId::From15y), - (&mut self.year_2009, UTXOCohortId::Year2009), - (&mut self.year_2010, UTXOCohortId::Year2010), - (&mut self.year_2011, UTXOCohortId::Year2011), - (&mut self.year_2012, UTXOCohortId::Year2012), - (&mut self.year_2013, UTXOCohortId::Year2013), - (&mut self.year_2014, UTXOCohortId::Year2014), - (&mut self.year_2015, UTXOCohortId::Year2015), - (&mut self.year_2016, UTXOCohortId::Year2016), - (&mut self.year_2017, UTXOCohortId::Year2017), - (&mut self.year_2018, UTXOCohortId::Year2018), - (&mut self.year_2019, UTXOCohortId::Year2019), - (&mut self.year_2020, UTXOCohortId::Year2020), - (&mut self.year_2021, UTXOCohortId::Year2021), - (&mut self.year_2022, UTXOCohortId::Year2022), - (&mut self.year_2023, UTXOCohortId::Year2023), - (&mut self.year_2024, UTXOCohortId::Year2024), + (&mut self.epoch_1, UTXOCohortId::Epoch1), + (&mut self.epoch_2, UTXOCohortId::Epoch2), + (&mut self.epoch_3, UTXOCohortId::Epoch3), + (&mut self.epoch_4, UTXOCohortId::Epoch4), + (&mut self.epoch_5, UTXOCohortId::Epoch5), (&mut self.sth, UTXOCohortId::ShortTermHolders), (&mut self.lth, UTXOCohortId::LongTermHolders), ] diff --git a/parser/src/structs/epoch.rs b/parser/src/structs/epoch.rs new file mode 100644 index 000000000..ed7108a4c --- /dev/null +++ b/parser/src/structs/epoch.rs @@ -0,0 +1,20 @@ +use super::{Height, MapKey}; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Epoch(pub u16); + +impl Epoch { + pub const BLOCKS_PER_EPOCH: usize = 210_000; +} + +impl From for Epoch { + fn from(height: Height) -> Self { + Self(((height.to_usize() / Self::BLOCKS_PER_EPOCH) + 1) as u16) + } +} + +impl From<&Height> for Epoch { + fn from(height: &Height) -> Self { + Self(((height.to_usize() / Self::BLOCKS_PER_EPOCH) + 1) as u16) + } +} diff --git a/parser/src/structs/generic_map.rs b/parser/src/structs/generic_map.rs index 5cb48761c..55ca45a0d 100644 --- a/parser/src/structs/generic_map.rs +++ b/parser/src/structs/generic_map.rs @@ -202,10 +202,12 @@ where } pub fn insert(&mut self, key: Key, value: Value) -> Value { - self.to_insert - .entry(key.to_chunk_id()) - .or_default() - .insert(key.to_serialized_key(), value); + if !self.is_key_safe(key) { + self.to_insert + .entry(key.to_chunk_id()) + .or_default() + .insert(key.to_serialized_key(), value); + } value } @@ -443,15 +445,14 @@ where SourceValue, &Key, &mut GenericMap, - &Self, + &mut Self, ), ) -> Value, { keys.iter().for_each(|key| { - self.insert( - *key, - transform((source.get_or_import(key).unwrap(), key, source, self)), - ); + let value = transform((source.get_or_import(key).unwrap(), key, source, self)); + + self.insert(*key, value); }); } diff --git a/parser/src/structs/mod.rs b/parser/src/structs/mod.rs index 5131aa8e7..7104b51f0 100644 --- a/parser/src/structs/mod.rs +++ b/parser/src/structs/mod.rs @@ -17,6 +17,7 @@ mod date_data; mod date_map; mod date_map_chunk_id; mod empty_address_data; +mod epoch; mod exit; mod generic_map; mod height; @@ -53,6 +54,7 @@ pub use date_data::*; pub use date_map::*; pub use date_map_chunk_id::*; pub use empty_address_data::*; +pub use epoch::*; pub use exit::*; pub use generic_map::*; pub use height::*; diff --git a/website/index.html b/website/index.html index 908673216..77b142a42 100644 --- a/website/index.html +++ b/website/index.html @@ -795,6 +795,11 @@ color: var(--off-color); font-size: var(--font-size-xs); line-height: var(--line-height-xs); + + h4 + & { + font-size: var(--font-size-sm); + line-height: var(--line-height-sm); + } } span { @@ -960,6 +965,14 @@ details[open] > summary > & { background-color: var(--background-color); } + + summary:hover > & { + border-color: var(--orange) !important; + } + + details:not([open]) > summary:hover > & { + background-color: var(--orange); + } } > small { @@ -2041,79 +2054,7 @@ - +