diff --git a/CHANGELOG.md b/CHANGELOG.md index a3f7480a1..1f99515f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Changelog +## v. 0.2.1 | WIP + +### Parser + +- Datasets + - Added the followinf datasets for all entities: + - Value destroyed + - Value created + - Spent Output Profit Ratio (SOPR) + - Market Price to Realized Price + - Market Price to Realized Price Ratio + - Market Price to Realized Price Ratio 1 Week SMA + - Market Price to Realized Price Ratio 1 Month SMA + - Market Price to Realized Price Ratio 1 Year SMA + - Market Price to Realized Price Ratio 99th Percentile + - Market Price to Realized Price Ratio 99.5th Percentile + - Market Price to Realized Price Ratio 99.9th Percentile + - Market Price to Realized Price Ratio 1st Percentile + - Market Price to Realized Price Ratio 0.5th Percentile + - Realized Price 0.1th Percentile + - Realized Price 99.5th Percentile + - Realized Price 99.9th Percentile + - Realized Price 1st Percentile + - Realized Price 0.5th Percentile + - Realized Price 0.1th Percentile + +### App + +- General + - Added a backup API in case the main one fails or is offline +- Chart + - Fixed series color being set to default ones after hovering the legend +- Settings + - Removed the horizontal scroll bar which was unintended + +### Server + +- Run file + - Only run with a watcher if `cargo watch` is available + ## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08 ![Image of the Satonomics Web App version 0.2.0](./assets/v0.2.0.jpg) @@ -7,7 +47,7 @@ ### App - General - - Added height datasets and many optimizations to make them usable but only available on desktop and tablets for now + - Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now - Added a light theme - Charts - Added split panes in order to have the vertical axis visible for all datasets @@ -89,3 +129,7 @@ ## v. 0.1.0 | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19 ![Image of the Satonomics Web App version 0.1.0](./assets/v0.1.0.jpg) + +## v. 0.0.X | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20 + +![Image of the Satonomics Web App version 0.0.X](./assets/v0.0.X.jpg) diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 9c6070831..65d717f6f 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "parser" -version = "0.2.0" +version = "0.2.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -27,5 +27,5 @@ par-iter-sync = "0.1.11" rayon = "1.10.0" reqwest = { version = "0.12.5", features = ["blocking", "json"] } sanakirja = "1.4.2" -serde = { version = "1.0.203", features = ["derive"] } +serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.120" diff --git a/parser/src/actions/export.rs b/parser/src/actions/export.rs index 8906eeaae..3a91cce8d 100644 --- a/parser/src/actions/export.rs +++ b/parser/src/actions/export.rs @@ -25,7 +25,8 @@ pub fn export( date, }: ExportedData, ) -> color_eyre::Result<()> { - log("Exporting... (Don't close !!)"); + log("Exporting..."); + log("WARNING: NOT SAFE TO STOP !!!"); time("Total save time", || -> color_eyre::Result<()> { time("Datasets saved", || datasets.export())?; @@ -43,5 +44,7 @@ pub fn export( Ok(()) })?; + log("Export done - Safe to stop now"); + Ok(()) } diff --git a/parser/src/actions/iter_blocks.rs b/parser/src/actions/iter_blocks.rs index 1e6452969..60ecac3b3 100644 --- a/parser/src/actions/iter_blocks.rs +++ b/parser/src/actions/iter_blocks.rs @@ -166,9 +166,11 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re )); if first_unsafe_heights.computed <= last_height { - datasets.compute(ComputeData { - dates: &processed_dates.into_iter().collect_vec(), - heights: &processed_heights.into_iter().collect_vec(), + time("Computing datasets", || { + datasets.compute(ComputeData { + dates: &processed_dates.into_iter().collect_vec(), + heights: &processed_heights.into_iter().collect_vec(), + }) }); } diff --git a/parser/src/actions/parse.rs b/parser/src/actions/parse.rs index 7cacda1d6..a0cc431f5 100644 --- a/parser/src/actions/parse.rs +++ b/parser/src/actions/parse.rs @@ -476,9 +476,11 @@ pub fn parse( AddressRealizedData::default(input_address_data) }); + let previous_price = input_block_data.price; + // MUST be after `or_insert_with` - let address_realized_profit_or_loss = input_address_data - .send(input_amount, block_price, input_block_data.price) + input_address_data + .send(input_amount, previous_price) .unwrap_or_else(|_| { dbg!( input_address_index, @@ -494,8 +496,11 @@ pub fn parse( panic!() }); - input_address_realized_data - .send(input_amount, address_realized_profit_or_loss); + input_address_realized_data.send( + input_amount, + block_price, + previous_price, + ); }; is_tx_data_from_cached_puts && input_tx_data.is_empty() diff --git a/parser/src/datasets/cointime.rs b/parser/src/datasets/cointime.rs index 04a0454db..64ea222d9 100644 --- a/parser/src/datasets/cointime.rs +++ b/parser/src/datasets/cointime.rs @@ -86,7 +86,7 @@ impl CointimeDataset { cointime_value_stored: BiMap::new_bin(1, &f("cointime_value_stored")), concurrent_liveliness: BiMap::new_bin(1, &f("concurrent_liveliness")), concurrent_liveliness_2w_median: BiMap::new_bin( - 1, + 2, &f("concurrent_liveliness_2w_median"), ), cumulative_coinblocks_created: BiMap::new_bin(1, &f("cumulative_coinblocks_created")), @@ -100,7 +100,7 @@ impl CointimeDataset { liveliness: BiMap::new_bin(1, &f("liveliness")), liveliness_net_change: BiMap::new_bin(1, &f("liveliness_net_change")), liveliness_net_change_2w_median: BiMap::new_bin( - 1, + 2, &f("liveliness_net_change_2w_median"), ), producerness: BiMap::new_bin(1, &f("producerness")), diff --git a/parser/src/datasets/price/mod.rs b/parser/src/datasets/price/mod.rs index db3637c8d..8b07a5f49 100644 --- a/parser/src/datasets/price/mod.rs +++ b/parser/src/datasets/price/mod.rs @@ -24,6 +24,7 @@ pub struct PriceDatasets { kraken_1mn: Option>, binance_1mn: Option>, binance_har: Option>, + satonomics_by_height: BTreeMap>>, // Inserted pub ohlcs: BiMap, @@ -74,6 +75,7 @@ impl PriceDatasets { binance_har: None, kraken_1mn: None, kraken_daily: None, + satonomics_by_height: BTreeMap::default(), ohlcs: BiMap::new_json(1, &format!("{price_path}/ohlc")), closes: BiMap::new_json(1, &f("close")), diff --git a/parser/src/datasets/subs/capitalization.rs b/parser/src/datasets/subs/capitalization.rs index f9cc17acd..a9c04eaf7 100644 --- a/parser/src/datasets/subs/capitalization.rs +++ b/parser/src/datasets/subs/capitalization.rs @@ -7,6 +7,8 @@ use crate::{ utils::ONE_MONTH_IN_DAYS, }; +use super::RatioDataset; + #[derive(Default, Allocative)] pub struct CapitalizationDataset { min_initial_states: MinInitialStates, @@ -16,8 +18,8 @@ pub struct CapitalizationDataset { // Computed pub realized_price: BiMap, - mvrv: BiMap, realized_cap_1m_net_change: BiMap, + ratio: RatioDataset, } impl CapitalizationDataset { @@ -30,7 +32,7 @@ impl CapitalizationDataset { realized_cap: BiMap::new_bin(1, &f("realized_cap")), realized_cap_1m_net_change: BiMap::new_bin(1, &f("realized_cap_1m_net_change")), realized_price: BiMap::new_bin(1, &f("realized_price")), - mvrv: BiMap::new_bin(1, &f("mvrv")), + ratio: RatioDataset::import(parent_path, "realized_price")?, }; s.min_initial_states @@ -61,10 +63,12 @@ impl CapitalizationDataset { pub fn compute( &mut self, - &ComputeData { heights, dates }: &ComputeData, + compute_data: &ComputeData, closes: &mut BiMap, cohort_supply: &mut BiMap, ) { + let &ComputeData { heights, dates } = compute_data; + self.realized_price.multi_insert_divide( heights, dates, @@ -72,21 +76,15 @@ impl CapitalizationDataset { cohort_supply, ); - self.mvrv.height.multi_insert_divide( - heights, - &mut closes.height, - &mut self.realized_price.height, - ); - self.mvrv - .date - .multi_insert_divide(dates, &mut closes.date, &mut self.realized_price.date); - self.realized_cap_1m_net_change.multi_insert_net_change( heights, dates, &mut self.realized_cap, ONE_MONTH_IN_DAYS, - ) + ); + + self.ratio + .compute(compute_data, closes, &mut self.realized_price); } } @@ -104,18 +102,20 @@ impl AnyDataset for CapitalizationDataset { } fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { - vec![ - &self.realized_price, - &self.mvrv, + let mut v = vec![ + &self.realized_price as &(dyn AnyBiMap + Send + Sync), &self.realized_cap_1m_net_change, - ] + ]; + v.append(&mut self.ratio.to_computed_bi_map_vec()); + v } fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { - vec![ - &mut self.realized_price, - &mut self.mvrv, + let mut v = vec![ + &mut self.realized_price as &mut dyn AnyBiMap, &mut self.realized_cap_1m_net_change, - ] + ]; + v.append(&mut self.ratio.to_computed_mut_bi_map_vec()); + v } } diff --git a/parser/src/datasets/subs/input.rs b/parser/src/datasets/subs/input.rs index 68b119e29..662862322 100644 --- a/parser/src/datasets/subs/input.rs +++ b/parser/src/datasets/subs/input.rs @@ -10,8 +10,10 @@ use crate::{ pub struct InputSubDataset { min_initial_states: MinInitialStates, + // Inserted pub count: BiMap, pub volume: BiMap, + // Computed // add inputs_per_second } diff --git a/parser/src/datasets/subs/mod.rs b/parser/src/datasets/subs/mod.rs index 55ac3ca35..860b3e09b 100644 --- a/parser/src/datasets/subs/mod.rs +++ b/parser/src/datasets/subs/mod.rs @@ -2,6 +2,7 @@ use allocative::Allocative; mod capitalization; mod input; +mod ratio; // mod output; mod price_paid; mod realized; @@ -11,6 +12,7 @@ mod utxo; pub use capitalization::*; pub use input::*; +pub use ratio::*; // pub use output::*; pub use price_paid::*; pub use realized::*; diff --git a/parser/src/datasets/subs/ratio.rs b/parser/src/datasets/subs/ratio.rs new file mode 100644 index 000000000..2aff48186 --- /dev/null +++ b/parser/src/datasets/subs/ratio.rs @@ -0,0 +1,207 @@ +use allocative::Allocative; + +use crate::{ + datasets::{AnyDataset, ComputeData, MinInitialStates}, + structs::{AnyBiMap, BiMap}, + utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, +}; + +#[derive(Default, Allocative)] +pub struct RatioDataset { + min_initial_states: MinInitialStates, + + // Computed + ratio: BiMap, + ratio_1w_sma: BiMap, + ratio_1m_sma: BiMap, + ratio_1y_sma: BiMap, + ratio_1y_sma_momentum_oscillator: BiMap, + ratio_99p: BiMap, + ratio_99_5p: BiMap, + ratio_99_9p: BiMap, + ratio_1p: BiMap, + ratio_0_5p: BiMap, + ratio_0_1p: BiMap, + price_99p: BiMap, + price_99_5p: BiMap, + price_99_9p: BiMap, + price_1p: BiMap, + price_0_5p: BiMap, + price_0_1p: BiMap, +} + +impl RatioDataset { + pub fn import(parent_path: &str, name: &str) -> color_eyre::Result { + let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}"); + let f_price = |s: &str| format!("{parent_path}/{name}_{s}"); + + let mut s = Self { + min_initial_states: MinInitialStates::default(), + + ratio: BiMap::new_bin(1, &f_ratio("ratio")), + ratio_1w_sma: BiMap::new_bin(1, &f_ratio("ratio_1w_sma")), + ratio_1m_sma: BiMap::new_bin(1, &f_ratio("ratio_1m_sma")), + ratio_1y_sma: BiMap::new_bin(1, &f_ratio("ratio_1y_sma")), + ratio_1y_sma_momentum_oscillator: BiMap::new_bin( + 1, + &f_ratio("ratio_1y_sma_momentum_oscillator"), + ), + ratio_99p: BiMap::new_bin(1, &f_ratio("ratio_99p")), + ratio_99_5p: BiMap::new_bin(1, &f_ratio("ratio_99_5p")), + ratio_99_9p: BiMap::new_bin(1, &f_ratio("ratio_99_9p")), + ratio_1p: BiMap::new_bin(1, &f_ratio("ratio_1p")), + ratio_0_5p: BiMap::new_bin(1, &f_ratio("ratio_0_5p")), + ratio_0_1p: BiMap::new_bin(1, &f_ratio("ratio_0_1p")), + price_99p: BiMap::new_bin(1, &f_price("99p")), + price_99_5p: BiMap::new_bin(1, &f_price("99_5p")), + price_99_9p: BiMap::new_bin(1, &f_price("99_9p")), + price_1p: BiMap::new_bin(1, &f_price("1p")), + price_0_5p: BiMap::new_bin(1, &f_price("0_5p")), + price_0_1p: BiMap::new_bin(1, &f_price("0_1p")), + }; + + s.min_initial_states + .consume(MinInitialStates::compute_from_dataset(&s)); + + Ok(s) + } + + pub fn compute( + &mut self, + &ComputeData { heights, dates }: &ComputeData, + market_price: &mut BiMap, + other_price: &mut BiMap, + ) { + self.ratio.height.multi_insert_divide( + heights, + &mut market_price.height, + &mut other_price.height, + ); + + self.ratio + .date + .multi_insert_divide(dates, &mut market_price.date, &mut other_price.date); + + self.ratio_1w_sma.multi_insert_simple_average( + heights, + dates, + &mut self.ratio, + ONE_WEEK_IN_DAYS, + ); + + self.ratio_1m_sma.multi_insert_simple_average( + heights, + dates, + &mut self.ratio, + ONE_MONTH_IN_DAYS, + ); + + self.ratio_1m_sma.multi_insert_simple_average( + heights, + dates, + &mut self.ratio, + ONE_MONTH_IN_DAYS, + ); + + self.ratio_1y_sma.multi_insert_simple_average( + heights, + dates, + &mut self.ratio, + ONE_YEAR_IN_DAYS, + ); + + self.ratio_1y_sma_momentum_oscillator + .height + .multi_insert_complex_transform(heights, &mut self.ratio.height, |(ratio, height)| { + (ratio / self.ratio_1y_sma.height.get_or_import(height)) - 1.0 + }); + + self.ratio_1y_sma_momentum_oscillator + .date + .multi_insert_complex_transform(dates, &mut self.ratio.date, |(ratio, date, _)| { + (ratio / self.ratio_1y_sma.date.get_or_import(date).unwrap()) - 1.0 + }); + + self.ratio.multi_insert_percentile( + heights, + dates, + vec![ + (&mut self.ratio_99p, 0.99), + (&mut self.ratio_99_5p, 0.995), + (&mut self.ratio_99_9p, 0.999), + (&mut self.ratio_1p, 0.1), + (&mut self.ratio_0_5p, 0.005), + (&mut self.ratio_0_1p, 0.001), + ], + None, + ); + + self.price_99p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99p); + + self.price_99_5p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99_5p); + + self.price_99_9p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99_9p); + + self.price_1p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_1p); + + self.price_0_5p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_0_5p); + + self.price_0_1p + .multi_insert_multiply(heights, dates, market_price, &mut self.ratio_0_1p); + } +} + +impl AnyDataset for RatioDataset { + fn get_min_initial_states(&self) -> &MinInitialStates { + &self.min_initial_states + } + + fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { + vec![ + &self.ratio, + &self.ratio_1w_sma, + &self.ratio_1m_sma, + &self.ratio_1y_sma, + &self.ratio_1y_sma_momentum_oscillator, + &self.ratio_99p, + &self.ratio_99_5p, + &self.ratio_99_9p, + &self.ratio_1p, + &self.ratio_0_5p, + &self.ratio_0_1p, + &self.price_99p, + &self.price_99_5p, + &self.price_99_9p, + &self.price_1p, + &self.price_0_5p, + &self.price_0_1p, + ] + } + + fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + vec![ + &mut self.ratio, + &mut self.ratio_1w_sma, + &mut self.ratio_1m_sma, + &mut self.ratio_1y_sma, + &mut self.ratio_1y_sma_momentum_oscillator, + &mut self.ratio_99p, + &mut self.ratio_99_5p, + &mut self.ratio_99_9p, + &mut self.ratio_1p, + &mut self.ratio_0_5p, + &mut self.ratio_0_1p, + &mut self.price_99p, + &mut self.price_99_5p, + &mut self.price_99_9p, + &mut self.price_1p, + &mut self.price_0_5p, + &mut self.price_0_1p, + ] + } +} diff --git a/parser/src/datasets/subs/realized.rs b/parser/src/datasets/subs/realized.rs index 529d0e140..68c6ae119 100644 --- a/parser/src/datasets/subs/realized.rs +++ b/parser/src/datasets/subs/realized.rs @@ -3,11 +3,10 @@ use allocative::Allocative; use crate::{ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates}, states::RealizedState, - structs::{AnyBiMap, BiMap}, + structs::{AnyBiMap, BiMap, Price}, utils::ONE_MONTH_IN_DAYS, }; -/// TODO: Fix fees not taken into account ? #[derive(Default, Allocative)] pub struct RealizedSubDataset { min_initial_states: MinInitialStates, @@ -15,6 +14,9 @@ pub struct RealizedSubDataset { // Inserted realized_profit: BiMap, realized_loss: BiMap, + value_destroyed: BiMap, + value_created: BiMap, + spent_output_profit_ratio: BiMap, // Computed negative_realized_loss: BiMap, @@ -35,6 +37,10 @@ impl RealizedSubDataset { realized_profit: BiMap::new_bin(1, &f("realized_profit")), realized_loss: BiMap::new_bin(1, &f("realized_loss")), + value_created: BiMap::new_bin(1, &f("value_created")), + value_destroyed: BiMap::new_bin(1, &f("value_destroyed")), + spent_output_profit_ratio: BiMap::new_bin(2, &f("spent_output_profit_ratio")), + negative_realized_loss: BiMap::new_bin(2, &f("negative_realized_loss")), net_realized_profit_and_loss: BiMap::new_bin(1, &f("net_realized_profit_and_loss")), net_realized_profit_and_loss_to_market_cap_ratio: BiMap::new_bin( @@ -78,12 +84,41 @@ impl RealizedSubDataset { .height .insert(height, height_state.realized_loss.to_dollar() as f32); + self.value_created + .height + .insert(height, height_state.value_created.to_dollar() as f32); + + self.value_destroyed + .height + .insert(height, height_state.value_destroyed.to_dollar() as f32); + + self.spent_output_profit_ratio.height.insert(height, { + if height_state.value_destroyed > Price::ZERO { + (height_state.value_created.to_cent() as f64 + / height_state.value_destroyed.to_cent() as f64) as f32 + } else { + 1.0 + } + }); + if is_date_last_block { self.realized_profit .date_insert_sum_range(date, date_blocks_range); self.realized_loss .date_insert_sum_range(date, date_blocks_range); + + self.value_created + .date_insert_sum_range(date, date_blocks_range); + + self.value_destroyed + .date_insert_sum_range(date, date_blocks_range); + + self.spent_output_profit_ratio.date.insert( + date, + self.value_created.height.sum_range(date_blocks_range) + / self.value_destroyed.height.sum_range(date_blocks_range), + ); } } @@ -145,11 +180,23 @@ impl AnyDataset for RealizedSubDataset { } fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { - vec![&self.realized_loss, &self.realized_profit] + vec![ + &self.realized_loss, + &self.realized_profit, + &self.value_created, + &self.value_destroyed, + &self.spent_output_profit_ratio, + ] } fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { - vec![&mut self.realized_loss, &mut self.realized_profit] + vec![ + &mut self.realized_loss, + &mut self.realized_profit, + &mut self.value_created, + &mut self.value_destroyed, + &mut self.spent_output_profit_ratio, + ] } fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { diff --git a/parser/src/states/cohorts_states/address/cohorts_realized_states.rs b/parser/src/states/cohorts_states/address/cohorts_realized_states.rs index 5478fe62b..80863dd2c 100644 --- a/parser/src/states/cohorts_states/address/cohorts_realized_states.rs +++ b/parser/src/states/cohorts_states/address/cohorts_realized_states.rs @@ -16,28 +16,45 @@ impl AddressCohortsRealizedStates { realized_data: &AddressRealizedData, liquidity_classification: &LiquidityClassification, ) -> color_eyre::Result<()> { - let profit = realized_data.profit; - let loss = realized_data.loss; + let realized_profit = realized_data.profit; + let realized_loss = realized_data.loss; + let value_created = realized_data.value_created; + let value_destroyed = realized_data.value_destroyed; - let split_profit = liquidity_classification.split(profit.to_cent() as f64); - let split_loss = liquidity_classification.split(loss.to_cent() as f64); + let split_realized_profit = + liquidity_classification.split(realized_profit.to_cent() as f64); + let split_realized_loss = liquidity_classification.split(realized_loss.to_cent() as f64); + let split_value_created = liquidity_classification.split(value_created.to_cent() as f64); + let split_value_destroyed = + liquidity_classification.split(value_destroyed.to_cent() as f64); let iterate = move |state: &mut SplitByLiquidity| -> color_eyre::Result<()> { - state.all.iterate(profit, loss); + state.all.iterate( + realized_profit, + realized_loss, + value_created, + value_destroyed, + ); state.illiquid.iterate( - Price::from_cent(split_profit.illiquid as u64), - Price::from_cent(split_loss.illiquid as u64), + Price::from_cent(split_realized_profit.illiquid as u64), + Price::from_cent(split_realized_loss.illiquid as u64), + Price::from_cent(split_value_created.illiquid as u64), + Price::from_cent(split_value_destroyed.illiquid as u64), ); state.liquid.iterate( - Price::from_cent(split_profit.liquid as u64), - Price::from_cent(split_loss.liquid as u64), + Price::from_cent(split_realized_profit.liquid as u64), + Price::from_cent(split_realized_loss.liquid as u64), + Price::from_cent(split_value_created.liquid as u64), + Price::from_cent(split_value_destroyed.liquid as u64), ); state.highly_liquid.iterate( - Price::from_cent(split_profit.highly_liquid as u64), - Price::from_cent(split_loss.highly_liquid as u64), + Price::from_cent(split_realized_profit.highly_liquid as u64), + Price::from_cent(split_realized_loss.highly_liquid as u64), + Price::from_cent(split_value_created.highly_liquid as u64), + Price::from_cent(split_value_destroyed.highly_liquid as u64), ); Ok(()) diff --git a/parser/src/states/cohorts_states/any/realized_state.rs b/parser/src/states/cohorts_states/any/realized_state.rs index 0b03a8683..e8dcdcb9d 100644 --- a/parser/src/states/cohorts_states/any/realized_state.rs +++ b/parser/src/states/cohorts_states/any/realized_state.rs @@ -4,11 +4,21 @@ use crate::structs::Price; pub struct RealizedState { pub realized_profit: Price, pub realized_loss: Price, + pub value_created: Price, + pub value_destroyed: Price, } impl RealizedState { - pub fn iterate(&mut self, realized_profit: Price, realized_loss: Price) { + pub fn iterate( + &mut self, + realized_profit: Price, + realized_loss: Price, + value_created: Price, + value_destroyed: Price, + ) { self.realized_profit += realized_profit; self.realized_loss += realized_loss; + self.value_created += value_created; + self.value_destroyed += value_destroyed; } } diff --git a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs b/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs index 3fd27fc33..1674e51db 100644 --- a/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs +++ b/parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs @@ -52,6 +52,9 @@ impl UTXOCohortsSentStates { let previous_value = previous_price * amount_sent; let current_value = current_price * amount_sent; + state.realized.value_destroyed += previous_value; + state.realized.value_created += current_value; + match previous_value.cmp(¤t_value) { Ordering::Less => { state.realized.realized_profit += current_value - previous_value; diff --git a/parser/src/structs/address_data.rs b/parser/src/structs/address_data.rs index 86cb71ab4..788167dd2 100644 --- a/parser/src/structs/address_data.rs +++ b/parser/src/structs/address_data.rs @@ -43,12 +43,7 @@ impl AddressData { self.realized_cap += received_value; } - pub fn send( - &mut self, - amount: WAmount, - current_price: Price, - sent_amount_price: Price, - ) -> color_eyre::Result { + pub fn send(&mut self, amount: WAmount, previous_price: Price) -> color_eyre::Result<()> { let previous_amount = self.amount; if previous_amount < amount { @@ -63,18 +58,10 @@ impl AddressData { self.outputs_len -= 1; - let previous_sent_dollar_value = sent_amount_price * amount; + let previous_sent_dollar_value = previous_price * amount; self.realized_cap -= previous_sent_dollar_value; - let current_sent_dollar_value = current_price * amount; - - let profit_or_loss = if current_sent_dollar_value >= previous_sent_dollar_value { - ProfitOrLoss::Profit(current_sent_dollar_value - previous_sent_dollar_value) - } else { - ProfitOrLoss::Loss(previous_sent_dollar_value - current_sent_dollar_value) - }; - - Ok(profit_or_loss) + Ok(()) } #[inline(always)] @@ -105,8 +92,3 @@ impl AddressData { LiquidityClassification::new(self.sent, self.received) } } - -pub enum ProfitOrLoss { - Profit(Price), - Loss(Price), -} diff --git a/parser/src/structs/address_realized_data.rs b/parser/src/structs/address_realized_data.rs index a15d204d0..44c348502 100644 --- a/parser/src/structs/address_realized_data.rs +++ b/parser/src/structs/address_realized_data.rs @@ -1,4 +1,4 @@ -use super::{AddressData, Price, ProfitOrLoss, WAmount}; +use super::{AddressData, Price, WAmount}; #[derive(Debug)] pub struct AddressRealizedData { @@ -7,6 +7,8 @@ pub struct AddressRealizedData { pub sent: WAmount, pub profit: Price, pub loss: Price, + pub value_created: Price, + pub value_destroyed: Price, pub utxos_created: u32, pub utxos_destroyed: u32, } @@ -20,6 +22,8 @@ impl AddressRealizedData { loss: Price::ZERO, utxos_created: 0, utxos_destroyed: 0, + value_created: Price::ZERO, + value_destroyed: Price::ZERO, initial_address_data: *initial_address_data, } } @@ -29,18 +33,21 @@ impl AddressRealizedData { self.utxos_created += 1; } - pub fn send(&mut self, amount: WAmount, realized_profit_or_loss: ProfitOrLoss) { + pub fn send(&mut self, amount: WAmount, current_price: Price, previous_price: Price) { self.sent += amount; self.utxos_destroyed += 1; - match realized_profit_or_loss { - ProfitOrLoss::Profit(price) => { - self.profit += price; - } - ProfitOrLoss::Loss(price) => { - self.loss += price; - } + let current_value = current_price * amount; + let previous_value = previous_price * amount; + + self.value_created += current_value; + self.value_destroyed += previous_value; + + if current_value >= previous_value { + self.profit += current_value - previous_value; + } else { + self.loss += previous_value - current_value; } } } diff --git a/parser/src/structs/bi_map.rs b/parser/src/structs/bi_map.rs index 6a1300be2..96b1375c9 100644 --- a/parser/src/structs/bi_map.rs +++ b/parser/src/structs/bi_map.rs @@ -277,20 +277,29 @@ where &mut self, heights: &[usize], dates: &[WNaiveDate], - source: &mut BiMap, - percentile: f32, + mut map_and_percentiles: Vec<(&mut BiMap, f32)>, days: Option, ) where T: FloatCore, { + let mut date_map_and_percentiles = vec![]; + let mut height_map_and_percentiles = vec![]; + + map_and_percentiles + .iter_mut() + .for_each(|(map, percentile)| { + date_map_and_percentiles.push((&mut map.date, *percentile)); + height_map_and_percentiles.push((&mut map.height, *percentile)); + }); + self.height.multi_insert_percentile( heights, - &mut source.height, - percentile, + height_map_and_percentiles, days.map(|days| TARGET_BLOCKS_PER_DAY * days), ); + self.date - .multi_insert_percentile(dates, &mut source.date, percentile, days); + .multi_insert_percentile(dates, date_map_and_percentiles, days); } } diff --git a/parser/src/structs/date_map.rs b/parser/src/structs/date_map.rs index 4c14a3999..137b18fc6 100644 --- a/parser/src/structs/date_map.rs +++ b/parser/src/structs/date_map.rs @@ -423,10 +423,10 @@ where &mut self, dates: &[WNaiveDate], source: &mut DateMap, - transform: F, + mut transform: F, ) where K: MapValue, - F: Fn((K, &WNaiveDate, &mut DateMap)) -> T, + F: FnMut((K, &WNaiveDate, &mut DateMap)) -> T, { dates.iter().for_each(|date| { self.insert( @@ -706,113 +706,129 @@ where ) where T: FloatCore, { - self.multi_insert_percentile(dates, source, 0.5, days); + source.multi_insert_percentile(dates, vec![(self, 0.5)], days); } pub fn multi_insert_percentile( &mut self, dates: &[WNaiveDate], - source: &mut DateMap, - percentile: f32, + mut map_and_percentiles: Vec<(&mut DateMap, f32)>, days: Option, ) where T: FloatCore, { - if !(0.0..=1.0).contains(&percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } - if days.map_or(false, |size| size < 3) { - panic!("Computing a median for a size lower than 3 is useless"); + panic!("Computing a percentile for a size lower than 3 is useless"); } let mut ordered_vec = None; let mut sorted_vec = None; dates.iter().for_each(|date| { - let value = { - if let Some(start) = days - .map_or(chrono::NaiveDate::from_ymd_opt(2009, 3, 1), |size| { - date.checked_sub_days(Days::new(size as u64)) - }) - { - if ordered_vec.is_none() { - let mut vec = start - .iter_days() - .take_while(|d| *d != **date) - .flat_map(|date| source.get_or_import(&WNaiveDate::wrap(date))) - .map(|f| OrderedFloat(f)) - .collect_vec(); + if let Some(start) = days.map_or(chrono::NaiveDate::from_ymd_opt(2009, 1, 3), |size| { + date.checked_sub_days(Days::new(size as u64)) + }) { + if sorted_vec.is_none() { + let mut vec = start + .iter_days() + .take_while(|d| *d != **date) + .flat_map(|date| self.get_or_import(&WNaiveDate::wrap(date))) + .map(|f| OrderedFloat(f)) + .collect_vec(); - if days.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } - - vec.sort_unstable(); - sorted_vec.replace(vec); - } else { - let float_value = OrderedFloat(source.get_or_import(date).unwrap()); - - if let Some(days) = days { - if let Some(ordered_vec) = ordered_vec.as_mut() { - if ordered_vec.len() == days { - let first = ordered_vec.pop_front().unwrap(); - - let pos = - sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); - - sorted_vec.as_mut().unwrap().remove(pos); - } - - ordered_vec.push_back(float_value); - } - } - - let pos = sorted_vec - .as_ref() - .unwrap() - .binary_search(&float_value) - .unwrap_or_else(|pos| pos); - sorted_vec.as_mut().unwrap().insert(pos, float_value); + if days.is_some() { + ordered_vec.replace(VecDeque::from(vec.clone())); } - let vec = sorted_vec.as_ref().unwrap(); - - if vec.is_empty() { - T::default() - } else { - let index = vec.len() as f32 * percentile; - - if index.fract() != 0.0 && vec.len() > 1 { - (vec.get(index.ceil() as usize) - .unwrap_or_else(|| { - dbg!(vec, index, &self.path_all, &source.path_all, days); - panic!() - }) - .0 - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!(vec, index, &self.path_all, &source.path_all, days); - panic!() - }) - .0) - / T::from(2.0).unwrap() - } else { - vec.get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!(vec, index); - panic!(); - }) - .0 - } - } + vec.sort_unstable(); + sorted_vec.replace(vec); } else { - T::default() - } - }; + let float_value = OrderedFloat(self.get_or_import(date).unwrap()); - self.insert(*date, value); + if let Some(days) = days { + if let Some(ordered_vec) = ordered_vec.as_mut() { + if ordered_vec.len() == days { + let first = ordered_vec.pop_front().unwrap(); + + let pos = + sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); + + sorted_vec.as_mut().unwrap().remove(pos); + } + + ordered_vec.push_back(float_value); + } + } + + let pos = sorted_vec + .as_ref() + .unwrap() + .binary_search(&float_value) + .unwrap_or_else(|pos| pos); + + sorted_vec.as_mut().unwrap().insert(pos, float_value); + } + + let vec = sorted_vec.as_ref().unwrap(); + + let len = vec.len(); + + map_and_percentiles + .iter_mut() + .for_each(|(map, percentile)| { + if !(0.0..=1.0).contains(percentile) { + panic!("The percentile should be between 0.0 and 1.0"); + } + + let value = { + if vec.is_empty() { + T::default() + } else { + let index = (len - 1) as f32 * *percentile; + + let fract = index.fract(); + let fract_t = T::from(fract).unwrap(); + + if fract != 0.0 { + (vec.get(index.ceil() as usize) + .unwrap_or_else(|| { + dbg!(vec, index, &self.path_all, &self.path_all, days); + panic!() + }) + .0 + * fract_t + + vec + .get(index.floor() as usize) + .unwrap_or_else(|| { + dbg!( + vec, + index, + &self.path_all, + &self.path_all, + days + ); + panic!() + }) + .0) + * T::from(1.0 - fract).unwrap() + } else { + vec.get(index.floor() as usize) + .unwrap_or_else(|| { + dbg!(vec, index); + panic!(); + }) + .0 + } + } + }; + + (*map).insert(*date, value); + }); + } else { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(*date, T::default()); + }); + } }); } diff --git a/parser/src/structs/height_map.rs b/parser/src/structs/height_map.rs index c67dcf638..6be276f20 100644 --- a/parser/src/structs/height_map.rs +++ b/parser/src/structs/height_map.rs @@ -444,10 +444,10 @@ where &mut self, heights: &[usize], source: &mut HeightMap, - transform: F, + mut transform: F, ) where K: MapValue, - F: Fn((K, &usize)) -> T, + F: FnMut((K, &usize)) -> T, { heights.iter().for_each(|height| { self.insert(*height, transform((source.get_or_import(height), height))); @@ -695,24 +695,19 @@ where ) where T: FloatCore, { - self.multi_insert_percentile(heights, source, 0.5, block_time); + source.multi_insert_percentile(heights, vec![(self, 0.5)], block_time); } pub fn multi_insert_percentile( &mut self, heights: &[usize], - source: &mut HeightMap, - percentile: f32, + mut map_and_percentiles: Vec<(&mut HeightMap, f32)>, block_time: Option, ) where T: FloatCore, { - if !(0.0..=1.0).contains(&percentile) { - panic!("The percentile should be between 0.0 and 1.0"); - } - if block_time.map_or(false, |size| size < 3) { - panic!("Computing a median for a size lower than 3 is useless"); + panic!("Computing a percentile for a size lower than 3 is useless"); } let mut ordered_vec = None; @@ -721,66 +716,100 @@ where heights.iter().for_each(|height| { let height = *height; - let value = { - if let Some(start) = block_time.map_or(Some(0), |size| height.checked_sub(size)) { - if ordered_vec.is_none() { - let mut vec = (start..=height) - .map(|height| OrderedFloat(source.get_or_import(&height))) - .collect_vec(); + if let Some(start) = block_time.map_or(Some(0), |size| height.checked_sub(size)) { + if sorted_vec.is_none() { + let mut vec = (start..=height) + .map(|height| OrderedFloat(self.get_or_import(&height))) + .collect_vec(); - if block_time.is_some() { - ordered_vec.replace(VecDeque::from(vec.clone())); - } - - vec.sort_unstable(); - sorted_vec.replace(vec); - } else { - let float_value = OrderedFloat(source.get_or_import(&height)); - - if block_time.is_some() { - let first = ordered_vec.as_mut().unwrap().pop_front().unwrap(); - let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); - sorted_vec.as_mut().unwrap().remove(pos); - - ordered_vec.as_mut().unwrap().push_back(float_value); - } - - let pos = sorted_vec - .as_ref() - .unwrap() - .binary_search(&float_value) - .unwrap_or_else(|pos| pos); - sorted_vec.as_mut().unwrap().insert(pos, float_value); + if block_time.is_some() { + ordered_vec.replace(VecDeque::from(vec.clone())); } - let vec = sorted_vec.as_ref().unwrap(); + vec.sort_unstable(); - let index = vec.len() as f32 * percentile; - - if index.fract() != 0.0 { - (vec.get(index.ceil() as usize) - .unwrap_or_else(|| { - dbg!(index, &self.path_all, &source.path_all, block_time); - panic!() - }) - .0 - + vec - .get(index.floor() as usize) - .unwrap_or_else(|| { - dbg!(index, &self.path_all, &source.path_all, block_time); - panic!() - }) - .0) - / T::from(2.0).unwrap() - } else { - vec.get(index as usize).unwrap().0 - } + sorted_vec.replace(vec); } else { - T::default() - } - }; + let float_value = OrderedFloat(self.get_or_import(&height)); - self.insert(height, value); + if block_time.is_some() { + let first = ordered_vec.as_mut().unwrap().pop_front().unwrap(); + let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap(); + sorted_vec.as_mut().unwrap().remove(pos); + + ordered_vec.as_mut().unwrap().push_back(float_value); + } + + let pos = sorted_vec + .as_ref() + .unwrap() + .binary_search(&float_value) + .unwrap_or_else(|pos| pos); + + sorted_vec.as_mut().unwrap().insert(pos, float_value); + } + + let vec = sorted_vec.as_ref().unwrap(); + + let len = vec.len(); + + map_and_percentiles + .iter_mut() + .for_each(|(map, percentile)| { + if !(0.0..=1.0).contains(percentile) { + panic!("The percentile should be between 0.0 and 1.0"); + } + + let value = { + if vec.is_empty() { + T::default() + } else { + let index = (len - 1) as f32 * *percentile; + + let fract = index.fract(); + let fract_t = T::from(fract).unwrap(); + + if fract != 0.0 { + (vec.get(index.ceil() as usize) + .unwrap_or_else(|| { + dbg!( + index, + &self.path_all, + &self.path_all, + &self.to_insert, + block_time, + vec + ); + panic!() + }) + .0 + * fract_t + + vec + .get(index.floor() as usize) + .unwrap_or_else(|| { + dbg!( + index, + &self.path_all, + &self.path_all, + block_time + ); + panic!() + }) + .0) + * T::from(1.0 - fract).unwrap() + } else { + vec.get(index as usize).unwrap().0 + } + } + }; + + (*map).insert(height, value); + }); + } else { + map_and_percentiles.iter_mut().for_each(|(map, _)| { + (*map).insert(height, T::default()); + }); + } }); }