diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c217d1f..dfc044c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,12 +27,12 @@ ## Website -- Added dashboards -- Added a Home option -- Added a library of PDFs viewable inside the app +- Added a dashboard +- Added a library of PDFs - Fixed service worker not passing 304 (not modified) response and instead serving cached responses - Fixed history not being properly registered - Fixed window being moveable on iOS when in standalone mode when it shouldn't be +- Split `index.html` and `script.js` into multiple js and css files to load only what's necessary at a given time ## Parser @@ -40,13 +40,16 @@ - Added `--rpcconnect` parameter to the config - Added handling of SIGINT and SIGTERM terminal signals which menas you can now safely CTRL+C or kill the parser while it's exporting - Added config print at the start of the program -- Compressed `empty_address_data` struct to save space (should shave of between 25 and 50% of the `address_index_to_empty_address_data` database) +- Compressed `empty_address_data` struct to save space (should shave of between up to 50% of the `address_index_to_empty_address_data` database) +- Doubled the number of `txid_to_tx_data` databases from 4096 to 8192 ## Server - Fixed links in several places missing the `/api` part and thus not working - Fixed broken last values routes -- Add support for the `/datasets/last` file via the `/api/last` route +- Added support for the `/datasets/last` file via the `/api/last` route +- Added support for `.json` (won't change anything) and `.csv` (will download a csv file) extension at the end of datasets routes +- Added `all=true` query parameter to dataset routes to get to full history ## Biter diff --git a/parser/src/databases/txid_to_tx_data.rs b/parser/src/databases/txid_to_tx_data.rs index 696516a22..43c611815 100644 --- a/parser/src/databases/txid_to_tx_data.rs +++ b/parser/src/databases/txid_to_tx_data.rs @@ -114,7 +114,7 @@ impl TxidToTxData { } fn db_index(txid: &Txid) -> u16 { - ((txid[0] as u16) << 4) + ((txid[1] as u16) >> 4) + ((txid[0] as u16) << 5) + ((txid[1] as u16) >> 3) } } diff --git a/parser/src/datasets/coindays.rs b/parser/src/datasets/coindays.rs index ec163a79c..1cce5a07d 100644 --- a/parser/src/datasets/coindays.rs +++ b/parser/src/datasets/coindays.rs @@ -2,7 +2,8 @@ use allocative::Allocative; use crate::{ datasets::AnyDataset, - structs::{AnyBiMap, BiMap}, + structs::{AnyDateMap, AnyHeightMap}, + DateMap, HeightMap, }; use super::{InsertData, MinInitialStates}; @@ -12,7 +13,8 @@ pub struct CoindaysDataset { min_initial_states: MinInitialStates, // Inserted - pub coindays_destroyed: BiMap, + pub coindays_destroyed: HeightMap, + pub coindays_destroyed_1d_sum: DateMap, } impl CoindaysDataset { @@ -22,7 +24,8 @@ impl CoindaysDataset { let mut s = Self { min_initial_states: MinInitialStates::default(), - coindays_destroyed: BiMap::new_bin(1, &f("coindays_destroyed")), + coindays_destroyed: HeightMap::new_bin(1, &f("coindays_destroyed")), + coindays_destroyed_1d_sum: DateMap::new_bin(1, &f("coindays_destroyed_1d_sum")), }; s.min_initial_states @@ -43,25 +46,32 @@ impl CoindaysDataset { }: &InsertData, ) { self.coindays_destroyed - .height .insert(height, satdays_destroyed.to_btc() as f32); if is_date_last_block { - self.coindays_destroyed - .date_insert_sum_range(date, date_blocks_range) + self.coindays_destroyed_1d_sum + .insert(date, self.coindays_destroyed.sum_range(date_blocks_range)); } } } impl AnyDataset for CoindaysDataset { - fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { + fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { vec![&self.coindays_destroyed] } - fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![&self.coindays_destroyed_1d_sum] + } + + fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![&mut self.coindays_destroyed] } + fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![&mut self.coindays_destroyed_1d_sum] + } + fn get_min_initial_states(&self) -> &MinInitialStates { &self.min_initial_states } diff --git a/parser/src/datasets/cointime.rs b/parser/src/datasets/cointime.rs index 6224ee36e..cdeaa4b87 100644 --- a/parser/src/datasets/cointime.rs +++ b/parser/src/datasets/cointime.rs @@ -1,8 +1,9 @@ use allocative::Allocative; use crate::{ - structs::{AnyBiMap, BiMap, DateMap, Height}, + structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, DateMap, Height}, utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS}, + HeightMap, }; use super::{AnyDataset, ComputeData, InsertData, MinInitialStates, RatioDataset}; @@ -12,7 +13,8 @@ pub struct CointimeDataset { min_initial_states: MinInitialStates, // Inserted - pub coinblocks_destroyed: BiMap, + pub coinblocks_destroyed: HeightMap, + pub coinblocks_destroyed_1d_sum: DateMap, // Computed pub active_cap: BiMap, @@ -22,18 +24,24 @@ pub struct CointimeDataset { pub active_supply_3m_net_change: BiMap, pub active_supply_net_change: BiMap, pub activity_to_vaultedness_ratio: BiMap, - pub coinblocks_created: BiMap, - pub coinblocks_stored: BiMap, - pub cointime_adjusted_velocity: BiMap, - pub cointime_adjusted_yearly_inflation_rate: BiMap, + pub coinblocks_created: HeightMap, + pub coinblocks_created_1d_sum: DateMap, + pub coinblocks_stored: HeightMap, + pub coinblocks_stored_1d_sum: DateMap, + pub cointime_adjusted_velocity: DateMap, + pub cointime_adjusted_inflation_rate: DateMap, + pub cointime_adjusted_yearly_inflation_rate: DateMap, pub cointime_cap: BiMap, pub cointime_price: BiMap, pub cointime_price_ratio: RatioDataset, - pub cointime_value_created: BiMap, - pub cointime_value_destroyed: BiMap, - pub cointime_value_stored: BiMap, - pub concurrent_liveliness: BiMap, - pub concurrent_liveliness_2w_median: BiMap, + pub cointime_value_created: HeightMap, + pub cointime_value_created_1d_sum: DateMap, + pub cointime_value_destroyed: HeightMap, + pub cointime_value_destroyed_1d_sum: DateMap, + pub cointime_value_stored: HeightMap, + pub cointime_value_stored_1d_sum: DateMap, + pub concurrent_liveliness: DateMap, + pub concurrent_liveliness_2w_median: DateMap, pub cumulative_coinblocks_created: BiMap, pub cumulative_coinblocks_destroyed: BiMap, pub cumulative_coinblocks_stored: BiMap, @@ -76,22 +84,35 @@ impl CointimeDataset { active_supply_3m_net_change: BiMap::new_bin(1, &f("active_supply_3m_net_change")), active_supply_net_change: BiMap::new_bin(1, &f("active_supply_net_change")), activity_to_vaultedness_ratio: BiMap::new_bin(2, &f("activity_to_vaultedness_ratio")), - coinblocks_created: BiMap::new_bin(1, &f("coinblocks_created")), - coinblocks_destroyed: BiMap::new_bin(1, &f("coinblocks_destroyed")), - coinblocks_stored: BiMap::new_bin(1, &f("coinblocks_stored")), - cointime_adjusted_velocity: BiMap::new_bin(1, &f("cointime_adjusted_velocity")), - cointime_adjusted_yearly_inflation_rate: BiMap::new_bin( + coinblocks_created: HeightMap::new_bin(1, &f("coinblocks_created")), + coinblocks_created_1d_sum: DateMap::new_bin(1, &f("coinblocks_created_1d_sum")), + coinblocks_destroyed: HeightMap::new_bin(1, &f("coinblocks_destroyed")), + coinblocks_destroyed_1d_sum: DateMap::new_bin(1, &f("coinblocks_destroyed_1d_sum")), + coinblocks_stored: HeightMap::new_bin(1, &f("coinblocks_stored")), + coinblocks_stored_1d_sum: DateMap::new_bin(1, &f("coinblocks_stored_1d_sum")), + cointime_adjusted_velocity: DateMap::new_bin(1, &f("cointime_adjusted_velocity")), + cointime_adjusted_inflation_rate: DateMap::new_bin( + 1, + &f("cointime_adjusted_inflation_rate"), + ), + cointime_adjusted_yearly_inflation_rate: DateMap::new_bin( 1, &f("cointime_adjusted_yearly_inflation_rate"), ), cointime_cap: BiMap::new_bin(1, &f("cointime_cap")), cointime_price: BiMap::new_bin(1, &f("cointime_price")), cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price")?, - cointime_value_created: BiMap::new_bin(1, &f("cointime_value_created")), - cointime_value_destroyed: BiMap::new_bin(1, &f("cointime_value_destroyed")), - cointime_value_stored: BiMap::new_bin(1, &f("cointime_value_stored")), - concurrent_liveliness: BiMap::new_bin(1, &f("concurrent_liveliness")), - concurrent_liveliness_2w_median: BiMap::new_bin( + cointime_value_created: HeightMap::new_bin(1, &f("cointime_value_created")), + cointime_value_created_1d_sum: DateMap::new_bin(1, &f("cointime_value_created_1d_sum")), + cointime_value_destroyed: HeightMap::new_bin(1, &f("cointime_value_destroyed")), + cointime_value_destroyed_1d_sum: DateMap::new_bin( + 1, + &f("cointime_value_destroyed_1d_sum"), + ), + cointime_value_stored: HeightMap::new_bin(1, &f("cointime_value_stored")), + cointime_value_stored_1d_sum: DateMap::new_bin(1, &f("cointime_value_stored_1d_sum")), + concurrent_liveliness: DateMap::new_bin(1, &f("concurrent_liveliness")), + concurrent_liveliness_2w_median: DateMap::new_bin( 2, &f("concurrent_liveliness_2w_median"), ), @@ -153,12 +174,11 @@ impl CointimeDataset { }: &InsertData, ) { self.coinblocks_destroyed - .height .insert(height, satblocks_destroyed.to_btc() as f32); if is_date_last_block { - self.coinblocks_destroyed - .date_insert_sum_range(date, date_blocks_range); + self.coinblocks_destroyed_1d_sum + .insert(date, self.coinblocks_destroyed.sum_range(date_blocks_range)); } } @@ -172,45 +192,58 @@ impl CointimeDataset { circulating_supply: &mut BiMap, realized_cap: &mut BiMap, realized_price: &mut BiMap, - yearly_inflation_rate: &mut BiMap, - annualized_transaction_volume: &mut BiMap, + inflation_rate: &mut DateMap, + yearly_inflation_rate: &mut DateMap, + annualized_transaction_volume: &mut DateMap, cumulative_subsidy_in_dollars: &mut BiMap, ) { let &ComputeData { heights, dates, .. } = compute_data; self.cumulative_coinblocks_destroyed - .multi_insert_cumulative(heights, dates, &mut self.coinblocks_destroyed); - - self.coinblocks_created .height - .multi_insert_simple_transform( - heights, - &mut circulating_supply.height, - |circulating_supply, _| circulating_supply as f32, - ); - self.coinblocks_created - .multi_date_insert_sum_range(dates, first_height, last_height); + .multi_insert_cumulative(heights, &mut self.coinblocks_destroyed); + self.cumulative_coinblocks_destroyed + .date + .multi_insert_cumulative(dates, &mut self.coinblocks_destroyed_1d_sum); - self.cumulative_coinblocks_created.multi_insert_cumulative( + self.coinblocks_created.multi_insert_simple_transform( heights, + &mut circulating_supply.height, + |circulating_supply, _| circulating_supply as f32, + ); + self.coinblocks_created_1d_sum.multi_insert_sum_range( dates, + &self.coinblocks_created, + first_height, + last_height, + ); + + self.cumulative_coinblocks_created + .height + .multi_insert_cumulative(heights, &mut self.coinblocks_created); + self.cumulative_coinblocks_created + .date + .multi_insert_cumulative(dates, &mut self.coinblocks_created_1d_sum); + + self.coinblocks_stored.multi_insert_subtract( + heights, &mut self.coinblocks_created, + &mut self.coinblocks_destroyed, ); - - self.coinblocks_stored.height.multi_insert_subtract( - heights, - &mut self.coinblocks_created.height, - &mut self.coinblocks_destroyed.height, - ); - self.coinblocks_stored - .multi_date_insert_sum_range(dates, first_height, last_height); - - self.cumulative_coinblocks_stored.multi_insert_cumulative( - heights, + self.coinblocks_stored_1d_sum.multi_insert_sum_range( dates, - &mut self.coinblocks_stored, + &self.coinblocks_stored, + first_height, + last_height, ); + self.cumulative_coinblocks_stored + .height + .multi_insert_cumulative(heights, &mut self.coinblocks_stored); + self.cumulative_coinblocks_stored + .date + .multi_insert_cumulative(dates, &mut self.coinblocks_stored_1d_sum); + self.liveliness.multi_insert_divide( heights, dates, @@ -233,14 +266,12 @@ impl CointimeDataset { ); self.concurrent_liveliness.multi_insert_divide( - heights, dates, - &mut self.coinblocks_destroyed, - &mut self.coinblocks_created, + &mut self.coinblocks_destroyed_1d_sum, + &mut self.coinblocks_created_1d_sum, ); self.concurrent_liveliness_2w_median.multi_insert_median( - heights, dates, &mut self.concurrent_liveliness, Some(TWO_WEEK_IN_DAYS), @@ -309,19 +340,23 @@ impl CointimeDataset { // let min_vaulted_supply = ; // let max_active_supply = ; + self.cointime_adjusted_inflation_rate.multi_insert_multiply( + dates, + &mut self.activity_to_vaultedness_ratio.date, + inflation_rate, + ); + self.cointime_adjusted_yearly_inflation_rate .multi_insert_multiply( - heights, dates, - &mut self.activity_to_vaultedness_ratio, + &mut self.activity_to_vaultedness_ratio.date, yearly_inflation_rate, ); self.cointime_adjusted_velocity.multi_insert_divide( - heights, dates, annualized_transaction_volume, - &mut self.active_supply, + &mut self.active_supply.date, ); // TODO: @@ -426,56 +461,59 @@ impl CointimeDataset { self.producerness .multi_insert_divide(heights, dates, &mut self.thermo_cap, realized_cap); - self.cointime_value_destroyed.height.multi_insert_multiply( + self.cointime_value_destroyed.multi_insert_multiply( heights, - &mut self.coinblocks_destroyed.height, + &mut self.coinblocks_destroyed, &mut closes.height, ); - self.cointime_value_destroyed.date.multi_insert_multiply( + self.cointime_value_destroyed_1d_sum.multi_insert_multiply( dates, - &mut self.coinblocks_destroyed.date, + &mut self.coinblocks_destroyed_1d_sum, &mut closes.date, ); - self.cointime_value_created.height.multi_insert_multiply( + self.cointime_value_created.multi_insert_multiply( heights, - &mut self.coinblocks_created.height, + &mut self.coinblocks_created, &mut closes.height, ); - self.cointime_value_created.date.multi_insert_multiply( + self.cointime_value_created_1d_sum.multi_insert_multiply( dates, - &mut self.coinblocks_created.date, + &mut self.coinblocks_created_1d_sum, &mut closes.date, ); - self.cointime_value_stored.height.multi_insert_multiply( + self.cointime_value_stored.multi_insert_multiply( heights, - &mut self.coinblocks_stored.height, + &mut self.coinblocks_stored, &mut closes.height, ); - self.cointime_value_stored.date.multi_insert_multiply( + self.cointime_value_stored_1d_sum.multi_insert_multiply( dates, - &mut self.coinblocks_stored.date, + &mut self.coinblocks_stored_1d_sum, &mut closes.date, ); - self.total_cointime_value_created.multi_insert_cumulative( - heights, - dates, - &mut self.cointime_value_created, - ); + self.total_cointime_value_created + .height + .multi_insert_cumulative(heights, &mut self.cointime_value_created); + self.total_cointime_value_created + .date + .multi_insert_cumulative(dates, &mut self.cointime_value_created_1d_sum); - self.total_cointime_value_destroyed.multi_insert_cumulative( - heights, - dates, - &mut self.cointime_value_destroyed, - ); + self.total_cointime_value_destroyed + .height + .multi_insert_cumulative(heights, &mut self.cointime_value_destroyed); + self.total_cointime_value_destroyed + .date + .multi_insert_cumulative(dates, &mut self.cointime_value_destroyed_1d_sum); - self.total_cointime_value_stored.multi_insert_cumulative( - heights, - dates, - &mut self.cointime_value_stored, - ); + self.total_cointime_value_stored + .height + .multi_insert_cumulative(heights, &mut self.cointime_value_stored); + self.total_cointime_value_stored + .date + .multi_insert_cumulative(dates, &mut self.cointime_value_stored_1d_sum); self.cointime_price.multi_insert_divide( heights, @@ -506,14 +544,72 @@ impl CointimeDataset { } impl AnyDataset for CointimeDataset { - fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { + fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { vec![&self.coinblocks_destroyed] } - fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![&mut self.coinblocks_destroyed] } + fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![&self.coinblocks_destroyed_1d_sum] + } + + fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![&mut self.coinblocks_destroyed_1d_sum] + } + + fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![ + &self.coinblocks_created, + &self.coinblocks_stored, + &self.cointime_value_created, + &self.cointime_value_destroyed, + &self.cointime_value_stored, + ] + } + + fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { + vec![ + &mut self.coinblocks_created, + &mut self.coinblocks_stored, + &mut self.cointime_value_created, + &mut self.cointime_value_destroyed, + &mut self.cointime_value_stored, + ] + } + + fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![ + &self.coinblocks_created_1d_sum, + &self.coinblocks_stored_1d_sum, + &self.concurrent_liveliness, + &self.concurrent_liveliness_2w_median, + &self.cointime_adjusted_velocity, + &self.cointime_value_created_1d_sum, + &self.cointime_value_destroyed_1d_sum, + &self.cointime_value_stored_1d_sum, + &self.cointime_adjusted_inflation_rate, + &self.cointime_adjusted_yearly_inflation_rate, + ] + } + + fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![ + &mut self.coinblocks_created_1d_sum, + &mut self.coinblocks_stored_1d_sum, + &mut self.concurrent_liveliness, + &mut self.concurrent_liveliness_2w_median, + &mut self.cointime_adjusted_velocity, + &mut self.cointime_value_created_1d_sum, + &mut self.cointime_value_destroyed_1d_sum, + &mut self.cointime_value_stored_1d_sum, + &mut self.cointime_adjusted_inflation_rate, + &mut self.cointime_adjusted_yearly_inflation_rate, + ] + } + fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { let mut v = vec![ &self.active_cap as &(dyn AnyBiMap + Send + Sync), @@ -522,17 +618,8 @@ impl AnyDataset for CointimeDataset { &self.active_supply_3m_net_change, &self.active_supply_net_change, &self.activity_to_vaultedness_ratio, - &self.coinblocks_created, - &self.coinblocks_stored, - &self.cointime_adjusted_velocity, - &self.cointime_adjusted_yearly_inflation_rate, &self.cointime_cap, &self.cointime_price, - &self.cointime_value_created, - &self.cointime_value_destroyed, - &self.cointime_value_stored, - &self.concurrent_liveliness, - &self.concurrent_liveliness_2w_median, &self.cumulative_coinblocks_created, &self.cumulative_coinblocks_destroyed, &self.cumulative_coinblocks_stored, @@ -575,17 +662,8 @@ impl AnyDataset for CointimeDataset { &mut self.active_supply_3m_net_change, &mut self.active_supply_net_change, &mut self.activity_to_vaultedness_ratio, - &mut self.coinblocks_created, - &mut self.coinblocks_stored, - &mut self.cointime_adjusted_velocity, - &mut self.cointime_adjusted_yearly_inflation_rate, &mut self.cointime_cap, &mut self.cointime_price, - &mut self.cointime_value_created, - &mut self.cointime_value_destroyed, - &mut self.cointime_value_stored, - &mut self.concurrent_liveliness, - &mut self.concurrent_liveliness_2w_median, &mut self.cumulative_coinblocks_created, &mut self.cumulative_coinblocks_destroyed, &mut self.cumulative_coinblocks_stored, diff --git a/parser/src/datasets/mining.rs b/parser/src/datasets/mining.rs index e0f96c2b2..6452cc7ee 100644 --- a/parser/src/datasets/mining.rs +++ b/parser/src/datasets/mining.rs @@ -25,10 +25,14 @@ pub struct MiningDataset { // Inserted pub blocks_mined: DateMap, pub total_blocks_mined: DateMap, - pub coinbase: BiMap, - pub coinbase_in_dollars: BiMap, - pub fees: BiMap, - pub fees_in_dollars: BiMap, + pub coinbase: HeightMap, + pub coinbase_1d_sum: DateMap, + pub coinbase_in_dollars: HeightMap, + pub coinbase_in_dollars_1d_sum: DateMap, + pub fees: HeightMap, + pub fees_1d_sum: DateMap, + pub fees_in_dollars: HeightMap, + pub fees_in_dollars_1d_sum: DateMap, // Raw // pub average_fee_paid: BiMap, // pub max_fee_paid: BiMap, @@ -48,8 +52,10 @@ pub struct MiningDataset { // pub _10th_percentile_fee_price: BiMap, // pub min_fee_price: BiMap, // - - pub subsidy: BiMap, - pub subsidy_in_dollars: BiMap, + pub subsidy: HeightMap, + pub subsidy_1d_sum: DateMap, + pub subsidy_in_dollars: HeightMap, + pub subsidy_in_dollars_1d_sum: DateMap, pub last_coinbase: DateMap, pub last_coinbase_in_dollars: DateMap, pub last_fees: DateMap, @@ -57,17 +63,13 @@ pub struct MiningDataset { pub last_subsidy: DateMap, pub last_subsidy_in_dollars: DateMap, pub difficulty: BiMap, - pub block_size: HeightMap, // in MB - pub block_size_recap: DateRecapDataset, // in MB - pub block_weight: HeightMap, // in MB - pub block_weight_recap: DateRecapDataset, // in MB + pub block_size: HeightMap, // in MB + pub block_weight: HeightMap, // in MB pub block_vbytes: HeightMap, - pub block_vbytes_recap: DateRecapDataset, - pub block_interval: HeightMap, // in s - pub block_interval_recap: DateRecapDataset, // in s + pub block_interval: HeightMap, // in s // Computed - pub annualized_issuance: BiMap, // Same as subsidy_1y_sum + pub annualized_issuance: DateMap, // Same as subsidy_1y_sum pub blocks_mined_1d_target: DateMap, pub blocks_mined_1m_sma: DateMap, pub blocks_mined_1m_sum: DateMap, @@ -84,38 +86,30 @@ pub struct MiningDataset { pub cumulative_subsidy_in_dollars: BiMap, pub coinbase_1y_sum: DateMap, pub coinbase_in_dollars_1y_sum: DateMap, - pub coinbase_in_dollars_1y_sma: DateMap, + pub coinbase_in_dollars_1d_sum_1y_sma: DateMap, pub cumulative_coinbase: BiMap, pub cumulative_coinbase_in_dollars: BiMap, pub fees_1y_sum: DateMap, pub fees_in_dollars_1y_sum: DateMap, pub cumulative_fees: BiMap, pub cumulative_fees_in_dollars: BiMap, - pub yearly_inflation_rate: BiMap, - pub subsidy_to_coinbase_ratio: BiMap, - pub fees_to_coinbase_ratio: BiMap, + pub inflation_rate: DateMap, + pub yearly_inflation_rate: DateMap, + pub subsidy_to_coinbase_ratio: HeightMap, + pub subsidy_to_coinbase_1d_ratio: DateMap, + pub fees_to_coinbase_ratio: HeightMap, + pub fees_to_coinbase_1d_ratio: DateMap, pub hash_rate: DateMap, pub hash_rate_1w_sma: DateMap, pub hash_rate_1m_sma: DateMap, pub hash_rate_2m_sma: DateMap, pub hash_price: DateMap, pub difficulty_adjustment: DateMap, + pub block_size_recap: DateRecapDataset, // in MB + pub block_weight_recap: DateRecapDataset, // in MB + pub block_vbytes_recap: DateRecapDataset, + pub block_interval_recap: DateRecapDataset, // in s pub puell_multiple: DateMap, - // pub block_size_1d_sma: DateMap, // in MB - // pub block_weight_1d_sma: DateMap, // in MB - // pub block_vbytes_1d_sma: DateMap, - // pub block_interval_1d_sma: DateMap, // in s - // pub block_size_1d_median: DateMap, // in MB - // pub block_weight_1d_median: DateMap, // in MB - // pub block_vbytes_1d_median: DateMap, - // pub block_interval_1d_median: DateMap, - // pub blocks_size: DateMap, - // pub average_block_size: DateMap, - // pub median_block_size: DateMap, - // pub average_block_weight: DateMap, - // pub median_block_weight: DateMap, - // pub average_block_interval: DateMap, - // pub median_block_interval: DateMap, // pub hash_price_in_dollars: DateMap, // pub hash_price_30d_volatility: BiMap, // difficulty_adjustment @@ -136,31 +130,43 @@ impl MiningDataset { total_blocks_mined: DateMap::new_bin(1, &f("total_blocks_mined")), blocks_mined: DateMap::new_bin(1, &f("blocks_mined")), - coinbase: BiMap::new_bin(1, &f("coinbase")), - coinbase_in_dollars: BiMap::new_bin(1, &f("coinbase_in_dollars")), + coinbase: HeightMap::new_bin(1, &f("coinbase")), + coinbase_1d_sum: DateMap::new_bin(1, &f("coinbase_1d_sum")), + coinbase_in_dollars: HeightMap::new_bin(1, &f("coinbase_in_dollars")), + coinbase_in_dollars_1d_sum: DateMap::new_bin(1, &f("coinbase_in_dollars_1d_sum")), coinbase_1y_sum: DateMap::new_bin(1, &f("coinbase_1y_sum")), coinbase_in_dollars_1y_sum: DateMap::new_bin(1, &f("coinbase_in_dollars_1y_sum")), - coinbase_in_dollars_1y_sma: DateMap::new_bin(1, &f("coinbase_in_dollars_1y_sma")), + coinbase_in_dollars_1d_sum_1y_sma: DateMap::new_bin( + 1, + &f("coinbase_in_dollars_1d_sum_1y_sma"), + ), cumulative_coinbase: BiMap::new_bin(1, &f("cumulative_coinbase")), cumulative_coinbase_in_dollars: BiMap::new_bin(1, &f("cumulative_coinbase_in_dollars")), - fees: BiMap::new_bin(1, &f("fees")), - fees_in_dollars: BiMap::new_bin(1, &f("fees_in_dollars")), + fees: HeightMap::new_bin(1, &f("fees")), + fees_1d_sum: DateMap::new_bin(1, &f("fees_1d_sum")), + fees_in_dollars: HeightMap::new_bin(1, &f("fees_in_dollars")), + fees_in_dollars_1d_sum: DateMap::new_bin(1, &f("fees_in_dollars_1d_sum")), fees_1y_sum: DateMap::new_bin(1, &f("fees_1y_sum")), fees_in_dollars_1y_sum: DateMap::new_bin(1, &f("fees_in_dollars_1y_sum")), cumulative_fees: BiMap::new_bin(1, &f("cumulative_fees")), cumulative_fees_in_dollars: BiMap::new_bin(1, &f("cumulative_fees_in_dollars")), - subsidy: BiMap::new_bin(1, &f("subsidy")), - subsidy_in_dollars: BiMap::new_bin(1, &f("subsidy_in_dollars")), + subsidy: HeightMap::new_bin(1, &f("subsidy")), + subsidy_1d_sum: DateMap::new_bin(1, &f("subsidy_1d_sum")), + subsidy_in_dollars: HeightMap::new_bin(1, &f("subsidy_in_dollars")), + subsidy_in_dollars_1d_sum: DateMap::new_bin(1, &f("subsidy_in_dollars_1d_sum")), subsidy_1y_sum: DateMap::new_bin(1, &f("subsidy_1y_sum")), subsidy_in_dollars_1y_sum: DateMap::new_bin(1, &f("subsidy_in_dollars_1y_sum")), cumulative_subsidy: BiMap::new_bin(1, &f("cumulative_subsidy")), cumulative_subsidy_in_dollars: BiMap::new_bin(1, &f("cumulative_subsidy_in_dollars")), - subsidy_to_coinbase_ratio: BiMap::new_bin(1, &f("subsidy_to_coinbase_ratio")), - fees_to_coinbase_ratio: BiMap::new_bin(1, &f("fees_to_coinbase_ratio")), + subsidy_to_coinbase_ratio: HeightMap::new_bin(1, &f("subsidy_to_coinbase_ratio")), + subsidy_to_coinbase_1d_ratio: DateMap::new_bin(1, &f("subsidy_to_coinbase_1d_ratio")), + fees_to_coinbase_ratio: HeightMap::new_bin(1, &f("fees_to_coinbase_ratio")), + fees_to_coinbase_1d_ratio: DateMap::new_bin(1, &f("fees_to_coinbase_1d_ratio")), - annualized_issuance: BiMap::new_bin(1, &f("annualized_issuance")), - yearly_inflation_rate: BiMap::new_bin(1, &f("yearly_inflation_rate")), + annualized_issuance: DateMap::new_bin(1, &f("annualized_issuance")), + inflation_rate: DateMap::new_bin(2, &f("inflation_rate")), + yearly_inflation_rate: DateMap::new_bin(1, &f("yearly_inflation_rate")), last_subsidy: DateMap::new_bin(1, &f("last_subsidy")), last_subsidy_in_dollars: DateMap::new_bin(1, &f("last_subsidy_in_dollars")), @@ -271,28 +277,25 @@ impl MiningDataset { .. }: &InsertData, ) { - self.coinbase.height.insert(height, coinbase.to_btc()); + self.coinbase.insert(height, coinbase.to_btc()); let coinbase_in_dollars = self .coinbase_in_dollars - .height .insert(height, (block_price * coinbase).to_dollar() as f32); let sumed_fees = Amount::from_sat(fees.iter().map(|amount| amount.to_sat()).sum()); - self.fees.height.insert(height, sumed_fees.to_btc()); + self.fees.insert(height, sumed_fees.to_btc()); let sumed_fees_in_dollars = self .fees_in_dollars - .height .insert(height, (block_price * sumed_fees).to_dollar() as f32); let subsidy = coinbase - sumed_fees; - self.subsidy.height.insert(height, subsidy.to_btc()); + self.subsidy.insert(height, subsidy.to_btc()); let subsidy_in_dollars = self .subsidy_in_dollars - .height .insert(height, (block_price * subsidy).to_dollar() as f32); self.difficulty.height.insert(height, difficulty); @@ -305,20 +308,23 @@ impl MiningDataset { self.block_interval.insert(height, *block_interval); if is_date_last_block { - self.coinbase.date_insert_sum_range(date, date_blocks_range); + self.coinbase_1d_sum + .insert(date, self.coinbase.sum_range(date_blocks_range)); - self.coinbase_in_dollars - .date_insert_sum_range(date, date_blocks_range); + self.coinbase_in_dollars_1d_sum + .insert(date, self.coinbase_in_dollars.sum_range(date_blocks_range)); - self.fees.date_insert_sum_range(date, date_blocks_range); + self.fees_1d_sum + .insert(date, self.fees.sum_range(date_blocks_range)); - self.fees_in_dollars - .date_insert_sum_range(date, date_blocks_range); + self.fees_in_dollars_1d_sum + .insert(date, self.fees_in_dollars.sum_range(date_blocks_range)); - self.subsidy.date_insert_sum_range(date, date_blocks_range); + self.subsidy_1d_sum + .insert(date, self.subsidy.sum_range(date_blocks_range)); - self.subsidy_in_dollars - .date_insert_sum_range(date, date_blocks_range); + self.subsidy_in_dollars_1d_sum + .insert(date, self.subsidy_in_dollars.sum_range(date_blocks_range)); self.last_coinbase.insert(date, coinbase.to_btc()); @@ -370,96 +376,128 @@ impl MiningDataset { self.subsidy_1y_sum.multi_insert_last_x_sum( dates, - &mut self.subsidy.date, + &mut self.subsidy_1d_sum, ONE_YEAR_IN_DAYS, ); self.subsidy_in_dollars_1y_sum.multi_insert_last_x_sum( dates, - &mut self.subsidy_in_dollars.date, + &mut self.subsidy_in_dollars_1d_sum, ONE_YEAR_IN_DAYS, ); self.cumulative_subsidy - .multi_insert_cumulative(heights, dates, &mut self.subsidy); + .height + .multi_insert_cumulative(heights, &mut self.subsidy); + self.cumulative_subsidy + .date + .multi_insert_cumulative(dates, &mut self.subsidy_1d_sum); - self.cumulative_subsidy_in_dollars.multi_insert_cumulative( - heights, - dates, - &mut self.subsidy_in_dollars, - ); + self.cumulative_subsidy_in_dollars + .height + .multi_insert_cumulative(heights, &mut self.subsidy_in_dollars); + self.cumulative_subsidy_in_dollars + .date + .multi_insert_cumulative(dates, &mut self.subsidy_in_dollars_1d_sum); self.fees_1y_sum - .multi_insert_last_x_sum(dates, &mut self.fees.date, ONE_YEAR_IN_DAYS); + .multi_insert_last_x_sum(dates, &mut self.fees_1d_sum, ONE_YEAR_IN_DAYS); self.fees_in_dollars_1y_sum.multi_insert_last_x_sum( dates, - &mut self.fees_in_dollars.date, + &mut self.fees_in_dollars_1d_sum, ONE_YEAR_IN_DAYS, ); self.cumulative_fees - .multi_insert_cumulative(heights, dates, &mut self.fees); + .height + .multi_insert_cumulative(heights, &mut self.fees); + self.cumulative_fees + .date + .multi_insert_cumulative(dates, &mut self.fees_1d_sum); - self.cumulative_fees_in_dollars.multi_insert_cumulative( - heights, - dates, - &mut self.fees_in_dollars, - ); + self.cumulative_fees_in_dollars + .height + .multi_insert_cumulative(heights, &mut self.fees_in_dollars); + self.cumulative_fees_in_dollars + .date + .multi_insert_cumulative(dates, &mut self.fees_in_dollars_1d_sum); self.coinbase_1y_sum.multi_insert_last_x_sum( dates, - &mut self.coinbase.date, + &mut self.coinbase_1d_sum, ONE_YEAR_IN_DAYS, ); self.coinbase_in_dollars_1y_sum.multi_insert_last_x_sum( dates, - &mut self.coinbase_in_dollars.date, + &mut self.coinbase_in_dollars_1d_sum, ONE_YEAR_IN_DAYS, ); - self.coinbase_in_dollars_1y_sma.multi_insert_simple_average( - dates, - &mut self.coinbase_in_dollars.date, - ONE_YEAR_IN_DAYS, - ); + self.coinbase_in_dollars_1d_sum_1y_sma + .multi_insert_simple_average( + dates, + &mut self.coinbase_in_dollars_1d_sum, + ONE_YEAR_IN_DAYS, + ); self.cumulative_coinbase - .multi_insert_cumulative(heights, dates, &mut self.coinbase); + .height + .multi_insert_cumulative(heights, &mut self.coinbase); + self.cumulative_coinbase + .date + .multi_insert_cumulative(dates, &mut self.coinbase_1d_sum); - self.cumulative_coinbase_in_dollars.multi_insert_cumulative( - heights, - dates, - &mut self.coinbase_in_dollars, - ); + self.cumulative_coinbase_in_dollars + .height + .multi_insert_cumulative(heights, &mut self.coinbase_in_dollars); + self.cumulative_coinbase_in_dollars + .date + .multi_insert_cumulative(dates, &mut self.coinbase_in_dollars_1d_sum); self.subsidy_to_coinbase_ratio.multi_insert_percentage( heights, - dates, &mut self.subsidy, &mut self.coinbase, ); + self.subsidy_to_coinbase_1d_ratio.multi_insert_percentage( + dates, + &mut self.subsidy_1d_sum, + &mut self.coinbase_1d_sum, + ); self.fees_to_coinbase_ratio.multi_insert_percentage( heights, - dates, &mut self.fees, &mut self.coinbase, ); + self.fees_to_coinbase_1d_ratio.multi_insert_percentage( + dates, + &mut self.fees_1d_sum, + &mut self.coinbase_1d_sum, + ); self.annualized_issuance.multi_insert_last_x_sum( - heights, dates, - &mut self.subsidy, + &mut self.subsidy_1d_sum, ONE_YEAR_IN_DAYS, ); + self.inflation_rate.multi_insert_simple_transform( + dates, + &mut self.subsidy_1d_sum, + |subsidy_1d_sum, date| { + subsidy_1d_sum * ONE_YEAR_IN_DAYS as f64 + / self.cumulative_subsidy.date.get_or_import(date).unwrap() + * 100.0 + }, + ); + self.yearly_inflation_rate.multi_insert_percentage( - heights, dates, &mut self.annualized_issuance, - &mut self.cumulative_subsidy, + &mut self.cumulative_subsidy.date, ); self.blocks_mined_1d_target @@ -502,7 +540,9 @@ impl MiningDataset { let difficulty = self.difficulty.date.get_or_import(date).unwrap(); - ((blocks_mined as f64 / TARGET_BLOCKS_PER_DAY as f64) * difficulty * 2.0_f64.powi(32)) + ((blocks_mined as f64 / date.get_day_completion() * TARGET_BLOCKS_PER_DAY as f64) + * difficulty + * 2.0_f64.powi(32)) / 600.0 / 1_000_000_000_000_000_000.0 }); @@ -526,7 +566,7 @@ impl MiningDataset { ); self.hash_price.multi_insert(dates, |date| { - let coinbase_in_dollars = self.coinbase_in_dollars.date.get_or_import(date).unwrap(); + let coinbase_in_dollars = self.coinbase_in_dollars_1d_sum.get_or_import(date).unwrap(); let hashrate = self.hash_rate.get_or_import(date).unwrap(); @@ -535,8 +575,13 @@ impl MiningDataset { self.puell_multiple.multi_insert_divide( dates, - &mut self.coinbase_in_dollars.date, - &mut self.coinbase_in_dollars_1y_sma, + &mut self.coinbase_in_dollars_1d_sum, + &mut self.coinbase_in_dollars_1d_sum_1y_sma, + ); + self.puell_multiple.multi_insert_divide( + dates, + &mut self.coinbase_in_dollars_1d_sum, + &mut self.coinbase_in_dollars_1d_sum_1y_sma, ); self.difficulty_adjustment.multi_insert_percentage_change( @@ -590,31 +635,21 @@ impl AnyDataset for MiningDataset { } fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { - vec![ - &self.coinbase, - &self.coinbase_in_dollars, - &self.fees, - &self.fees_in_dollars, - &self.subsidy, - &self.subsidy_in_dollars, - &self.difficulty, - ] + vec![&self.difficulty] } fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { - vec![ - &mut self.coinbase, - &mut self.coinbase_in_dollars, - &mut self.fees, - &mut self.fees_in_dollars, - &mut self.subsidy, - &mut self.subsidy_in_dollars, - &mut self.difficulty, - ] + vec![&mut self.difficulty] } fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { vec![ + &self.coinbase_1d_sum, + &self.coinbase_in_dollars_1d_sum, + &self.fees_1d_sum, + &self.fees_in_dollars_1d_sum, + &self.subsidy_1d_sum, + &self.subsidy_in_dollars_1d_sum, &self.total_blocks_mined, &self.blocks_mined, &self.last_subsidy, @@ -628,6 +663,12 @@ impl AnyDataset for MiningDataset { fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { vec![ + &mut self.coinbase_1d_sum, + &mut self.coinbase_in_dollars_1d_sum, + &mut self.fees_1d_sum, + &mut self.fees_in_dollars_1d_sum, + &mut self.subsidy_1d_sum, + &mut self.subsidy_in_dollars_1d_sum, &mut self.total_blocks_mined, &mut self.blocks_mined, &mut self.last_subsidy, @@ -641,6 +682,12 @@ impl AnyDataset for MiningDataset { fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { vec![ + &self.coinbase, + &self.coinbase_in_dollars, + &self.fees, + &self.fees_in_dollars, + &self.subsidy, + &self.subsidy_in_dollars, &self.block_size, &self.block_weight, &self.block_vbytes, @@ -650,6 +697,12 @@ impl AnyDataset for MiningDataset { fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![ + &mut self.coinbase, + &mut self.coinbase_in_dollars, + &mut self.fees, + &mut self.fees_in_dollars, + &mut self.subsidy, + &mut self.subsidy_in_dollars, &mut self.block_size, &mut self.block_weight, &mut self.block_vbytes, @@ -665,11 +718,7 @@ impl AnyDataset for MiningDataset { &self.cumulative_fees_in_dollars, &self.cumulative_subsidy, &self.cumulative_subsidy_in_dollars, - &self.annualized_issuance, - &self.yearly_inflation_rate, &self.cumulative_block_size, - &self.subsidy_to_coinbase_ratio, - &self.fees_to_coinbase_ratio, ] } @@ -681,9 +730,19 @@ impl AnyDataset for MiningDataset { &mut self.cumulative_fees_in_dollars, &mut self.cumulative_subsidy, &mut self.cumulative_subsidy_in_dollars, - &mut self.annualized_issuance, - &mut self.yearly_inflation_rate, &mut self.cumulative_block_size, + ] + } + + fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![ + &self.subsidy_to_coinbase_ratio, + &self.fees_to_coinbase_ratio, + ] + } + + fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { + vec![ &mut self.subsidy_to_coinbase_ratio, &mut self.fees_to_coinbase_ratio, ] @@ -704,9 +763,14 @@ impl AnyDataset for MiningDataset { &self.subsidy_in_dollars_1y_sum, &self.coinbase_1y_sum, &self.coinbase_in_dollars_1y_sum, - &self.coinbase_in_dollars_1y_sma, + &self.coinbase_in_dollars_1d_sum_1y_sma, + &self.fees_to_coinbase_1d_ratio, + &self.annualized_issuance, &self.fees_1y_sum, &self.fees_in_dollars_1y_sum, + &self.inflation_rate, + &self.yearly_inflation_rate, + &self.subsidy_to_coinbase_1d_ratio, &self.hash_rate, &self.hash_rate_1w_sma, &self.hash_rate_1m_sma, @@ -742,11 +806,16 @@ impl AnyDataset for MiningDataset { &mut self.blocks_mined_1w_target, &mut self.blocks_mined_1m_target, &mut self.blocks_mined_1y_target, + &mut self.annualized_issuance, &mut self.subsidy_1y_sum, &mut self.subsidy_in_dollars_1y_sum, + &mut self.fees_to_coinbase_1d_ratio, + &mut self.inflation_rate, + &mut self.yearly_inflation_rate, + &mut self.subsidy_to_coinbase_1d_ratio, &mut self.coinbase_1y_sum, &mut self.coinbase_in_dollars_1y_sum, - &mut self.coinbase_in_dollars_1y_sma, + &mut self.coinbase_in_dollars_1d_sum_1y_sma, &mut self.fees_1y_sum, &mut self.fees_in_dollars_1y_sum, &mut self.hash_rate, diff --git a/parser/src/datasets/mod.rs b/parser/src/datasets/mod.rs index 73e343d37..6af3c5fcc 100644 --- a/parser/src/datasets/mod.rs +++ b/parser/src/datasets/mod.rs @@ -259,6 +259,7 @@ impl AllDatasets { &mut self.mining.cumulative_subsidy, &mut self.address.cohorts.all.subs.capitalization.realized_cap, &mut self.address.cohorts.all.subs.capitalization.realized_price, + &mut self.mining.inflation_rate, &mut self.mining.yearly_inflation_rate, &mut self.transaction.annualized_volume, &mut self.mining.cumulative_subsidy_in_dollars, diff --git a/parser/src/datasets/subs/input.rs b/parser/src/datasets/subs/input.rs index b22e1a045..07a6f0ffc 100644 --- a/parser/src/datasets/subs/input.rs +++ b/parser/src/datasets/subs/input.rs @@ -3,7 +3,8 @@ use allocative::Allocative; use crate::{ datasets::{AnyDataset, InsertData, MinInitialStates}, states::InputState, - structs::{AnyBiMap, BiMap}, + structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap}, + DateMap, HeightMap, }; #[derive(Default, Allocative)] @@ -12,7 +13,8 @@ pub struct InputSubDataset { // Inserted pub count: BiMap, - pub volume: BiMap, + pub volume: HeightMap, + pub volume_1d_sum: DateMap, // Computed // add inputs_per_second } @@ -31,7 +33,8 @@ impl InputSubDataset { min_initial_states: MinInitialStates::default(), count: BiMap::new_bin(1, &f("input_count")), - volume: BiMap::new_bin(1, &f("input_volume")), + volume: HeightMap::new_bin(1, &f("input_volume")), + volume_1d_sum: DateMap::new_bin(1, &f("input_volume_1d_sum")), }; s.min_initial_states @@ -56,12 +59,13 @@ impl InputSubDataset { .height .insert(height, state.count().round() as u64); - self.volume.height.insert(height, state.volume().to_btc()); + self.volume.insert(height, state.volume().to_btc()); if is_date_last_block { self.count.date.insert(date, count); - self.volume.date_insert_sum_range(date, date_blocks_range); + self.volume_1d_sum + .insert(date, self.volume.sum_range(date_blocks_range)); } } } @@ -72,10 +76,26 @@ impl AnyDataset for InputSubDataset { } fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { - vec![&self.count, &self.volume] + vec![&self.count] } fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { - vec![&mut self.count, &mut self.volume] + vec![&mut self.count] + } + + fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![&self.volume] + } + + fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { + vec![&mut self.volume] + } + + fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![&self.volume_1d_sum] + } + + fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![&mut self.volume_1d_sum] } } diff --git a/parser/src/datasets/subs/realized.rs b/parser/src/datasets/subs/realized.rs index 3882457e1..87adbb9b6 100644 --- a/parser/src/datasets/subs/realized.rs +++ b/parser/src/datasets/subs/realized.rs @@ -3,9 +3,9 @@ use allocative::Allocative; use crate::{ datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates}, states::RealizedState, - structs::{AnyBiMap, AnyDateMap, BiMap, Price}, + structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Price}, utils::ONE_MONTH_IN_DAYS, - DateMap, + DateMap, HeightMap, }; #[derive(Default, Allocative)] @@ -13,26 +13,36 @@ pub struct RealizedSubDataset { min_initial_states: MinInitialStates, // Inserted - realized_profit: BiMap, - realized_loss: BiMap, - value_created: BiMap, - adjusted_value_created: BiMap, - value_destroyed: BiMap, - adjusted_value_destroyed: BiMap, + realized_profit: HeightMap, + realized_loss: HeightMap, + value_created: HeightMap, + adjusted_value_created: HeightMap, + value_destroyed: HeightMap, + adjusted_value_destroyed: HeightMap, + realized_profit_1d_sum: DateMap, + realized_loss_1d_sum: DateMap, + value_created_1d_sum: DateMap, + adjusted_value_created_1d_sum: DateMap, + value_destroyed_1d_sum: DateMap, + adjusted_value_destroyed_1d_sum: DateMap, spent_output_profit_ratio: BiMap, adjusted_spent_output_profit_ratio: BiMap, // Computed - negative_realized_loss: BiMap, - net_realized_profit_and_loss: BiMap, - net_realized_profit_and_loss_to_market_cap_ratio: BiMap, + negative_realized_loss: HeightMap, + negative_realized_loss_1d_sum: DateMap, + net_realized_profit_and_loss: HeightMap, + net_realized_profit_and_loss_1d_sum: DateMap, + net_realized_profit_and_loss_1d_sum_to_market_cap_ratio: DateMap, cumulative_realized_profit: BiMap, cumulative_realized_loss: BiMap, cumulative_net_realized_profit_and_loss: BiMap, cumulative_net_realized_profit_and_loss_1m_net_change: BiMap, - realized_value: BiMap, + realized_value: HeightMap, + realized_value_1d_sum: DateMap, sell_side_risk_ratio: DateMap, - realized_profit_to_loss_ratio: BiMap, + realized_profit_to_loss_ratio: HeightMap, + realized_profit_to_loss_1d_sum_ratio: DateMap, } impl RealizedSubDataset { @@ -48,21 +58,35 @@ impl RealizedSubDataset { let mut s = Self { min_initial_states: MinInitialStates::default(), - realized_profit: BiMap::new_bin(1, &f("realized_profit")), - realized_loss: BiMap::new_bin(1, &f("realized_loss")), - value_created: BiMap::new_bin(1, &f("value_created")), - adjusted_value_created: BiMap::new_bin(1, &f("adjusted_value_created")), - value_destroyed: BiMap::new_bin(1, &f("value_destroyed")), - adjusted_value_destroyed: BiMap::new_bin(1, &f("adjusted_value_destroyed")), + realized_profit: HeightMap::new_bin(1, &f("realized_profit")), + realized_loss: HeightMap::new_bin(1, &f("realized_loss")), + value_created: HeightMap::new_bin(1, &f("value_created")), + adjusted_value_created: HeightMap::new_bin(1, &f("adjusted_value_created")), + value_destroyed: HeightMap::new_bin(1, &f("value_destroyed")), + adjusted_value_destroyed: HeightMap::new_bin(1, &f("adjusted_value_destroyed")), + realized_profit_1d_sum: DateMap::new_bin(1, &f("realized_profit_1d_sum")), + realized_loss_1d_sum: DateMap::new_bin(1, &f("realized_loss_1d_sum")), + value_created_1d_sum: DateMap::new_bin(1, &f("value_created_1d_sum")), + adjusted_value_created_1d_sum: DateMap::new_bin(1, &f("adjusted_value_created_1d_sum")), + value_destroyed_1d_sum: DateMap::new_bin(1, &f("value_destroyed_1d_sum")), + adjusted_value_destroyed_1d_sum: DateMap::new_bin( + 1, + &f("adjusted_value_destroyed_1d_sum"), + ), spent_output_profit_ratio: BiMap::new_bin(2, &f("spent_output_profit_ratio")), adjusted_spent_output_profit_ratio: BiMap::new_bin( 2, &f("adjusted_spent_output_profit_ratio"), ), - negative_realized_loss: BiMap::new_bin(2, &f("negative_realized_loss")), - net_realized_profit_and_loss: BiMap::new_bin(1, &f("net_realized_profit_and_loss")), - net_realized_profit_and_loss_to_market_cap_ratio: BiMap::new_bin( + negative_realized_loss: HeightMap::new_bin(2, &f("negative_realized_loss")), + negative_realized_loss_1d_sum: DateMap::new_bin(2, &f("negative_realized_loss_1d_sum")), + net_realized_profit_and_loss: HeightMap::new_bin(1, &f("net_realized_profit_and_loss")), + net_realized_profit_and_loss_1d_sum: DateMap::new_bin( + 1, + &f("net_realized_profit_and_loss_1d_sum"), + ), + net_realized_profit_and_loss_1d_sum_to_market_cap_ratio: DateMap::new_bin( 2, &f("net_realized_profit_and_loss_to_market_cap_ratio"), ), @@ -76,9 +100,17 @@ impl RealizedSubDataset { 1, &f("cumulative_net_realized_profit_and_loss_1m_net_change"), ), - realized_value: BiMap::new_bin(1, &f("realized_value")), + realized_value: HeightMap::new_bin(1, &f("realized_value")), + realized_value_1d_sum: DateMap::new_bin(1, &f("realized_value_1d_sum")), sell_side_risk_ratio: DateMap::new_bin(1, &f("sell_side_risk_ratio")), - realized_profit_to_loss_ratio: BiMap::new_bin(1, &f("realized_profit_to_loss_ratio")), + realized_profit_to_loss_ratio: HeightMap::new_bin( + 1, + &f("realized_profit_to_loss_ratio"), + ), + realized_profit_to_loss_1d_sum_ratio: DateMap::new_bin( + 1, + &f("realized_profit_to_loss_1d_sum_ratio"), + ), }; s.min_initial_states @@ -99,27 +131,23 @@ impl RealizedSubDataset { height_state: &RealizedState, ) { self.realized_profit - .height .insert(height, height_state.realized_profit().to_dollar() as f32); self.realized_loss - .height .insert(height, height_state.realized_loss().to_dollar() as f32); self.value_created - .height .insert(height, height_state.value_created().to_dollar() as f32); - self.adjusted_value_created.height.insert( + self.adjusted_value_created.insert( height, height_state.adjusted_value_created().to_dollar() as f32, ); self.value_destroyed - .height .insert(height, height_state.value_destroyed().to_dollar() as f32); - self.adjusted_value_destroyed.height.insert( + self.adjusted_value_destroyed.insert( height, height_state.adjusted_value_destroyed().to_dollar() as f32, ); @@ -146,39 +174,37 @@ impl RealizedSubDataset { }); if is_date_last_block { - self.realized_profit - .date_insert_sum_range(date, date_blocks_range); + self.realized_profit_1d_sum + .insert(date, self.realized_profit.sum_range(date_blocks_range)); - self.realized_loss - .date_insert_sum_range(date, date_blocks_range); + self.realized_loss_1d_sum + .insert(date, self.realized_loss.sum_range(date_blocks_range)); - self.value_created - .date_insert_sum_range(date, date_blocks_range); + let value_created_1d_sum = self + .value_created_1d_sum + .insert(date, self.value_created.sum_range(date_blocks_range)); - self.adjusted_value_created - .date_insert_sum_range(date, date_blocks_range); - - self.value_destroyed - .date_insert_sum_range(date, date_blocks_range); - - self.adjusted_value_destroyed - .date_insert_sum_range(date, date_blocks_range); - - self.spent_output_profit_ratio.date.insert( + let adjusted_value_created_1d_sum = self.adjusted_value_created_1d_sum.insert( date, - self.value_created.height.sum_range(date_blocks_range) - / self.value_destroyed.height.sum_range(date_blocks_range), + self.adjusted_value_created.sum_range(date_blocks_range), ); + let value_destroyed_1d_sum = self + .value_destroyed_1d_sum + .insert(date, self.value_destroyed.sum_range(date_blocks_range)); + + let adjusted_value_destroyed_1d_sum = self.adjusted_value_destroyed_1d_sum.insert( + date, + self.adjusted_value_destroyed.sum_range(date_blocks_range), + ); + + self.spent_output_profit_ratio + .date + .insert(date, value_created_1d_sum / value_destroyed_1d_sum); + self.adjusted_spent_output_profit_ratio.date.insert( date, - self.adjusted_value_created - .height - .sum_range(date_blocks_range) - / self - .adjusted_value_destroyed - .height - .sum_range(date_blocks_range), + adjusted_value_created_1d_sum / adjusted_value_destroyed_1d_sum, ); } } @@ -190,40 +216,53 @@ impl RealizedSubDataset { ) { self.negative_realized_loss.multi_insert_simple_transform( heights, - dates, &mut self.realized_loss, - &|v| v * -1.0, + |v, _| v * -1.0, ); + self.negative_realized_loss_1d_sum + .multi_insert_simple_transform(dates, &mut self.realized_loss_1d_sum, |v, _| v * -1.0); + self.net_realized_profit_and_loss.multi_insert_subtract( heights, - dates, &mut self.realized_profit, &mut self.realized_loss, ); - self.net_realized_profit_and_loss_to_market_cap_ratio - .multi_insert_percentage( - heights, + self.net_realized_profit_and_loss_1d_sum + .multi_insert_subtract( dates, - &mut self.net_realized_profit_and_loss, - market_cap, + &mut self.realized_profit_1d_sum, + &mut self.realized_loss_1d_sum, ); - self.cumulative_realized_profit.multi_insert_cumulative( - heights, - dates, - &mut self.realized_profit, - ); + self.net_realized_profit_and_loss_1d_sum_to_market_cap_ratio + .multi_insert_percentage( + dates, + &mut self.net_realized_profit_and_loss_1d_sum, + &mut market_cap.date, + ); - self.cumulative_realized_loss.multi_insert_cumulative( - heights, - dates, - &mut self.realized_loss, - ); + self.cumulative_realized_profit + .height + .multi_insert_cumulative(heights, &mut self.realized_profit); + self.cumulative_realized_profit + .date + .multi_insert_cumulative(dates, &mut self.realized_profit_1d_sum); + + self.cumulative_realized_loss + .height + .multi_insert_cumulative(heights, &mut self.realized_loss); + self.cumulative_realized_loss + .date + .multi_insert_cumulative(dates, &mut self.realized_loss_1d_sum); self.cumulative_net_realized_profit_and_loss - .multi_insert_cumulative(heights, dates, &mut self.net_realized_profit_and_loss); + .height + .multi_insert_cumulative(heights, &mut self.net_realized_profit_and_loss); + self.cumulative_net_realized_profit_and_loss + .date + .multi_insert_cumulative(dates, &mut self.net_realized_profit_and_loss_1d_sum); self.cumulative_net_realized_profit_and_loss_1m_net_change .multi_insert_net_change( @@ -235,23 +274,33 @@ impl RealizedSubDataset { self.realized_value.multi_insert_add( heights, - dates, &mut self.realized_profit, &mut self.realized_loss, ); + self.realized_value_1d_sum.multi_insert_add( + dates, + &mut self.realized_profit_1d_sum, + &mut self.realized_loss_1d_sum, + ); + self.sell_side_risk_ratio.multi_insert_percentage( dates, - &mut self.realized_value.date, + &mut self.realized_value_1d_sum, &mut market_cap.date, ); self.realized_profit_to_loss_ratio.multi_insert_divide( heights, - dates, &mut self.realized_profit, &mut self.realized_loss, ); + self.realized_profit_to_loss_1d_sum_ratio + .multi_insert_divide( + dates, + &mut self.realized_profit_1d_sum, + &mut self.realized_loss_1d_sum, + ); } } @@ -262,18 +311,30 @@ impl AnyDataset for RealizedSubDataset { fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { vec![ - &self.realized_loss, - &self.realized_profit, - &self.value_created, - &self.adjusted_value_created, - &self.value_destroyed, - &self.adjusted_value_destroyed, &self.spent_output_profit_ratio, &self.adjusted_spent_output_profit_ratio, ] } fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + vec![ + &mut self.spent_output_profit_ratio, + &mut self.adjusted_spent_output_profit_ratio, + ] + } + + fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![ + &self.realized_loss, + &self.realized_profit, + &self.value_created, + &self.adjusted_value_created, + &self.value_destroyed, + &self.adjusted_value_destroyed, + ] + } + + fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![ &mut self.realized_loss, &mut self.realized_profit, @@ -281,44 +342,86 @@ impl AnyDataset for RealizedSubDataset { &mut self.adjusted_value_created, &mut self.value_destroyed, &mut self.adjusted_value_destroyed, - &mut self.spent_output_profit_ratio, - &mut self.adjusted_spent_output_profit_ratio, + ] + } + + fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![ + &self.realized_loss_1d_sum, + &self.realized_profit_1d_sum, + &self.value_created_1d_sum, + &self.adjusted_value_created_1d_sum, + &self.value_destroyed_1d_sum, + &self.adjusted_value_destroyed_1d_sum, + ] + } + + fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![ + &mut self.realized_loss_1d_sum, + &mut self.realized_profit_1d_sum, + &mut self.value_created_1d_sum, + &mut self.adjusted_value_created_1d_sum, + &mut self.value_destroyed_1d_sum, + &mut self.adjusted_value_destroyed_1d_sum, ] } fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { vec![ - &self.negative_realized_loss, - &self.net_realized_profit_and_loss, - &self.net_realized_profit_and_loss_to_market_cap_ratio, &self.cumulative_realized_profit, &self.cumulative_realized_loss, &self.cumulative_net_realized_profit_and_loss, &self.cumulative_net_realized_profit_and_loss_1m_net_change, + ] + } + + fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + vec![ + &mut self.cumulative_realized_profit, + &mut self.cumulative_realized_loss, + &mut self.cumulative_net_realized_profit_and_loss, + &mut self.cumulative_net_realized_profit_and_loss_1m_net_change, + ] + } + + fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![ + &self.negative_realized_loss, + &self.net_realized_profit_and_loss, &self.realized_value, &self.realized_profit_to_loss_ratio, ] } - fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { - vec![&self.sell_side_risk_ratio] - } - - fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![ &mut self.negative_realized_loss, &mut self.net_realized_profit_and_loss, - &mut self.net_realized_profit_and_loss_to_market_cap_ratio, - &mut self.cumulative_realized_profit, - &mut self.cumulative_realized_loss, - &mut self.cumulative_net_realized_profit_and_loss, - &mut self.cumulative_net_realized_profit_and_loss_1m_net_change, &mut self.realized_value, &mut self.realized_profit_to_loss_ratio, ] } + fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![ + &self.sell_side_risk_ratio, + &self.negative_realized_loss_1d_sum, + &self.net_realized_profit_and_loss_1d_sum, + &self.net_realized_profit_and_loss_1d_sum_to_market_cap_ratio, + &self.realized_value_1d_sum, + &self.realized_profit_to_loss_1d_sum_ratio, + ] + } + fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { - vec![&mut self.sell_side_risk_ratio] + vec![ + &mut self.sell_side_risk_ratio, + &mut self.negative_realized_loss_1d_sum, + &mut self.net_realized_profit_and_loss_1d_sum, + &mut self.net_realized_profit_and_loss_1d_sum_to_market_cap_ratio, + &mut self.realized_value_1d_sum, + &mut self.realized_profit_to_loss_1d_sum_ratio, + ] } } diff --git a/parser/src/datasets/transaction.rs b/parser/src/datasets/transaction.rs index 3fac5c963..3355ae99d 100644 --- a/parser/src/datasets/transaction.rs +++ b/parser/src/datasets/transaction.rs @@ -2,8 +2,11 @@ use allocative::Allocative; use crate::{ datasets::InsertData, - structs::{AnyBiMap, BiMap, HeightMap}, - utils::{ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, + structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, HeightMap}, + utils::{ + ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY, + }, + DateMap, }; use super::{AnyDataset, ComputeData, MinInitialStates}; @@ -13,9 +16,12 @@ pub struct TransactionDataset { min_initial_states: MinInitialStates, // Inserted - pub count: BiMap, - pub volume: BiMap, - pub volume_in_dollars: BiMap, + pub count: HeightMap, + pub count_1d_sum: DateMap, + pub volume: HeightMap, + pub volume_1d_sum: DateMap, + pub volume_in_dollars: HeightMap, + pub volume_in_dollars_1d_sum: DateMap, // Average sent // Average sent in dollars // Median sent @@ -27,15 +33,21 @@ pub struct TransactionDataset { // version // Computed - pub count_1w_sma: BiMap, - pub count_1m_sma: BiMap, - pub volume_1w_sma: BiMap, - pub volume_1m_sma: BiMap, - pub volume_in_dollars_1w_sma: BiMap, - pub volume_in_dollars_1m_sma: BiMap, - pub annualized_volume: BiMap, - pub annualized_volume_in_dollars: BiMap, - pub velocity: BiMap, + pub count_1w_sma: HeightMap, + pub count_1d_sum_1w_sma: DateMap, + pub count_1m_sma: HeightMap, + pub count_1d_sum_1m_sma: DateMap, + pub volume_1w_sma: HeightMap, + pub volume_1d_sum_1w_sma: DateMap, + pub volume_1m_sma: HeightMap, + pub volume_1d_sum_1m_sma: DateMap, + pub volume_in_dollars_1w_sma: HeightMap, + pub volume_in_dollars_1d_sum_1w_sma: DateMap, + pub volume_in_dollars_1m_sma: HeightMap, + pub volume_in_dollars_1d_sum_1m_sma: DateMap, + pub annualized_volume: DateMap, + pub annualized_volume_in_dollars: DateMap, + pub velocity: DateMap, pub transactions_per_second: BiMap, pub transactions_per_second_1w_sma: BiMap, pub transactions_per_second_1m_sma: BiMap, @@ -48,21 +60,45 @@ impl TransactionDataset { let mut s = Self { min_initial_states: MinInitialStates::default(), - count: BiMap::new_bin(1, &f("transaction_count")), - count_1w_sma: BiMap::new_bin(1, &f("transaction_count_1w_sma")), - count_1m_sma: BiMap::new_bin(1, &f("transaction_count_1m_sma")), - volume: BiMap::new_bin(1, &f("transaction_volume")), - volume_1w_sma: BiMap::new_bin(1, &f("transaction_volume_1w_sma")), - volume_1m_sma: BiMap::new_bin(1, &f("transaction_volume_1m_sma")), - volume_in_dollars: BiMap::new_bin(1, &f("transaction_volume_in_dollars")), - volume_in_dollars_1w_sma: BiMap::new_bin(1, &f("transaction_volume_in_dollars_1w_sma")), - volume_in_dollars_1m_sma: BiMap::new_bin(1, &f("transaction_volume_in_dollars_1m_sma")), - annualized_volume: BiMap::new_bin(1, &f("annualized_transaction_volume")), - annualized_volume_in_dollars: BiMap::new_bin( + count: HeightMap::new_bin(1, &f("transaction_count")), + count_1d_sum: DateMap::new_bin(1, &f("transaction_count_1d_sum")), + count_1w_sma: HeightMap::new_bin(1, &f("transaction_count_1w_sma")), + count_1d_sum_1w_sma: DateMap::new_bin(1, &f("transaction_count_1d_sum_1w_sma")), + count_1m_sma: HeightMap::new_bin(1, &f("transaction_count_1m_sma")), + count_1d_sum_1m_sma: DateMap::new_bin(1, &f("transaction_count_1d_sum_1m_sma")), + volume: HeightMap::new_bin(1, &f("transaction_volume")), + volume_1d_sum: DateMap::new_bin(1, &f("transaction_volume_1d_sum")), + volume_1w_sma: HeightMap::new_bin(1, &f("transaction_volume_1w_sma")), + volume_1d_sum_1w_sma: DateMap::new_bin(1, &f("transaction_volume_1d_sum_1w_sma")), + volume_1m_sma: HeightMap::new_bin(1, &f("transaction_volume_1m_sma")), + volume_1d_sum_1m_sma: DateMap::new_bin(1, &f("transaction_volume_1d_sum_1m_sma")), + volume_in_dollars: HeightMap::new_bin(1, &f("transaction_volume_in_dollars")), + volume_in_dollars_1d_sum: DateMap::new_bin( + 1, + &f("transaction_volume_in_dollars_1d_sum"), + ), + volume_in_dollars_1w_sma: HeightMap::new_bin( + 1, + &f("transaction_volume_in_dollars_1w_sma"), + ), + volume_in_dollars_1d_sum_1w_sma: DateMap::new_bin( + 1, + &f("transaction_volume_in_dollars_1d_sum_1w_sma"), + ), + volume_in_dollars_1m_sma: HeightMap::new_bin( + 1, + &f("transaction_volume_in_dollars_1m_sma"), + ), + volume_in_dollars_1d_sum_1m_sma: DateMap::new_bin( + 1, + &f("transaction_volume_in_dollars_1d_sum_1m_sma"), + ), + annualized_volume: DateMap::new_bin(1, &f("annualized_transaction_volume")), + annualized_volume_in_dollars: DateMap::new_bin( 2, &f("annualized_transaction_volume_in_dollars"), ), - velocity: BiMap::new_bin(1, &f("transaction_velocity")), + velocity: DateMap::new_bin(1, &f("transaction_velocity")), transactions_per_second: BiMap::new_bin(1, &f("transactions_per_second")), transactions_per_second_1w_sma: BiMap::new_bin(1, &f("transactions_per_second_1w_sma")), transactions_per_second_1m_sma: BiMap::new_bin(1, &f("transactions_per_second_1m_sma")), @@ -87,21 +123,22 @@ impl TransactionDataset { .. }: &InsertData, ) { - self.count.height.insert(height, transaction_count); + self.count.insert(height, transaction_count); - self.volume.height.insert(height, amount_sent.to_btc()); + self.volume.insert(height, amount_sent.to_btc()); self.volume_in_dollars - .height .insert(height, (block_price * amount_sent).to_dollar() as f32); if is_date_last_block { - self.count.date_insert_sum_range(date, date_blocks_range); + self.count_1d_sum + .insert(date, self.count.sum_range(date_blocks_range)); - self.volume.date_insert_sum_range(date, date_blocks_range); + self.volume_1d_sum + .insert(date, self.volume.sum_range(date_blocks_range)); - self.volume_in_dollars - .date_insert_sum_range(date, date_blocks_range); + self.volume_in_dollars_1d_sum + .insert(date, self.volume_in_dollars.sum_range(date_blocks_range)); } } @@ -113,77 +150,100 @@ impl TransactionDataset { ) { self.count_1w_sma.multi_insert_simple_average( heights, - dates, &mut self.count, + TARGET_BLOCKS_PER_DAY * ONE_WEEK_IN_DAYS, + ); + self.count_1d_sum_1w_sma.multi_insert_simple_average( + dates, + &mut self.count_1d_sum, ONE_WEEK_IN_DAYS, ); self.count_1m_sma.multi_insert_simple_average( heights, - dates, &mut self.count, + TARGET_BLOCKS_PER_DAY * ONE_MONTH_IN_DAYS, + ); + self.count_1d_sum_1m_sma.multi_insert_simple_average( + dates, + &mut self.count_1d_sum, ONE_MONTH_IN_DAYS, ); self.volume_1w_sma.multi_insert_simple_average( heights, - dates, &mut self.volume, + TARGET_BLOCKS_PER_DAY * ONE_WEEK_IN_DAYS, + ); + self.volume_1d_sum_1w_sma.multi_insert_simple_average( + dates, + &mut self.volume_1d_sum, ONE_WEEK_IN_DAYS, ); self.volume_1m_sma.multi_insert_simple_average( heights, - dates, &mut self.volume, + TARGET_BLOCKS_PER_DAY * ONE_MONTH_IN_DAYS, + ); + self.volume_1d_sum_1m_sma.multi_insert_simple_average( + dates, + &mut self.volume_1d_sum, ONE_MONTH_IN_DAYS, ); self.volume_in_dollars_1w_sma.multi_insert_simple_average( heights, - dates, &mut self.volume_in_dollars, - ONE_WEEK_IN_DAYS, + TARGET_BLOCKS_PER_DAY * ONE_WEEK_IN_DAYS, ); + self.volume_in_dollars_1d_sum_1w_sma + .multi_insert_simple_average( + dates, + &mut self.volume_in_dollars_1d_sum, + ONE_WEEK_IN_DAYS, + ); self.volume_in_dollars_1m_sma.multi_insert_simple_average( heights, - dates, &mut self.volume_in_dollars, - ONE_MONTH_IN_DAYS, + TARGET_BLOCKS_PER_DAY * ONE_MONTH_IN_DAYS, ); + self.volume_in_dollars_1d_sum_1m_sma + .multi_insert_simple_average( + dates, + &mut self.volume_in_dollars_1d_sum, + ONE_MONTH_IN_DAYS, + ); self.annualized_volume.multi_insert_last_x_sum( - heights, dates, - &mut self.volume, + &mut self.volume_1d_sum, ONE_YEAR_IN_DAYS, ); self.annualized_volume_in_dollars.multi_insert_last_x_sum( - heights, dates, - &mut self.volume_in_dollars, + &mut self.volume_in_dollars_1d_sum, ONE_YEAR_IN_DAYS, ); self.velocity.multi_insert_divide( - heights, dates, &mut self.annualized_volume, - circulating_supply, + &mut circulating_supply.date, ); self.transactions_per_second.height.multi_insert_divide( heights, - &mut self.count.height, + &mut self.count, block_interval, ); self.transactions_per_second .date - .multi_insert_simple_transform(dates, &mut self.count.date, |count, _| { - count as f32 / ONE_DAY_IN_S as f32 + .multi_insert_simple_transform(dates, &mut self.count_1d_sum, |count, date| { + count as f32 / (date.get_day_completion() as f32 * ONE_DAY_IN_S as f32) }); self.transactions_per_second_1w_sma @@ -209,11 +269,11 @@ impl AnyDataset for TransactionDataset { &self.min_initial_states } - fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { + fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { vec![&self.count, &self.volume, &self.volume_in_dollars] } - fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![ &mut self.count, &mut self.volume, @@ -221,17 +281,24 @@ impl AnyDataset for TransactionDataset { ] } + fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![ + &self.count_1d_sum, + &self.volume_1d_sum, + &self.volume_in_dollars_1d_sum, + ] + } + + fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![ + &mut self.count_1d_sum, + &mut self.volume_1d_sum, + &mut self.volume_in_dollars_1d_sum, + ] + } + fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> { vec![ - &self.count_1w_sma, - &self.count_1m_sma, - &self.volume_1w_sma, - &self.volume_1m_sma, - &self.volume_in_dollars_1w_sma, - &self.volume_in_dollars_1m_sma, - &self.annualized_volume, - &self.annualized_volume_in_dollars, - &self.velocity, &self.transactions_per_second, &self.transactions_per_second_1w_sma, &self.transactions_per_second_1m_sma, @@ -239,6 +306,25 @@ impl AnyDataset for TransactionDataset { } fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> { + vec![ + &mut self.transactions_per_second, + &mut self.transactions_per_second_1w_sma, + &mut self.transactions_per_second_1m_sma, + ] + } + + fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> { + vec![ + &self.count_1w_sma, + &self.count_1m_sma, + &self.volume_1w_sma, + &self.volume_1m_sma, + &self.volume_in_dollars_1w_sma, + &self.volume_in_dollars_1m_sma, + ] + } + + fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> { vec![ &mut self.count_1w_sma, &mut self.count_1m_sma, @@ -246,12 +332,34 @@ impl AnyDataset for TransactionDataset { &mut self.volume_1m_sma, &mut self.volume_in_dollars_1w_sma, &mut self.volume_in_dollars_1m_sma, + ] + } + + fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> { + vec![ + &self.count_1d_sum_1w_sma, + &self.count_1d_sum_1m_sma, + &self.volume_1d_sum_1w_sma, + &self.volume_1d_sum_1m_sma, + &self.volume_in_dollars_1d_sum_1w_sma, + &self.volume_in_dollars_1d_sum_1m_sma, + &self.annualized_volume, + &self.annualized_volume_in_dollars, + &self.velocity, + ] + } + + fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> { + vec![ + &mut self.count_1d_sum_1w_sma, + &mut self.count_1d_sum_1m_sma, + &mut self.volume_1d_sum_1w_sma, + &mut self.volume_1d_sum_1m_sma, + &mut self.volume_in_dollars_1d_sum_1w_sma, + &mut self.volume_in_dollars_1d_sum_1m_sma, &mut self.annualized_volume, &mut self.annualized_volume_in_dollars, &mut self.velocity, - &mut self.transactions_per_second, - &mut self.transactions_per_second_1w_sma, - &mut self.transactions_per_second_1m_sma, ] } } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 06aaa7634..0b66a0770 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -12,7 +12,7 @@ pub use crate::{ datasets::OHLC, io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION}, structs::{ - Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, SerializedBTreeMap, + Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, MapValue, SerializedBTreeMap, SerializedVec, HEIGHT_MAP_CHUNK_SIZE, }, utils::{create_rpc, log, reset_logs}, diff --git a/parser/src/structs/bi_map.rs b/parser/src/structs/bi_map.rs index 11969869c..dd1472a5f 100644 --- a/parser/src/structs/bi_map.rs +++ b/parser/src/structs/bi_map.rs @@ -1,6 +1,6 @@ use std::{ iter::Sum, - ops::{Add, Div, Mul, RangeInclusive, Sub}, + ops::{Add, Div, Mul, Sub}, }; use allocative::Allocative; @@ -36,30 +36,30 @@ where } } - pub fn date_insert_sum_range(&mut self, date: Date, date_blocks_range: &RangeInclusive) - where - Value: Sum, - { - self.date - .insert(date, self.height.sum_range(date_blocks_range)); - } + // pub fn date_insert_sum_range(&mut self, date: Date, date_blocks_range: &RangeInclusive) + // where + // Value: Sum, + // { + // self.date + // .insert(date, self.height.sum_range(date_blocks_range)); + // } - pub fn multi_date_insert_sum_range( - &mut self, - dates: &[Date], - first_height: &mut DateMap, - last_height: &mut DateMap, - ) where - Value: Sum, - { - dates.iter().for_each(|date| { - let first_height = first_height.get_or_import(date).unwrap(); - let last_height = last_height.get_or_import(date).unwrap(); - let range = (*first_height)..=(*last_height); + // pub fn multi_date_insert_sum_range( + // &mut self, + // dates: &[Date], + // first_height: &mut DateMap, + // last_height: &mut DateMap, + // ) where + // Value: Sum, + // { + // dates.iter().for_each(|date| { + // let first_height = first_height.get_or_import(date).unwrap(); + // let last_height = last_height.get_or_import(date).unwrap(); + // let range = (*first_height)..=(*last_height); - self.date.insert(*date, self.height.sum_range(&range)); - }) - } + // self.date.insert(*date, self.height.sum_range(&range)); + // }) + // } pub fn multi_insert_const(&mut self, heights: &[Height], dates: &[Date], constant: Value) { self.height.multi_insert_const(heights, constant); @@ -176,6 +176,7 @@ where .multi_insert_percentage(dates, &mut divided.date, &mut divider.date); } + #[allow(unused)] pub fn multi_insert_cumulative( &mut self, heights: &[Height], @@ -192,27 +193,6 @@ where self.date.multi_insert_cumulative(dates, &mut source.date); } - pub fn multi_insert_last_x_sum( - &mut self, - heights: &[Height], - dates: &[Date], - source: &mut BiMap, - days: usize, - ) where - K: MapValue, - Value: LossyFrom, - Value: Add + Sub, - { - self.height.multi_insert_last_x_sum( - heights, - &mut source.height, - TARGET_BLOCKS_PER_DAY * days, - ); - - self.date - .multi_insert_last_x_sum(dates, &mut source.date, days); - } - pub fn multi_insert_simple_average( &mut self, heights: &[Height], @@ -251,6 +231,7 @@ where .multi_insert_net_change(dates, &mut source.date, days); } + #[allow(unused)] pub fn multi_insert_median( &mut self, heights: &[Height], diff --git a/parser/src/structs/config.rs b/parser/src/structs/config.rs index 02e19237d..d8b32215f 100644 --- a/parser/src/structs/config.rs +++ b/parser/src/structs/config.rs @@ -45,9 +45,8 @@ impl Config { const PATH: &'static str = "./config.toml"; pub fn import() -> color_eyre::Result { - let mut config_saved = - fs::read_to_string(Self::PATH).map_or(Config::default(), |contents| { - dbg!(&contents); + let mut config_saved = fs::read_to_string(Self::PATH) + .map_or(Config::default(), |contents| { toml::from_str(&contents).unwrap_or_default() }); diff --git a/parser/src/structs/date.rs b/parser/src/structs/date.rs index 358135deb..03b8b6362 100644 --- a/parser/src/structs/date.rs +++ b/parser/src/structs/date.rs @@ -1,4 +1,4 @@ -use std::{fmt, str::FromStr}; +use std::{cmp::Ordering, fmt, str::FromStr}; use allocative::{Allocative, Visitor}; use bincode::{ @@ -7,11 +7,13 @@ use bincode::{ error::{DecodeError, EncodeError}, BorrowDecode, Decode, Encode, }; -use chrono::{Datelike, Days, NaiveDate}; +use chrono::{Datelike, Days, NaiveDate, NaiveDateTime}; use derive_deref::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; -use super::{DateMapChunkId, MapKey}; +use crate::utils::ONE_DAY_IN_S; + +use super::{DateMapChunkId, MapKey, Timestamp}; const NUMBER_OF_UNSAFE_DATES: usize = 2; const MIN_YEAR: i32 = 2009; @@ -49,6 +51,28 @@ impl Date { pub fn difference_in_days_between(&self, older: Self) -> u32 { (**self - *older).num_days() as u32 } + + pub fn to_timestamp(self) -> Timestamp { + Timestamp::wrap(NaiveDateTime::from(*self).and_utc().timestamp() as u32) + } + + /// Returns value between 0.0 and 1.0 depending on its completion + /// + /// Any date before today (utc) will return 1.0 + /// + /// Any date after today (utc) will panic even though it should return 0.0, as it shouldn't happen in the code + /// + /// Any date equal to today will have a completion between 0.0 and 1.0 + pub fn get_day_completion(self) -> f64 { + let now = Timestamp::now(); + let today = Date::today(); + + match self.cmp(&today) { + Ordering::Less => 1.0, + Ordering::Equal => *(now - self.to_timestamp()) as f64 / ONE_DAY_IN_S as f64, + Ordering::Greater => unreachable!("0.0 but shouldn't be called"), + } + } } impl MapKey for Date { diff --git a/parser/src/structs/date_map.rs b/parser/src/structs/date_map.rs index d4e903808..420abb21e 100644 --- a/parser/src/structs/date_map.rs +++ b/parser/src/structs/date_map.rs @@ -1,17 +1,19 @@ +use std::iter::Sum; + use crate::{Date, HeightMap}; use super::{AnyMap, DateMapChunkId, GenericMap, Height, MapValue, SerializedBTreeMap}; pub type DateMap = GenericMap>; -impl DateMap +impl DateMap where - T: MapValue, + Value: MapValue, { pub fn multi_insert_last( &mut self, dates: &[Date], - source: &mut HeightMap, + source: &mut HeightMap, last_height: &mut DateMap, ) { dates.iter().for_each(|date| { @@ -23,6 +25,24 @@ where ); }); } + + pub fn multi_insert_sum_range( + &mut self, + dates: &[Date], + height_map: &HeightMap, + first_height: &mut DateMap, + last_height: &mut DateMap, + ) where + Value: Sum, + { + dates.iter().for_each(|date| { + let first_height = first_height.get_or_import(date).unwrap(); + let last_height = last_height.get_or_import(date).unwrap(); + let range = (*first_height)..=(*last_height); + + self.insert(*date, height_map.sum_range(&range)); + }) + } } pub trait AnyDateMap: AnyMap { diff --git a/parser/src/structs/generic_map.rs b/parser/src/structs/generic_map.rs index a53324880..04106b3df 100644 --- a/parser/src/structs/generic_map.rs +++ b/parser/src/structs/generic_map.rs @@ -413,11 +413,11 @@ where &mut self, keys: &[Key], source: &mut GenericMap, - transform: F, + mut transform: F, ) where SourceValue: MapValue, SourceSerialized: MapSerialized, - F: Fn(SourceValue, &Key) -> Value, + F: FnMut(SourceValue, &Key) -> Value, { keys.iter().for_each(|key| { self.insert(*key, transform(source.get_or_import(key).unwrap(), key)); diff --git a/parser/src/structs/map_value.rs b/parser/src/structs/map_value.rs index dfc1f53c2..1f0e08d53 100644 --- a/parser/src/structs/map_value.rs +++ b/parser/src/structs/map_value.rs @@ -23,6 +23,7 @@ pub trait MapValue: { } +impl MapValue for u8 {} impl MapValue for u16 {} impl MapValue for u32 {} impl MapValue for u64 {} diff --git a/parser/src/structs/serialized_btreemap.rs b/parser/src/structs/serialized_btreemap.rs index 64b3b0a95..2f1723ff1 100644 --- a/parser/src/structs/serialized_btreemap.rs +++ b/parser/src/structs/serialized_btreemap.rs @@ -1,10 +1,12 @@ -use std::{collections::BTreeMap, fmt::Debug}; +use std::{collections::BTreeMap, fmt::Debug, path::Path}; use allocative::Allocative; use bincode::{Decode, Encode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use super::{MapChunkId, MapKey, MapSerialized, MapValue}; +use crate::Serialization; + +use super::{DateMap, MapChunkId, MapKey, MapSerialized, MapValue}; #[derive(Debug, Default, Serialize, Deserialize, Encode, Decode, Allocative)] pub struct SerializedBTreeMap @@ -12,7 +14,37 @@ where Key: Ord, { version: u32, - map: BTreeMap, + pub map: BTreeMap, +} + +impl SerializedBTreeMap +where + Key: Ord, +{ + pub fn import_all(path: &Path, serialization: &Serialization) -> Self + where + Self: Debug + Serialize + DeserializeOwned + Encode + Decode, + ChunkId: MapChunkId, + Key: MapKey, + Value: MapValue, + { + let mut s = None; + + DateMap::::_read_dir(path, serialization) + .iter() + .for_each(|(_, path)| { + let map = serialization.import::(path).unwrap(); + + if s.is_none() { + s.replace(map); + } else { + #[allow(clippy::unnecessary_unwrap)] + s.as_mut().unwrap().map.extend(map.map); + } + }); + + s.unwrap() + } } impl MapSerialized for SerializedBTreeMap diff --git a/parser/src/structs/serialized_vec.rs b/parser/src/structs/serialized_vec.rs index 9678ef9fc..3d61a9738 100644 --- a/parser/src/structs/serialized_vec.rs +++ b/parser/src/structs/serialized_vec.rs @@ -1,15 +1,42 @@ -use std::{cmp::Ordering, collections::BTreeMap, fmt::Debug}; +use std::{cmp::Ordering, collections::BTreeMap, fmt::Debug, path::Path}; use allocative::Allocative; use bincode::{Decode, Encode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use super::{MapChunkId, MapKey, MapSerialized, MapValue}; +use crate::Serialization; + +use super::{HeightMap, MapChunkId, MapKey, MapSerialized, MapValue}; #[derive(Debug, Default, Serialize, Deserialize, Encode, Decode, Allocative)] pub struct SerializedVec { version: u32, - map: Vec, + pub map: Vec, +} + +impl SerializedVec { + pub fn import_all(path: &Path, serialization: &Serialization) -> Self + where + Self: Debug + Serialize + DeserializeOwned + Encode + Decode, + Value: MapValue, + { + let mut s = None; + + HeightMap::::_read_dir(path, serialization) + .iter() + .for_each(|(_, path)| { + let mut map = serialization.import::(path).unwrap(); + + if s.is_none() { + s.replace(map); + } else { + #[allow(clippy::unnecessary_unwrap)] + s.as_mut().unwrap().map.append(&mut map.map); + } + }); + + s.unwrap() + } } impl MapSerialized for SerializedVec diff --git a/parser/src/structs/timestamp.rs b/parser/src/structs/timestamp.rs index d5b5585cc..04ce1b1ad 100644 --- a/parser/src/structs/timestamp.rs +++ b/parser/src/structs/timestamp.rs @@ -36,6 +36,10 @@ impl Timestamp { Self(timestamp) } + pub fn now() -> Self { + Self(chrono::offset::Utc::now().timestamp() as u32) + } + pub fn to_date(self) -> Date { Date::wrap( Utc.timestamp_opt(i64::from(self.0), 0) diff --git a/server/src/api/handlers/dataset.rs b/server/src/api/handlers/dataset.rs index 903fd5a11..b0ab851f8 100644 --- a/server/src/api/handlers/dataset.rs +++ b/server/src/api/handlers/dataset.rs @@ -20,11 +20,15 @@ use crate::{ AppState, }; -use super::response::{typed_value_to_response, value_to_response}; +use super::{ + extension::Extension, + response::{typed_value_to_response, value_to_response}, +}; #[derive(Deserialize)] pub struct Params { chunk: Option, + all: Option, } pub async fn dataset_handler( @@ -46,43 +50,44 @@ pub async fn dataset_handler( } } +const DATE_PREFIX: &str = "date-to-"; +const HEIGHT_PREFIX: &str = "height-to-"; + fn _dataset_handler( headers: HeaderMap, Path(path): Path, query: Query, AppState { routes }: AppState, ) -> color_eyre::Result { + if query.chunk.is_some() && query.all.is_some() { + return Err(eyre!("chunk and all are exclusive")); + } + log(&format!( - "{}{}", + "{}{}{}", path, query.chunk.map_or("".to_string(), |chunk| format!( "{}{chunk}", "?chunk=".bright_black() + )), + query.all.map_or("".to_string(), |all| format!( + "{}{all}", + "?all=".bright_black() )) )); - let date_prefix = "date-to-"; - let height_prefix = "height-to-"; - - let (kind, route) = if path.starts_with(date_prefix) { - ( - Kind::Date, - routes.date.get(&replace_dash_by_underscore( - path.strip_prefix(date_prefix).unwrap(), - )), - ) - } else if path.starts_with(height_prefix) { - ( - Kind::Height, - routes.height.get(&replace_dash_by_underscore( - path.strip_prefix(height_prefix).unwrap(), - )), - ) + let (kind, id, route) = if path.starts_with(DATE_PREFIX) { + let id = convert_path_to_id(path.strip_prefix(DATE_PREFIX).unwrap()); + let route = routes.date.get(&id); + (Kind::Date, id, route) + } else if path.starts_with(HEIGHT_PREFIX) { + let id = convert_path_to_id(path.strip_prefix(HEIGHT_PREFIX).unwrap()); + let route = routes.height.get(&id); + (Kind::Height, id, route) } else { - ( - Kind::Last, - routes.last.get(&replace_dash_by_underscore(&path)), - ) + let id = convert_path_to_id(&path); + let route = routes.last.get(&id); + (Kind::Last, id, route) }; if route.is_none() { @@ -93,25 +98,28 @@ fn _dataset_handler( let mut chunk = None; - match kind { - Kind::Date => { - let datasets = DateMap::::_read_dir(&route.file_path, &route.serialization); + if query.all.map_or(true, |b| !b) { + match kind { + Kind::Date => { + let datasets = DateMap::::_read_dir(&route.file_path, &route.serialization); - process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?; - } - Kind::Height => { - let datasets = HeightMap::::_read_dir(&route.file_path, &route.serialization); + process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?; + } + Kind::Height => { + let datasets = + HeightMap::::_read_dir(&route.file_path, &route.serialization); - process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?; - } - Kind::Last => { - if !route.values_type.ends_with("Value") { - route.file_path.set_extension(COMPRESSED_BIN_EXTENSION); - } else { - route.file_path.set_extension(JSON_EXTENSION); + process_datasets(&headers, kind, &mut chunk, &mut route, query, datasets)?; + } + Kind::Last => { + if !route.values_type.ends_with("Value") { + route.file_path.set_extension(COMPRESSED_BIN_EXTENSION); + } else { + route.file_path.set_extension(JSON_EXTENSION); + } } } - }; + } let (date, response) = headers.check_if_modified_since(&route.file_path).unwrap(); @@ -121,18 +129,22 @@ fn _dataset_handler( let type_name = route.values_type.split("::").last().unwrap(); + let extension = Extension::from(&std::path::PathBuf::from(&path)); + let mut response = match type_name { - "u8" => typed_value_to_response::(kind, &route, chunk)?, - "u16" => typed_value_to_response::(kind, &route, chunk)?, - "u32" => typed_value_to_response::(kind, &route, chunk)?, - "u64" => typed_value_to_response::(kind, &route, chunk)?, - "usize" => typed_value_to_response::(kind, &route, chunk)?, - "f32" => typed_value_to_response::(kind, &route, chunk)?, - "f64" => typed_value_to_response::(kind, &route, chunk)?, - "OHLC" => typed_value_to_response::(kind, &route, chunk)?, - "Date" => typed_value_to_response::(kind, &route, chunk)?, - "Height" => typed_value_to_response::(kind, &route, chunk)?, - "Value" => value_to_response::(Json::import(&route.file_path)?), + "u8" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "u16" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "u32" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "u64" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "usize" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "f32" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "f64" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "OHLC" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "Date" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "Height" => typed_value_to_response::(kind, &route, chunk, id, extension)?, + "Value" => { + value_to_response::(Json::import(&route.file_path)?, extension) + } _ => panic!("Incompatible type: {type_name}"), }; @@ -142,8 +154,8 @@ fn _dataset_handler( Ok(response) } -fn replace_dash_by_underscore(s: &str) -> String { - s.replace('-', "_") +fn convert_path_to_id(s: &str) -> String { + Extension::remove_extension(s).replace('-', "_") } fn process_datasets( @@ -174,7 +186,6 @@ where } let path = path.unwrap(); - route.file_path = path.clone(); let offset = match kind { diff --git a/server/src/api/handlers/extension.rs b/server/src/api/handlers/extension.rs new file mode 100644 index 000000000..1516c1d90 --- /dev/null +++ b/server/src/api/handlers/extension.rs @@ -0,0 +1,41 @@ +use std::path::Path; + +#[derive(PartialEq, Eq)] +pub enum Extension { + CSV, + JSON, +} + +impl Extension { + pub fn from(path: &Path) -> Option { + if let Some(extension) = path.extension() { + let extension = extension.to_str().unwrap(); + + if extension == Self::CSV.to_str() { + Some(Self::CSV) + } else if extension == Self::JSON.to_str() { + Some(Self::JSON) + } else { + None + } + } else { + None + } + } + + pub fn to_str(&self) -> &str { + match self { + Extension::CSV => "csv", + Extension::JSON => "json", + } + } + + pub fn to_dot_str(&self) -> String { + format!(".{}", self.to_str()) + } + + pub fn remove_extension(s: &str) -> String { + s.replace(&Self::CSV.to_dot_str(), "") + .replace(&Self::JSON.to_dot_str(), "") + } +} diff --git a/server/src/api/handlers/fallback.rs b/server/src/api/handlers/fallback.rs index eecd1b3c7..158261d07 100644 --- a/server/src/api/handlers/fallback.rs +++ b/server/src/api/handlers/fallback.rs @@ -3,14 +3,17 @@ use reqwest::header::HOST; use crate::AppState; -use super::response::generic_to_reponse; +use super::response::{generic_to_reponse, update_reponse_headers}; pub async fn fallback(headers: HeaderMap, State(app_state): State) -> Response { - generic_to_reponse( - app_state - .routes - .to_full_paths(headers[HOST].to_str().unwrap().to_string()), - None, + update_reponse_headers( + generic_to_reponse( + app_state + .routes + .to_full_paths(headers[HOST].to_str().unwrap().to_string()), + None, + ), 60, + None, ) } diff --git a/server/src/api/handlers/mod.rs b/server/src/api/handlers/mod.rs index 78a45c0da..33474ff3a 100644 --- a/server/src/api/handlers/mod.rs +++ b/server/src/api/handlers/mod.rs @@ -1,5 +1,7 @@ mod dataset; +mod extension; mod fallback; + mod response; pub use dataset::*; diff --git a/server/src/api/handlers/response.rs b/server/src/api/handlers/response.rs index fef38f651..a18c9b0df 100644 --- a/server/src/api/handlers/response.rs +++ b/server/src/api/handlers/response.rs @@ -1,8 +1,8 @@ -use std::{fmt::Debug, path::Path}; +use std::fmt::Debug; use axum::response::{IntoResponse, Json, Response}; use bincode::Decode; -use parser::{Date, SerializedBTreeMap, SerializedVec}; +use parser::{Date, MapValue, SerializedBTreeMap, SerializedVec}; use serde::de::DeserializeOwned; use serde::Serialize; @@ -11,6 +11,8 @@ use crate::{ header_map::HeaderMapUtils, }; +use super::extension::Extension; + #[derive(Serialize)] struct WrappedDataset<'a, T> where @@ -25,70 +27,119 @@ pub fn typed_value_to_response( kind: Kind, route: &Route, chunk: Option, + id: String, + extension: Option, ) -> color_eyre::Result where - T: Serialize + Debug + DeserializeOwned + Decode, + T: Serialize + Debug + DeserializeOwned + Decode + MapValue, { Ok(match kind { - Kind::Date => dataset_to_response( - route - .serialization - .import::>(Path::new(&route.file_path))?, - chunk.unwrap(), - ), - Kind::Height => dataset_to_response( - route - .serialization - .import::>(Path::new(&route.file_path))?, - chunk.unwrap(), - ), + Kind::Date => { + let dataset = if chunk.is_some() { + route + .serialization + .import::>(&route.file_path)? + } else { + SerializedBTreeMap::::import_all(&route.file_path, &route.serialization) + }; + + if extension == Some(Extension::CSV) { + let mut csv = format!("date,{}\n", id); + + dataset.map.iter().for_each(|(k, v)| { + csv += &format!("{},{:?}\n", k, v); + }); + + string_to_response(csv, extension) + } else { + dataset_to_response(dataset, chunk, extension) + } + } + Kind::Height => { + let dataset = if chunk.is_some() { + route + .serialization + .import::>(&route.file_path)? + } else { + SerializedVec::::import_all(&route.file_path, &route.serialization) + }; + + if extension == Some(Extension::CSV) { + let mut csv = format!("height,{}\n", id); + + let starting_height = chunk.map_or(0, |chunk| chunk.id); + + dataset.map.iter().enumerate().for_each(|(k, v)| { + csv += &format!("{},{:?}\n", starting_height + k, v); + }); + + string_to_response(csv, extension) + } else { + dataset_to_response(dataset, chunk, extension) + } + } Kind::Last => value_to_response( - route - .serialization - .import::(Path::new(&route.file_path))?, + route.serialization.import::(&route.file_path)?, + extension, ), }) } -pub fn value_to_response(value: T) -> Response -where - T: Serialize, -{ - generic_to_reponse(value, None, 1) +pub fn string_to_response(s: String, extension: Option) -> Response { + update_reponse_headers(s.into_response(), 5, extension) } -fn dataset_to_response(dataset: T, chunk: Chunk) -> Response +pub fn value_to_response(value: T, extension: Option) -> Response where T: Serialize, { - generic_to_reponse(dataset, Some(chunk), 5) + update_reponse_headers(generic_to_reponse(value, None), 1, extension) } -pub fn generic_to_reponse(generic: T, chunk: Option, cache_time: u64) -> Response +fn dataset_to_response( + dataset: T, + chunk: Option, + extension: Option, +) -> Response where T: Serialize, { - let mut response = { - if let Some(chunk) = chunk { - Json(WrappedDataset { - source: "https://kibo.money", - chunk, - dataset: generic, - }) - .into_response() - } else { - Json(generic).into_response() - } - }; + update_reponse_headers(generic_to_reponse(dataset, chunk), 5, extension) +} +pub fn generic_to_reponse(generic: T, chunk: Option) -> Response +where + T: Serialize, +{ + if let Some(chunk) = chunk { + Json(WrappedDataset { + source: "https://kibo.money", + chunk, + dataset: generic, + }) + .into_response() + } else { + Json(generic).into_response() + } +} + +pub fn update_reponse_headers( + mut response: Response, + cache_time: u64, + extension: Option, +) -> Response { let headers = response.headers_mut(); let max_age = cache_time; let stale_while_revalidate = 2 * max_age; headers.insert_cors(); - headers.insert_content_type_application_json(); headers.insert_cache_control_revalidate(max_age, stale_while_revalidate); + match extension { + Some(Extension::CSV) => headers.insert_content_type_text_csv(), + _ => headers.insert_content_type_application_json(), + } + response } diff --git a/server/src/api/structs/routes.rs b/server/src/api/structs/routes.rs index 20f0d870d..fc33fbf5d 100644 --- a/server/src/api/structs/routes.rs +++ b/server/src/api/structs/routes.rs @@ -24,7 +24,7 @@ pub struct Route { pub struct Routes(pub Grouped>); const INPUTS_PATH: &str = "./in"; -const WEBSITE_TYPES_PATH: &str = "../website/types"; +const WEBSITE_TYPES_PATH: &str = "../website/scripts/types"; impl Routes { pub fn build() -> Self { diff --git a/server/src/header_map.rs b/server/src/header_map.rs index 8f09ddd66..f883006ea 100644 --- a/server/src/header_map.rs +++ b/server/src/header_map.rs @@ -34,6 +34,8 @@ pub trait HeaderMapUtils { fn insert_cache_control_revalidate(&mut self, max_age: u64, stale_while_revalidate: u64); fn insert_last_modified(&mut self, date: DateTime); + fn insert_content_disposition_attachment(&mut self); + fn insert_content_type(&mut self, path: &Path); fn insert_content_type_image_icon(&mut self); fn insert_content_type_image_jpeg(&mut self); @@ -43,6 +45,7 @@ pub trait HeaderMapUtils { fn insert_content_type_application_manifest_json(&mut self); fn insert_content_type_application_pdf(&mut self); fn insert_content_type_text_css(&mut self); + fn insert_content_type_text_csv(&mut self); fn insert_content_type_text_html(&mut self); fn insert_content_type_text_plain(&mut self); fn insert_content_type_font_woff2(&mut self); @@ -181,6 +184,10 @@ impl HeaderMapUtils for HeaderMap { ); } + fn insert_content_disposition_attachment(&mut self) { + self.insert(header::CONTENT_DISPOSITION, "attachment".parse().unwrap()); + } + fn insert_content_type_application_json(&mut self) { self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap()); } @@ -200,6 +207,10 @@ impl HeaderMapUtils for HeaderMap { self.insert(header::CONTENT_TYPE, "text/css".parse().unwrap()); } + fn insert_content_type_text_csv(&mut self) { + self.insert(header::CONTENT_TYPE, "text/csv".parse().unwrap()); + } + fn insert_content_type_text_html(&mut self) { self.insert(header::CONTENT_TYPE, "text/html".parse().unwrap()); } diff --git a/website/index.html b/website/index.html index 3826e9e87..403588efe 100644 --- a/website/index.html +++ b/website/index.html @@ -13,7 +13,7 @@ /> - + @@ -395,6 +395,11 @@ } } + details { + /* content-visibility: auto; + contain-intrinsic-size: auto none; */ + } + fieldset { > div.field { display: flex; @@ -1028,231 +1033,6 @@ max-width: 32rem; } } - - > #dashboards { - z-index: 50; - margin: -0.5px calc(-1.5rem - 1px); - display: flex; - - > details { - /* border: 1px; */ - - > summary { - margin: -0.5px; - display: block; - padding: 0.5rem 1.5rem; - text-align: left; - border: 1px; - - details[open] > & { - border-bottom-style: dashed !important; - border-bottom-width: 1px; - } - } - - > table { - border: 1px; - border-top-width: 0px; - margin: -0.5px; - width: calc(100% + 1px); - border-collapse: separate; - - table-layout: auto; - padding: 0.375rem 1.5rem; - - tr { - /* border: 1px; */ - - &:hover, - &:hover * { - color: var(--orange) !important; - } - - td { - padding: 0.125rem 0; - text-align: right; - - &:first-child { - text-align: left; - color: var(--off-color); - } - - > i { - color: var(--off-color); - } - } - } - } - } - } - - > #charts { - display: flex; - flex-direction: column; - flex: 1; - min-height: 0; - - > #legend { - display: flex; - align-items: center; - gap: 1.5rem; - margin: -1rem -1.5rem; - padding: 1rem 1.5rem; - overflow-x: auto; - min-width: 0; - - > div { - flex: 0; - display: flex; - align-items: center; - - > label { - margin: -0.375rem 0; - color: var(--color); - - &:has(input:not(:checked)) { - color: var(--off-color); - - > span.main > span.name { - text-decoration-style: wavy; - text-decoration-thickness: 1.5px; - text-decoration-color: var(--color); - text-decoration-line: line-through; - } - - &:hover { - * { - color: var(--off-color) !important; - } - - > span.main > span.name { - text-decoration-color: var(--orange) !important; - } - } - } - } - - > a { - padding: 0.375rem; - margin: -0.375rem; - - > svg { - /* padding: 0.375rem; */ - margin: 0rem; - width: 1rem; - height: 1rem; - } - } - } - } - - > #chart-list { - margin-top: 1rem; - position: relative; - margin-left: -1.5rem /* -24px */; - margin-right: -2rem /* -32px */; - display: flex; - flex-direction: column; - flex: 1; - min-height: 0; - z-index: 20; - - > .chart-wrapper { - height: 100%; - position: relative; - min-height: 0px; - width: 100%; - cursor: crosshair; - - &:has(+ .chart-wrapper:not([hidden])) { - height: calc(100% - 62px); - } - - > fieldset { - pointer-events: none; - position: absolute; - left: 0px; - top: 0px; - z-index: 10; - display: flex; - align-items: center; - padding-left: 1.5rem /* 24px */; - padding-right: 1.5rem /* 24px */; - font-size: var(--font-size-xs); - line-height: var(--line-height-xs); - - > * + * { - margin-left: 0.5rem; /* 8px */ - } - - > * + span { - color: var(--off-color); - } - } - - > .chart-div { - width: 100%; - height: 100%; - } - } - } - - > #timescale { - margin-top: 1rem; - display: flex; - overflow: hidden; - z-index: 50; - - @media (max-width: 767px) { - margin-bottom: -1.5rem; - border-top: 1px; - margin-left: -1.5rem; - margin-right: -1.5rem; - } - - @media (min-width: 768px) { - border-radius: 9999px; - border: 1px; - } - - > button { - flex: 0; - } - - > div { - min-width: 0; - flex: 1; - display: flex; - position: relative; - - > #timescale-date-buttons, - > #timescale-height-buttons { - display: flex; - overflow-x: auto; - - > button { - color: var(--off-color); - flex-shrink: 0; - flex-grow: 1; - padding: 0.5rem; - white-space: nowrap; - min-width: 5rem; - } - } - } - - > *:not(:first-child):not(:last-child) { - border-left: 1px; - border-right: 1px; - } - } - } - - > object { - margin: -1.5rem; - flex: 1; - z-index: 100; - } } #share-div { @@ -1600,7 +1380,7 @@ } if ("serviceWorker" in navigator) { - navigator.serviceWorker.register("/service-worker.js"); + navigator.serviceWorker.register("/scripts/service-worker.js"); } @@ -2444,7 +2224,7 @@ - +