Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-03 10:57:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: added ratio and co datasets

Browse Source
This commit is contained in:
k
2024-07-10 18:34:01 +02:00
parent 6976f5af0f
commit 4a82ee0b05
20 changed files with 624 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
parser/src/actions/export.rs
View File

@@ -25,7 +25,8 @@ pub fn export(
date,
}: ExportedData,
) -> color_eyre::Result<()> {
log("Exporting... (Don't close !!)");
log("Exporting...");
log("WARNING: NOT SAFE TO STOP !!!");
time("Total save time", || -> color_eyre::Result<()> {
time("Datasets saved", || datasets.export())?;
@@ -43,5 +44,7 @@ pub fn export(
Ok(())
})?;
log("Export done - Safe to stop now");
Ok(())
}

8
parser/src/actions/iter_blocks.rs
View File

@@ -166,9 +166,11 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
));
if first_unsafe_heights.computed <= last_height {
datasets.compute(ComputeData {
dates: &processed_dates.into_iter().collect_vec(),
heights: &processed_heights.into_iter().collect_vec(),
time("Computing datasets", || {
datasets.compute(ComputeData {
dates: &processed_dates.into_iter().collect_vec(),
heights: &processed_heights.into_iter().collect_vec(),
})
});
}

13
parser/src/actions/parse.rs
View File

@@ -476,9 +476,11 @@ pub fn parse(
AddressRealizedData::default(input_address_data)
});
let previous_price = input_block_data.price;
// MUST be after `or_insert_with`
let address_realized_profit_or_loss = input_address_data
.send(input_amount, block_price, input_block_data.price)
input_address_data
.send(input_amount, previous_price)
.unwrap_or_else(|_| {
dbg!(
input_address_index,
@@ -494,8 +496,11 @@ pub fn parse(
panic!()
});
input_address_realized_data
.send(input_amount, address_realized_profit_or_loss);
input_address_realized_data.send(
input_amount,
block_price,
previous_price,
);
};
is_tx_data_from_cached_puts && input_tx_data.is_empty()

4
parser/src/datasets/cointime.rs
View File

@@ -86,7 +86,7 @@ impl CointimeDataset {
cointime_value_stored: BiMap::new_bin(1, &f("cointime_value_stored")),
concurrent_liveliness: BiMap::new_bin(1, &f("concurrent_liveliness")),
concurrent_liveliness_2w_median: BiMap::new_bin(
1,
2,
&f("concurrent_liveliness_2w_median"),
),
cumulative_coinblocks_created: BiMap::new_bin(1, &f("cumulative_coinblocks_created")),
@@ -100,7 +100,7 @@ impl CointimeDataset {
liveliness: BiMap::new_bin(1, &f("liveliness")),
liveliness_net_change: BiMap::new_bin(1, &f("liveliness_net_change")),
liveliness_net_change_2w_median: BiMap::new_bin(
1,
2,
&f("liveliness_net_change_2w_median"),
),
producerness: BiMap::new_bin(1, &f("producerness")),

2
parser/src/datasets/price/mod.rs
View File

@@ -24,6 +24,7 @@ pub struct PriceDatasets {
kraken_1mn: Option<BTreeMap<u32, OHLC>>,
binance_1mn: Option<BTreeMap<u32, OHLC>>,
binance_har: Option<BTreeMap<u32, OHLC>>,
satonomics_by_height: BTreeMap<usize, Option<BTreeMap<usize, OHLC>>>,
// Inserted
pub ohlcs: BiMap<OHLC>,
@@ -74,6 +75,7 @@ impl PriceDatasets {
binance_har: None,
kraken_1mn: None,
kraken_daily: None,
satonomics_by_height: BTreeMap::default(),
ohlcs: BiMap::new_json(1, &format!("{price_path}/ohlc")),
closes: BiMap::new_json(1, &f("close")),

42
parser/src/datasets/subs/capitalization.rs
View File

@@ -7,6 +7,8 @@ use crate::{
utils::ONE_MONTH_IN_DAYS,
};
use super::RatioDataset;
#[derive(Default, Allocative)]
pub struct CapitalizationDataset {
min_initial_states: MinInitialStates,
@@ -16,8 +18,8 @@ pub struct CapitalizationDataset {
// Computed
pub realized_price: BiMap<f32>,
mvrv: BiMap<f32>,
realized_cap_1m_net_change: BiMap<f32>,
ratio: RatioDataset,
}
impl CapitalizationDataset {
@@ -30,7 +32,7 @@ impl CapitalizationDataset {
realized_cap: BiMap::new_bin(1, &f("realized_cap")),
realized_cap_1m_net_change: BiMap::new_bin(1, &f("realized_cap_1m_net_change")),
realized_price: BiMap::new_bin(1, &f("realized_price")),
mvrv: BiMap::new_bin(1, &f("mvrv")),
ratio: RatioDataset::import(parent_path, "realized_price")?,
};
s.min_initial_states
@@ -61,10 +63,12 @@ impl CapitalizationDataset {
pub fn compute(
&mut self,
&ComputeData { heights, dates }: &ComputeData,
compute_data: &ComputeData,
closes: &mut BiMap<f32>,
cohort_supply: &mut BiMap<f64>,
) {
let &ComputeData { heights, dates } = compute_data;
self.realized_price.multi_insert_divide(
heights,
dates,
@@ -72,21 +76,15 @@ impl CapitalizationDataset {
cohort_supply,
);
self.mvrv.height.multi_insert_divide(
heights,
&mut closes.height,
&mut self.realized_price.height,
);
self.mvrv
.date
.multi_insert_divide(dates, &mut closes.date, &mut self.realized_price.date);
self.realized_cap_1m_net_change.multi_insert_net_change(
heights,
dates,
&mut self.realized_cap,
ONE_MONTH_IN_DAYS,
)
);
self.ratio
.compute(compute_data, closes, &mut self.realized_price);
}
}
@@ -104,18 +102,20 @@ impl AnyDataset for CapitalizationDataset {
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.realized_price,
&self.mvrv,
let mut v = vec![
&self.realized_price as &(dyn AnyBiMap + Send + Sync),
&self.realized_cap_1m_net_change,
]
];
v.append(&mut self.ratio.to_computed_bi_map_vec());
v
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.realized_price,
&mut self.mvrv,
let mut v = vec![
&mut self.realized_price as &mut dyn AnyBiMap,
&mut self.realized_cap_1m_net_change,
]
];
v.append(&mut self.ratio.to_computed_mut_bi_map_vec());
v
}
}

2
parser/src/datasets/subs/input.rs
View File

@@ -10,8 +10,10 @@ use crate::{
pub struct InputSubDataset {
min_initial_states: MinInitialStates,
// Inserted
pub count: BiMap<u64>,
pub volume: BiMap<f64>,
// Computed
// add inputs_per_second
}

2
parser/src/datasets/subs/mod.rs
View File

@@ -2,6 +2,7 @@ use allocative::Allocative;
mod capitalization;
mod input;
mod ratio;
// mod output;
mod price_paid;
mod realized;
@@ -11,6 +12,7 @@ mod utxo;
pub use capitalization::*;
pub use input::*;
pub use ratio::*;
// pub use output::*;
pub use price_paid::*;
pub use realized::*;

207
parser/src/datasets/subs/ratio.rs Normal file
View File

@@ -0,0 +1,207 @@
use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, MinInitialStates},
structs::{AnyBiMap, BiMap},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
#[derive(Default, Allocative)]
pub struct RatioDataset {
min_initial_states: MinInitialStates,
// Computed
ratio: BiMap<f32>,
ratio_1w_sma: BiMap<f32>,
ratio_1m_sma: BiMap<f32>,
ratio_1y_sma: BiMap<f32>,
ratio_1y_sma_momentum_oscillator: BiMap<f32>,
ratio_99p: BiMap<f32>,
ratio_99_5p: BiMap<f32>,
ratio_99_9p: BiMap<f32>,
ratio_1p: BiMap<f32>,
ratio_0_5p: BiMap<f32>,
ratio_0_1p: BiMap<f32>,
price_99p: BiMap<f32>,
price_99_5p: BiMap<f32>,
price_99_9p: BiMap<f32>,
price_1p: BiMap<f32>,
price_0_5p: BiMap<f32>,
price_0_1p: BiMap<f32>,
}
impl RatioDataset {
pub fn import(parent_path: &str, name: &str) -> color_eyre::Result<Self> {
let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
let mut s = Self {
min_initial_states: MinInitialStates::default(),
ratio: BiMap::new_bin(1, &f_ratio("ratio")),
ratio_1w_sma: BiMap::new_bin(1, &f_ratio("ratio_1w_sma")),
ratio_1m_sma: BiMap::new_bin(1, &f_ratio("ratio_1m_sma")),
ratio_1y_sma: BiMap::new_bin(1, &f_ratio("ratio_1y_sma")),
ratio_1y_sma_momentum_oscillator: BiMap::new_bin(
1,
&f_ratio("ratio_1y_sma_momentum_oscillator"),
),
ratio_99p: BiMap::new_bin(1, &f_ratio("ratio_99p")),
ratio_99_5p: BiMap::new_bin(1, &f_ratio("ratio_99_5p")),
ratio_99_9p: BiMap::new_bin(1, &f_ratio("ratio_99_9p")),
ratio_1p: BiMap::new_bin(1, &f_ratio("ratio_1p")),
ratio_0_5p: BiMap::new_bin(1, &f_ratio("ratio_0_5p")),
ratio_0_1p: BiMap::new_bin(1, &f_ratio("ratio_0_1p")),
price_99p: BiMap::new_bin(1, &f_price("99p")),
price_99_5p: BiMap::new_bin(1, &f_price("99_5p")),
price_99_9p: BiMap::new_bin(1, &f_price("99_9p")),
price_1p: BiMap::new_bin(1, &f_price("1p")),
price_0_5p: BiMap::new_bin(1, &f_price("0_5p")),
price_0_1p: BiMap::new_bin(1, &f_price("0_1p")),
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
Ok(s)
}
pub fn compute(
&mut self,
&ComputeData { heights, dates }: &ComputeData,
market_price: &mut BiMap<f32>,
other_price: &mut BiMap<f32>,
) {
self.ratio.height.multi_insert_divide(
heights,
&mut market_price.height,
&mut other_price.height,
);
self.ratio
.date
.multi_insert_divide(dates, &mut market_price.date, &mut other_price.date);
self.ratio_1w_sma.multi_insert_simple_average(
heights,
dates,
&mut self.ratio,
ONE_WEEK_IN_DAYS,
);
self.ratio_1m_sma.multi_insert_simple_average(
heights,
dates,
&mut self.ratio,
ONE_MONTH_IN_DAYS,
);
self.ratio_1m_sma.multi_insert_simple_average(
heights,
dates,
&mut self.ratio,
ONE_MONTH_IN_DAYS,
);
self.ratio_1y_sma.multi_insert_simple_average(
heights,
dates,
&mut self.ratio,
ONE_YEAR_IN_DAYS,
);
self.ratio_1y_sma_momentum_oscillator
.height
.multi_insert_complex_transform(heights, &mut self.ratio.height, |(ratio, height)| {
(ratio / self.ratio_1y_sma.height.get_or_import(height)) - 1.0
});
self.ratio_1y_sma_momentum_oscillator
.date
.multi_insert_complex_transform(dates, &mut self.ratio.date, |(ratio, date, _)| {
(ratio / self.ratio_1y_sma.date.get_or_import(date).unwrap()) - 1.0
});
self.ratio.multi_insert_percentile(
heights,
dates,
vec![
(&mut self.ratio_99p, 0.99),
(&mut self.ratio_99_5p, 0.995),
(&mut self.ratio_99_9p, 0.999),
(&mut self.ratio_1p, 0.1),
(&mut self.ratio_0_5p, 0.005),
(&mut self.ratio_0_1p, 0.001),
],
None,
);
self.price_99p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99p);
self.price_99_5p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99_5p);
self.price_99_9p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_99_9p);
self.price_1p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_1p);
self.price_0_5p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_0_5p);
self.price_0_1p
.multi_insert_multiply(heights, dates, market_price, &mut self.ratio_0_1p);
}
}
impl AnyDataset for RatioDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.ratio,
&self.ratio_1w_sma,
&self.ratio_1m_sma,
&self.ratio_1y_sma,
&self.ratio_1y_sma_momentum_oscillator,
&self.ratio_99p,
&self.ratio_99_5p,
&self.ratio_99_9p,
&self.ratio_1p,
&self.ratio_0_5p,
&self.ratio_0_1p,
&self.price_99p,
&self.price_99_5p,
&self.price_99_9p,
&self.price_1p,
&self.price_0_5p,
&self.price_0_1p,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.ratio,
&mut self.ratio_1w_sma,
&mut self.ratio_1m_sma,
&mut self.ratio_1y_sma,
&mut self.ratio_1y_sma_momentum_oscillator,
&mut self.ratio_99p,
&mut self.ratio_99_5p,
&mut self.ratio_99_9p,
&mut self.ratio_1p,
&mut self.ratio_0_5p,
&mut self.ratio_0_1p,
&mut self.price_99p,
&mut self.price_99_5p,
&mut self.price_99_9p,
&mut self.price_1p,
&mut self.price_0_5p,
&mut self.price_0_1p,
]
}
}

55
parser/src/datasets/subs/realized.rs
View File

@@ -3,11 +3,10 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::RealizedState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Price},
utils::ONE_MONTH_IN_DAYS,
};
/// TODO: Fix fees not taken into account ?
#[derive(Default, Allocative)]
pub struct RealizedSubDataset {
min_initial_states: MinInitialStates,
@@ -15,6 +14,9 @@ pub struct RealizedSubDataset {
// Inserted
realized_profit: BiMap<f32>,
realized_loss: BiMap<f32>,
value_destroyed: BiMap<f32>,
value_created: BiMap<f32>,
spent_output_profit_ratio: BiMap<f32>,
// Computed
negative_realized_loss: BiMap<f32>,
@@ -35,6 +37,10 @@ impl RealizedSubDataset {
realized_profit: BiMap::new_bin(1, &f("realized_profit")),
realized_loss: BiMap::new_bin(1, &f("realized_loss")),
value_created: BiMap::new_bin(1, &f("value_created")),
value_destroyed: BiMap::new_bin(1, &f("value_destroyed")),
spent_output_profit_ratio: BiMap::new_bin(2, &f("spent_output_profit_ratio")),
negative_realized_loss: BiMap::new_bin(2, &f("negative_realized_loss")),
net_realized_profit_and_loss: BiMap::new_bin(1, &f("net_realized_profit_and_loss")),
net_realized_profit_and_loss_to_market_cap_ratio: BiMap::new_bin(
@@ -78,12 +84,41 @@ impl RealizedSubDataset {
.height
.insert(height, height_state.realized_loss.to_dollar() as f32);
self.value_created
.height
.insert(height, height_state.value_created.to_dollar() as f32);
self.value_destroyed
.height
.insert(height, height_state.value_destroyed.to_dollar() as f32);
self.spent_output_profit_ratio.height.insert(height, {
if height_state.value_destroyed > Price::ZERO {
(height_state.value_created.to_cent() as f64
/ height_state.value_destroyed.to_cent() as f64) as f32
} else {
1.0
}
});
if is_date_last_block {
self.realized_profit
.date_insert_sum_range(date, date_blocks_range);
self.realized_loss
.date_insert_sum_range(date, date_blocks_range);
self.value_created
.date_insert_sum_range(date, date_blocks_range);
self.value_destroyed
.date_insert_sum_range(date, date_blocks_range);
self.spent_output_profit_ratio.date.insert(
date,
self.value_created.height.sum_range(date_blocks_range)
/ self.value_destroyed.height.sum_range(date_blocks_range),
);
}
}
@@ -145,11 +180,23 @@ impl AnyDataset for RealizedSubDataset {
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.realized_loss, &self.realized_profit]
vec![
&self.realized_loss,
&self.realized_profit,
&self.value_created,
&self.value_destroyed,
&self.spent_output_profit_ratio,
]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.realized_loss, &mut self.realized_profit]
vec![
&mut self.realized_loss,
&mut self.realized_profit,
&mut self.value_created,
&mut self.value_destroyed,
&mut self.spent_output_profit_ratio,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {

39
parser/src/states/cohorts_states/address/cohorts_realized_states.rs
View File

@@ -16,28 +16,45 @@ impl AddressCohortsRealizedStates {
realized_data: &AddressRealizedData,
liquidity_classification: &LiquidityClassification,
) -> color_eyre::Result<()> {
let profit = realized_data.profit;
let loss = realized_data.loss;
let realized_profit = realized_data.profit;
let realized_loss = realized_data.loss;
let value_created = realized_data.value_created;
let value_destroyed = realized_data.value_destroyed;
let split_profit = liquidity_classification.split(profit.to_cent() as f64);
let split_loss = liquidity_classification.split(loss.to_cent() as f64);
let split_realized_profit =
liquidity_classification.split(realized_profit.to_cent() as f64);
let split_realized_loss = liquidity_classification.split(realized_loss.to_cent() as f64);
let split_value_created = liquidity_classification.split(value_created.to_cent() as f64);
let split_value_destroyed =
liquidity_classification.split(value_destroyed.to_cent() as f64);
let iterate = move |state: &mut SplitByLiquidity<RealizedState>| -> color_eyre::Result<()> {
state.all.iterate(profit, loss);
state.all.iterate(
realized_profit,
realized_loss,
value_created,
value_destroyed,
);
state.illiquid.iterate(
Price::from_cent(split_profit.illiquid as u64),
Price::from_cent(split_loss.illiquid as u64),
Price::from_cent(split_realized_profit.illiquid as u64),
Price::from_cent(split_realized_loss.illiquid as u64),
Price::from_cent(split_value_created.illiquid as u64),
Price::from_cent(split_value_destroyed.illiquid as u64),
);
state.liquid.iterate(
Price::from_cent(split_profit.liquid as u64),
Price::from_cent(split_loss.liquid as u64),
Price::from_cent(split_realized_profit.liquid as u64),
Price::from_cent(split_realized_loss.liquid as u64),
Price::from_cent(split_value_created.liquid as u64),
Price::from_cent(split_value_destroyed.liquid as u64),
);
state.highly_liquid.iterate(
Price::from_cent(split_profit.highly_liquid as u64),
Price::from_cent(split_loss.highly_liquid as u64),
Price::from_cent(split_realized_profit.highly_liquid as u64),
Price::from_cent(split_realized_loss.highly_liquid as u64),
Price::from_cent(split_value_created.highly_liquid as u64),
Price::from_cent(split_value_destroyed.highly_liquid as u64),
);
Ok(())

12
parser/src/states/cohorts_states/any/realized_state.rs
View File

@@ -4,11 +4,21 @@ use crate::structs::Price;
pub struct RealizedState {
pub realized_profit: Price,
pub realized_loss: Price,
pub value_created: Price,
pub value_destroyed: Price,
}
impl RealizedState {
pub fn iterate(&mut self, realized_profit: Price, realized_loss: Price) {
pub fn iterate(
&mut self,
realized_profit: Price,
realized_loss: Price,
value_created: Price,
value_destroyed: Price,
) {
self.realized_profit += realized_profit;
self.realized_loss += realized_loss;
self.value_created += value_created;
self.value_destroyed += value_destroyed;
}
}

3
parser/src/states/cohorts_states/utxo/cohorts_sent_states.rs
View File

@@ -52,6 +52,9 @@ impl UTXOCohortsSentStates {
let previous_value = previous_price * amount_sent;
let current_value = current_price * amount_sent;
state.realized.value_destroyed += previous_value;
state.realized.value_created += current_value;
match previous_value.cmp(&current_value) {
Ordering::Less => {
state.realized.realized_profit += current_value - previous_value;

24
parser/src/structs/address_data.rs
View File

@@ -43,12 +43,7 @@ impl AddressData {
self.realized_cap += received_value;
}
pub fn send(
&mut self,
amount: WAmount,
current_price: Price,
sent_amount_price: Price,
) -> color_eyre::Result<ProfitOrLoss> {
pub fn send(&mut self, amount: WAmount, previous_price: Price) -> color_eyre::Result<()> {
let previous_amount = self.amount;
if previous_amount < amount {
@@ -63,18 +58,10 @@ impl AddressData {
self.outputs_len -= 1;
let previous_sent_dollar_value = sent_amount_price * amount;
let previous_sent_dollar_value = previous_price * amount;
self.realized_cap -= previous_sent_dollar_value;
let current_sent_dollar_value = current_price * amount;
let profit_or_loss = if current_sent_dollar_value >= previous_sent_dollar_value {
ProfitOrLoss::Profit(current_sent_dollar_value - previous_sent_dollar_value)
} else {
ProfitOrLoss::Loss(previous_sent_dollar_value - current_sent_dollar_value)
};
Ok(profit_or_loss)
Ok(())
}
#[inline(always)]
@@ -105,8 +92,3 @@ impl AddressData {
LiquidityClassification::new(self.sent, self.received)
}
}
pub enum ProfitOrLoss {
Profit(Price),
Loss(Price),
}

25
parser/src/structs/address_realized_data.rs
View File

@@ -1,4 +1,4 @@
use super::{AddressData, Price, ProfitOrLoss, WAmount};
use super::{AddressData, Price, WAmount};
#[derive(Debug)]
pub struct AddressRealizedData {
@@ -7,6 +7,8 @@ pub struct AddressRealizedData {
pub sent: WAmount,
pub profit: Price,
pub loss: Price,
pub value_created: Price,
pub value_destroyed: Price,
pub utxos_created: u32,
pub utxos_destroyed: u32,
}
@@ -20,6 +22,8 @@ impl AddressRealizedData {
loss: Price::ZERO,
utxos_created: 0,
utxos_destroyed: 0,
value_created: Price::ZERO,
value_destroyed: Price::ZERO,
initial_address_data: *initial_address_data,
}
}
@@ -29,18 +33,21 @@ impl AddressRealizedData {
self.utxos_created += 1;
}
pub fn send(&mut self, amount: WAmount, realized_profit_or_loss: ProfitOrLoss) {
pub fn send(&mut self, amount: WAmount, current_price: Price, previous_price: Price) {
self.sent += amount;
self.utxos_destroyed += 1;
match realized_profit_or_loss {
ProfitOrLoss::Profit(price) => {
self.profit += price;
}
ProfitOrLoss::Loss(price) => {
self.loss += price;
}
let current_value = current_price * amount;
let previous_value = previous_price * amount;
self.value_created += current_value;
self.value_destroyed += previous_value;
if current_value >= previous_value {
self.profit += current_value - previous_value;
} else {
self.loss += previous_value - current_value;
}
}
}

19
parser/src/structs/bi_map.rs
View File

@@ -277,20 +277,29 @@ where
&mut self,
heights: &[usize],
dates: &[WNaiveDate],
source: &mut BiMap<T>,
percentile: f32,
mut map_and_percentiles: Vec<(&mut BiMap<T>, f32)>,
days: Option<usize>,
) where
T: FloatCore,
{
let mut date_map_and_percentiles = vec![];
let mut height_map_and_percentiles = vec![];
map_and_percentiles
.iter_mut()
.for_each(|(map, percentile)| {
date_map_and_percentiles.push((&mut map.date, *percentile));
height_map_and_percentiles.push((&mut map.height, *percentile));
});
self.height.multi_insert_percentile(
heights,
&mut source.height,
percentile,
height_map_and_percentiles,
days.map(|days| TARGET_BLOCKS_PER_DAY * days),
);
self.date
.multi_insert_percentile(dates, &mut source.date, percentile, days);
.multi_insert_percentile(dates, date_map_and_percentiles, days);
}
}

192
parser/src/structs/date_map.rs
View File

@@ -423,10 +423,10 @@ where
&mut self,
dates: &[WNaiveDate],
source: &mut DateMap<K>,
transform: F,
mut transform: F,
) where
K: MapValue,
F: Fn((K, &WNaiveDate, &mut DateMap<K>)) -> T,
F: FnMut((K, &WNaiveDate, &mut DateMap<K>)) -> T,
{
dates.iter().for_each(|date| {
self.insert(
@@ -706,113 +706,129 @@ where
) where
T: FloatCore,
{
self.multi_insert_percentile(dates, source, 0.5, days);
source.multi_insert_percentile(dates, vec![(self, 0.5)], days);
}
pub fn multi_insert_percentile(
&mut self,
dates: &[WNaiveDate],
source: &mut DateMap<T>,
percentile: f32,
mut map_and_percentiles: Vec<(&mut DateMap<T>, f32)>,
days: Option<usize>,
) where
T: FloatCore,
{
if !(0.0..=1.0).contains(&percentile) {
panic!("The percentile should be between 0.0 and 1.0");
}
if days.map_or(false, |size| size < 3) {
panic!("Computing a median for a size lower than 3 is useless");
panic!("Computing a percentile for a size lower than 3 is useless");
}
let mut ordered_vec = None;
let mut sorted_vec = None;
dates.iter().for_each(|date| {
let value = {
if let Some(start) = days
.map_or(chrono::NaiveDate::from_ymd_opt(2009, 3, 1), |size| {
date.checked_sub_days(Days::new(size as u64))
})
{
if ordered_vec.is_none() {
let mut vec = start
.iter_days()
.take_while(|d| *d != **date)
.flat_map(|date| source.get_or_import(&WNaiveDate::wrap(date)))
.map(|f| OrderedFloat(f))
.collect_vec();
if let Some(start) = days.map_or(chrono::NaiveDate::from_ymd_opt(2009, 1, 3), |size| {
date.checked_sub_days(Days::new(size as u64))
}) {
if sorted_vec.is_none() {
let mut vec = start
.iter_days()
.take_while(|d| *d != **date)
.flat_map(|date| self.get_or_import(&WNaiveDate::wrap(date)))
.map(|f| OrderedFloat(f))
.collect_vec();
if days.is_some() {
ordered_vec.replace(VecDeque::from(vec.clone()));
}
vec.sort_unstable();
sorted_vec.replace(vec);
} else {
let float_value = OrderedFloat(source.get_or_import(date).unwrap());
if let Some(days) = days {
if let Some(ordered_vec) = ordered_vec.as_mut() {
if ordered_vec.len() == days {
let first = ordered_vec.pop_front().unwrap();
let pos =
sorted_vec.as_ref().unwrap().binary_search(&first).unwrap();
sorted_vec.as_mut().unwrap().remove(pos);
}
ordered_vec.push_back(float_value);
}
}
let pos = sorted_vec
.as_ref()
.unwrap()
.binary_search(&float_value)
.unwrap_or_else(|pos| pos);
sorted_vec.as_mut().unwrap().insert(pos, float_value);
if days.is_some() {
ordered_vec.replace(VecDeque::from(vec.clone()));
}
let vec = sorted_vec.as_ref().unwrap();
if vec.is_empty() {
T::default()
} else {
let index = vec.len() as f32 * percentile;
if index.fract() != 0.0 && vec.len() > 1 {
(vec.get(index.ceil() as usize)
.unwrap_or_else(|| {
dbg!(vec, index, &self.path_all, &source.path_all, days);
panic!()
})
.0
+ vec
.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(vec, index, &self.path_all, &source.path_all, days);
panic!()
})
.0)
/ T::from(2.0).unwrap()
} else {
vec.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(vec, index);
panic!();
})
.0
}
}
vec.sort_unstable();
sorted_vec.replace(vec);
} else {
T::default()
}
};
let float_value = OrderedFloat(self.get_or_import(date).unwrap());
self.insert(*date, value);
if let Some(days) = days {
if let Some(ordered_vec) = ordered_vec.as_mut() {
if ordered_vec.len() == days {
let first = ordered_vec.pop_front().unwrap();
let pos =
sorted_vec.as_ref().unwrap().binary_search(&first).unwrap();
sorted_vec.as_mut().unwrap().remove(pos);
}
ordered_vec.push_back(float_value);
}
}
let pos = sorted_vec
.as_ref()
.unwrap()
.binary_search(&float_value)
.unwrap_or_else(|pos| pos);
sorted_vec.as_mut().unwrap().insert(pos, float_value);
}
let vec = sorted_vec.as_ref().unwrap();
let len = vec.len();
map_and_percentiles
.iter_mut()
.for_each(|(map, percentile)| {
if !(0.0..=1.0).contains(percentile) {
panic!("The percentile should be between 0.0 and 1.0");
}
let value = {
if vec.is_empty() {
T::default()
} else {
let index = (len - 1) as f32 * *percentile;
let fract = index.fract();
let fract_t = T::from(fract).unwrap();
if fract != 0.0 {
(vec.get(index.ceil() as usize)
.unwrap_or_else(|| {
dbg!(vec, index, &self.path_all, &self.path_all, days);
panic!()
})
.0
* fract_t
+ vec
.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(
vec,
index,
&self.path_all,
&self.path_all,
days
);
panic!()
})
.0)
* T::from(1.0 - fract).unwrap()
} else {
vec.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(vec, index);
panic!();
})
.0
}
}
};
(*map).insert(*date, value);
});
} else {
map_and_percentiles.iter_mut().for_each(|(map, _)| {
(*map).insert(*date, T::default());
});
}
});
}

157
parser/src/structs/height_map.rs
View File

@@ -444,10 +444,10 @@ where
&mut self,
heights: &[usize],
source: &mut HeightMap<K>,
transform: F,
mut transform: F,
) where
K: MapValue,
F: Fn((K, &usize)) -> T,
F: FnMut((K, &usize)) -> T,
{
heights.iter().for_each(|height| {
self.insert(*height, transform((source.get_or_import(height), height)));
@@ -695,24 +695,19 @@ where
) where
T: FloatCore,
{
self.multi_insert_percentile(heights, source, 0.5, block_time);
source.multi_insert_percentile(heights, vec![(self, 0.5)], block_time);
}
pub fn multi_insert_percentile(
&mut self,
heights: &[usize],
source: &mut HeightMap<T>,
percentile: f32,
mut map_and_percentiles: Vec<(&mut HeightMap<T>, f32)>,
block_time: Option<usize>,
) where
T: FloatCore,
{
if !(0.0..=1.0).contains(&percentile) {
panic!("The percentile should be between 0.0 and 1.0");
}
if block_time.map_or(false, |size| size < 3) {
panic!("Computing a median for a size lower than 3 is useless");
panic!("Computing a percentile for a size lower than 3 is useless");
}
let mut ordered_vec = None;
@@ -721,66 +716,100 @@ where
heights.iter().for_each(|height| {
let height = *height;
let value = {
if let Some(start) = block_time.map_or(Some(0), |size| height.checked_sub(size)) {
if ordered_vec.is_none() {
let mut vec = (start..=height)
.map(|height| OrderedFloat(source.get_or_import(&height)))
.collect_vec();
if let Some(start) = block_time.map_or(Some(0), |size| height.checked_sub(size)) {
if sorted_vec.is_none() {
let mut vec = (start..=height)
.map(|height| OrderedFloat(self.get_or_import(&height)))
.collect_vec();
if block_time.is_some() {
ordered_vec.replace(VecDeque::from(vec.clone()));
}
vec.sort_unstable();
sorted_vec.replace(vec);
} else {
let float_value = OrderedFloat(source.get_or_import(&height));
if block_time.is_some() {
let first = ordered_vec.as_mut().unwrap().pop_front().unwrap();
let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap();
sorted_vec.as_mut().unwrap().remove(pos);
ordered_vec.as_mut().unwrap().push_back(float_value);
}
let pos = sorted_vec
.as_ref()
.unwrap()
.binary_search(&float_value)
.unwrap_or_else(|pos| pos);
sorted_vec.as_mut().unwrap().insert(pos, float_value);
if block_time.is_some() {
ordered_vec.replace(VecDeque::from(vec.clone()));
}
let vec = sorted_vec.as_ref().unwrap();
vec.sort_unstable();
let index = vec.len() as f32 * percentile;
if index.fract() != 0.0 {
(vec.get(index.ceil() as usize)
.unwrap_or_else(|| {
dbg!(index, &self.path_all, &source.path_all, block_time);
panic!()
})
.0
+ vec
.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(index, &self.path_all, &source.path_all, block_time);
panic!()
})
.0)
/ T::from(2.0).unwrap()
} else {
vec.get(index as usize).unwrap().0
}
sorted_vec.replace(vec);
} else {
T::default()
}
};
let float_value = OrderedFloat(self.get_or_import(&height));
self.insert(height, value);
if block_time.is_some() {
let first = ordered_vec.as_mut().unwrap().pop_front().unwrap();
let pos = sorted_vec.as_ref().unwrap().binary_search(&first).unwrap();
sorted_vec.as_mut().unwrap().remove(pos);
ordered_vec.as_mut().unwrap().push_back(float_value);
}
let pos = sorted_vec
.as_ref()
.unwrap()
.binary_search(&float_value)
.unwrap_or_else(|pos| pos);
sorted_vec.as_mut().unwrap().insert(pos, float_value);
}
let vec = sorted_vec.as_ref().unwrap();
let len = vec.len();
map_and_percentiles
.iter_mut()
.for_each(|(map, percentile)| {
if !(0.0..=1.0).contains(percentile) {
panic!("The percentile should be between 0.0 and 1.0");
}
let value = {
if vec.is_empty() {
T::default()
} else {
let index = (len - 1) as f32 * *percentile;
let fract = index.fract();
let fract_t = T::from(fract).unwrap();
if fract != 0.0 {
(vec.get(index.ceil() as usize)
.unwrap_or_else(|| {
dbg!(
index,
&self.path_all,
&self.path_all,
&self.to_insert,
block_time,
vec
);
panic!()
})
.0
* fract_t
+ vec
.get(index.floor() as usize)
.unwrap_or_else(|| {
dbg!(
index,
&self.path_all,
&self.path_all,
block_time
);
panic!()
})
.0)
* T::from(1.0 - fract).unwrap()
} else {
vec.get(index as usize).unwrap().0
}
}
};
(*map).insert(height, value);
});
} else {
map_and_percentiles.iter_mut().for_each(|(map, _)| {
(*map).insert(height, T::default());
});
}
});
}