Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-26 15:49:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
k
2024-10-20 18:31:43 +02:00
parent ffa4871035
commit 5b9d599e83
47 changed files with 666 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
parser/src/actions/iter_blocks.rs
View File

@@ -1,6 +1,5 @@
use std::{collections::BTreeSet, time::Instant};
use chrono::Datelike;
use export::ExportedData;
use itertools::Itertools;
@@ -13,9 +12,9 @@ use crate::{
datasets::{AllDatasets, ComputeData},
io::OUTPUTS_FOLDER_PATH,
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{DateData, MapKey, Timestamp},
structs::{DateData, MapKey, Timestamp, RAM},
utils::{generate_allocation_files, log, time},
Config, Exit, Height,
Config, Date, Exit, Height,
};
pub fn iter_blocks(
@@ -26,7 +25,7 @@ pub fn iter_blocks(
) -> color_eyre::Result<()> {
log("Starting...");
let mut datasets = AllDatasets::import()?;
let mut datasets = AllDatasets::import(config)?;
log("Imported datasets");
@@ -58,6 +57,8 @@ pub fn iter_blocks(
let mut block_iter = block_receiver.iter();
let ram = RAM::new();
'parsing: loop {
let instant = Instant::now();
@@ -174,10 +175,12 @@ pub fn iter_blocks(
if is_date_last_block {
height += blocks_loop_i;
let is_new_month = next_block_date
.map_or(true, |next_block_date| next_block_date.day() == 1);
let is_new_year = next_block_date.as_ref().map_or(true, Date::is_new_year);
if is_new_month || height.is_close_to_end(approx_block_count) {
if is_new_year
|| ram.max_exceeded(config)
|| height.is_close_to_end(approx_block_count)
{
break 'days;
}
@@ -193,11 +196,12 @@ pub fn iter_blocks(
let last_height = height - 1_u32;
log(&format!(
"Parsing month took {} seconds (last height: {last_height})\n",
"Parsing group took {} seconds (last height: {last_height})\n",
instant.elapsed().as_secs_f32(),
));
if first_unsafe_heights.computed <= last_height {
log("Computing datasets...");
time("Computing datasets", || {
let dates = processed_dates.into_iter().collect_vec();

2
parser/src/actions/min_height.rs
View File

@@ -87,7 +87,7 @@ pub fn find_first_inserted_unsafe_height(
let computed = datasets_min_initial_states.computed.last_date.and_then(
|last_date| datasets.date_metadata
.last_height
.get(&last_date)
.get_or_import(&last_date)
.and_then(|last_date_height| {
if datasets_min_initial_states.computed.last_height.map_or(true, |last_height| {
last_height < last_date_height

9
parser/src/datasets/_traits/any_dataset.rs
View File

@@ -283,4 +283,13 @@ pub trait AnyDataset {
.for_each(|map| map.post_export())
});
}
fn reset_computed(&self) {
self.to_all_computed_date_map_vec()
.iter()
.for_each(|map| map.delete_files());
self.to_all_computed_height_map_vec()
.iter()
.for_each(|map| map.delete_files());
}
}

28
parser/src/datasets/_traits/min_initial_state.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use crate::structs::{AnyDateMap, AnyHeightMap, Date, Height};
use crate::structs::{AnyDateMap, AnyHeightMap, Config, Date, Height};
use super::{AnyDataset, AnyDatasets};
@@ -16,17 +16,17 @@ impl MinInitialStates {
self.computed = other.computed;
}
pub fn compute_from_dataset(dataset: &dyn AnyDataset) -> Self {
pub fn compute_from_dataset(dataset: &dyn AnyDataset, config: &Config) -> Self {
Self {
inserted: MinInitialState::compute_from_dataset(dataset, Mode::Inserted),
computed: MinInitialState::compute_from_dataset(dataset, Mode::Computed),
inserted: MinInitialState::compute_from_dataset(dataset, Mode::Inserted, config),
computed: MinInitialState::compute_from_dataset(dataset, Mode::Computed, config),
}
}
pub fn compute_from_datasets(datasets: &dyn AnyDatasets) -> Self {
pub fn compute_from_datasets(datasets: &dyn AnyDatasets, config: &Config) -> Self {
Self {
inserted: MinInitialState::compute_from_datasets(datasets, Mode::Inserted),
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed),
inserted: MinInitialState::compute_from_datasets(datasets, Mode::Inserted, config),
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed, config),
}
}
}
@@ -52,7 +52,7 @@ impl MinInitialState {
// self.last_height = other.last_height;
// }
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode) -> Self {
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {
let contains_date_maps = |dataset: &&(dyn AnyDataset + Sync + Send)| {
@@ -111,6 +111,11 @@ impl MinInitialState {
}
}
Mode::Computed => {
if config.recompute_computed() {
// datasets.reset_computed();
return Self::default();
}
let contains_date_maps = |dataset: &&(dyn AnyDataset + Sync + Send)| {
!dataset.to_all_computed_date_map_vec().is_empty()
};
@@ -197,7 +202,7 @@ impl MinInitialState {
)
}
fn compute_from_dataset(dataset: &dyn AnyDataset, mode: Mode) -> Self {
fn compute_from_dataset(dataset: &dyn AnyDataset, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {
let date_vec = dataset.to_all_inserted_date_map_vec();
@@ -215,6 +220,11 @@ impl MinInitialState {
}
}
Mode::Computed => {
if config.recompute_computed() {
dataset.reset_computed();
return Self::default();
}
let date_vec = dataset.to_all_computed_date_map_vec();
let height_vec = dataset.to_all_computed_height_map_vec();

6
parser/src/datasets/address/all_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Allocative)]
@@ -18,7 +18,7 @@ pub struct AllAddressesMetadataDataset {
}
impl AllAddressesMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -30,7 +30,7 @@ impl AllAddressesMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

15
parser/src/datasets/address/cohort.rs
View File

@@ -1,4 +1,5 @@
use allocative::Allocative;
use itertools::Itertools;
use crate::{
@@ -6,7 +7,7 @@ use crate::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
states::{AddressCohortId, DurableStates},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Date, Height},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
};
use super::cohort_metadata::MetadataDataset;
@@ -23,19 +24,23 @@ pub struct CohortDataset {
}
impl CohortDataset {
pub fn import(parent_path: &str, id: AddressCohortId) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
id: AddressCohortId,
config: &Config,
) -> color_eyre::Result<Self> {
let name = id.as_name().map(|s| s.to_owned());
let split = id.as_split();
let mut s = Self {
min_initial_states: MinInitialStates::default(),
split,
metadata: MetadataDataset::import(parent_path, &name)?,
subs: SubDataset::import(parent_path, &name)?,
metadata: MetadataDataset::import(parent_path, &name, config)?,
subs: SubDataset::import(parent_path, &name, config)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/address/cohort_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -18,7 +18,11 @@ pub struct MetadataDataset {
}
impl MetadataDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -35,7 +39,7 @@ impl MetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/address/mod.rs
View File

@@ -8,7 +8,7 @@ use rayon::prelude::*;
use crate::{
states::SplitByAddressCohort,
structs::{BiMap, Height},
structs::{BiMap, Config, Height},
Date,
};
@@ -26,13 +26,13 @@ pub struct AddressDatasets {
}
impl AddressDatasets {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByAddressCohort::<CohortDataset>::default();
cohorts
.as_vec()
.into_par_iter()
.map(|(_, id)| (id, CohortDataset::import(parent_path, id)))
.map(|(_, id)| (id, CohortDataset::import(parent_path, id, config)))
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
@@ -43,13 +43,13 @@ impl AddressDatasets {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
metadata: AllAddressesMetadataDataset::import(parent_path)?,
metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
cohorts,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
Ok(s)
}

6
parser/src/datasets/block_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyHeightMap, Date, HeightMap, Timestamp},
structs::{AnyHeightMap, Config, Date, HeightMap, Timestamp},
};
use super::{InsertData, MinInitialStates};
@@ -17,7 +17,7 @@ pub struct BlockMetadataDataset {
}
impl BlockMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -28,7 +28,7 @@ impl BlockMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/coindays.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, AnyHeightMap},
structs::{AnyDateMap, AnyHeightMap, Config},
DateMap, HeightMap,
};
@@ -18,7 +18,7 @@ pub struct CoindaysDataset {
}
impl CoindaysDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -29,7 +29,7 @@ impl CoindaysDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

14
parser/src/datasets/cointime.rs
View File

@@ -1,7 +1,7 @@
use allocative::Allocative;
use crate::{
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, DateMap, Height},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height},
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
HeightMap,
};
@@ -71,7 +71,7 @@ pub struct CointimeDataset {
}
impl CointimeDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -79,7 +79,7 @@ impl CointimeDataset {
active_cap: BiMap::new_bin(1, &f("active_cap")),
active_price: BiMap::new_bin(1, &f("active_price")),
active_price_ratio: RatioDataset::import(parent_path, "active_price")?,
active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
active_supply: BiMap::new_bin(1, &f("active_supply")),
active_supply_3m_net_change: BiMap::new_bin(1, &f("active_supply_3m_net_change")),
active_supply_net_change: BiMap::new_bin(1, &f("active_supply_net_change")),
@@ -101,7 +101,7 @@ impl CointimeDataset {
),
cointime_cap: BiMap::new_bin(1, &f("cointime_cap")),
cointime_price: BiMap::new_bin(1, &f("cointime_price")),
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price")?,
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
cointime_value_created: HeightMap::new_bin(1, &f("cointime_value_created")),
cointime_value_created_1d_sum: DateMap::new_bin(1, &f("cointime_value_created_1d_sum")),
cointime_value_destroyed: HeightMap::new_bin(1, &f("cointime_value_destroyed")),
@@ -141,14 +141,14 @@ impl CointimeDataset {
total_cointime_value_stored: BiMap::new_bin(1, &f("total_cointime_value_stored")),
true_market_deviation: BiMap::new_bin(1, &f("true_market_deviation")),
true_market_mean: BiMap::new_bin(1, &f("true_market_mean")),
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean")?,
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
1,
&f("true_market_net_unrealized_profit_and_loss"),
),
vaulted_cap: BiMap::new_bin(1, &f("vaulted_cap")),
vaulted_price: BiMap::new_bin(1, &f("vaulted_price")),
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price")?,
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
vaulted_supply: BiMap::new_bin(1, &f("vaulted_supply")),
vaulted_supply_3m_net_change: BiMap::new_bin(1, &f("vaulted_supply_3m_net_change")),
vaulted_supply_net_change: BiMap::new_bin(1, &f("vaulted_supply_net_change")),
@@ -157,7 +157,7 @@ impl CointimeDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/constant.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use crate::structs::{AnyBiMap, BiMap};
use crate::structs::{AnyBiMap, BiMap, Config};
use super::{AnyDataset, ComputeData, MinInitialStates};
@@ -16,7 +16,7 @@ pub struct ConstantDataset {
}
impl ConstantDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -29,7 +29,7 @@ impl ConstantDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/date_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, DateMap, Height},
structs::{AnyDateMap, Config, DateMap, Height},
};
use super::{InsertData, MinInitialStates};
@@ -17,7 +17,7 @@ pub struct DateMetadataDataset {
}
impl DateMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -28,7 +28,7 @@ impl DateMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

47
parser/src/datasets/mining.rs
View File

@@ -6,7 +6,7 @@ use crate::{
datasets::AnyDataset,
structs::{
date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap,
AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey,
AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height, HeightMap, MapKey,
},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
@@ -80,6 +80,7 @@ pub struct MiningDataset {
pub blocks_mined_1y_sum: DateMap<usize>,
pub blocks_mined_1y_target: DateMap<usize>,
pub cumulative_block_size: BiMap<f32>,
pub cumulative_block_size_gigabytes: BiMap<f32>,
pub subsidy_1y_sum: DateMap<f64>,
pub subsidy_in_dollars_1y_sum: DateMap<f64>,
pub cumulative_subsidy: BiMap<f64>,
@@ -104,8 +105,8 @@ pub struct MiningDataset {
pub hash_rate_1m_sma: DateMap<f32>,
pub hash_rate_2m_sma: DateMap<f32>,
pub hash_price: DateMap<f64>,
// pub hash_price_min: DateMap<f64>,
// pub hash_price_rebound: DateMap<f64>,
pub hash_price_min: DateMap<f64>,
pub hash_price_rebound: DateMap<f64>,
pub difficulty_adjustment: DateMap<f64>,
pub block_size_recap: DateRecapDataset<f32>, // in MB
pub block_weight_recap: DateRecapDataset<f32>, // in MB
@@ -124,7 +125,7 @@ pub struct MiningDataset {
}
impl MiningDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -206,6 +207,10 @@ impl MiningDataset {
.add_min(),
)?,
cumulative_block_size: BiMap::new_bin(1, &f("cumulative_block_size")),
cumulative_block_size_gigabytes: BiMap::new_bin(
1,
&f("cumulative_block_size_gigabytes"),
),
block_weight: HeightMap::new_bin(1, &f("block_weight")),
block_weight_recap: RecapDataset::import(
&f("block_weight_1d"),
@@ -251,11 +256,13 @@ impl MiningDataset {
hash_rate_1m_sma: DateMap::new_bin(1, &f("hash_rate_1m_sma")),
hash_rate_2m_sma: DateMap::new_bin(1, &f("hash_rate_2m_sma")),
hash_price: DateMap::new_bin(1, &f("hash_price")),
hash_price_min: DateMap::new_bin(1, &f("hash_price_min")),
hash_price_rebound: DateMap::new_bin(1, &f("hash_price_rebound")),
puell_multiple: DateMap::new_bin(1, &f("puell_multiple")),
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}
@@ -536,13 +543,22 @@ impl MiningDataset {
last_height,
);
self.cumulative_block_size
.height
.multi_insert_cumulative(heights, &mut self.block_size);
self.cumulative_block_size_gigabytes
.multi_insert_simple_transform(heights, dates, &mut self.cumulative_block_size, &|v| {
v / 1000.0
});
// https://hashrateindex.com/blog/what-is-bitcoins-hashrate/
self.hash_rate.multi_insert(dates, |date| {
let blocks_mined = self.blocks_mined.get_or_import(date).unwrap();
let difficulty = self.difficulty.date.get_or_import(date).unwrap();
((blocks_mined as f64 / date.get_day_completion() * TARGET_BLOCKS_PER_DAY as f64)
(blocks_mined as f64 / (date.get_day_completion() * TARGET_BLOCKS_PER_DAY as f64)
* difficulty
* 2.0_f64.powi(32))
/ 600.0
@@ -570,11 +586,20 @@ impl MiningDataset {
self.hash_price.multi_insert(dates, |date| {
let coinbase_in_dollars = self.coinbase_in_dollars_1d_sum.get_or_import(date).unwrap();
let hashrate = self.hash_rate.get_or_import(date).unwrap();
let hash_rate = self.hash_rate.get_or_import(date).unwrap();
coinbase_in_dollars as f64 / hashrate / 1_000.0
coinbase_in_dollars as f64 / hash_rate / 1_000.0
});
self.hash_price_min
.multi_insert_min(dates, &mut self.hash_price, 0.0);
self.hash_price_rebound.multi_insert_percentage(
dates,
&mut self.hash_price,
&mut self.hash_price_min,
);
self.puell_multiple.multi_insert_divide(
dates,
&mut self.coinbase_in_dollars_1d_sum,
@@ -721,6 +746,7 @@ impl AnyDataset for MiningDataset {
&self.cumulative_subsidy,
&self.cumulative_subsidy_in_dollars,
&self.cumulative_block_size,
&self.cumulative_block_size_gigabytes,
]
}
@@ -733,6 +759,7 @@ impl AnyDataset for MiningDataset {
&mut self.cumulative_subsidy,
&mut self.cumulative_subsidy_in_dollars,
&mut self.cumulative_block_size,
&mut self.cumulative_block_size_gigabytes,
]
}
@@ -778,6 +805,8 @@ impl AnyDataset for MiningDataset {
&self.hash_rate_1m_sma,
&self.hash_rate_2m_sma,
&self.hash_price,
&self.hash_price_min,
&self.hash_price_rebound,
&self.puell_multiple,
&self.difficulty_adjustment,
]
@@ -825,6 +854,8 @@ impl AnyDataset for MiningDataset {
&mut self.hash_rate_1m_sma,
&mut self.hash_rate_2m_sma,
&mut self.hash_price,
&mut self.hash_price_min,
&mut self.hash_price_rebound,
&mut self.puell_multiple,
&mut self.difficulty_adjustment,
]

26
parser/src/datasets/mod.rs
View File

@@ -45,7 +45,7 @@ use crate::{
// UTXOCohortsReceivedStates,
UTXOCohortsSentStates,
},
structs::{Amount, Date, Height, Price, Timestamp},
structs::{Amount, Config, Date, Height, Price, Timestamp},
};
pub struct InsertData<'a> {
@@ -102,28 +102,28 @@ pub struct AllDatasets {
const DATASETS_PATH: &str = "../datasets";
impl AllDatasets {
pub fn import() -> color_eyre::Result<Self> {
pub fn import(config: &Config) -> color_eyre::Result<Self> {
let path = DATASETS_PATH;
let price = PriceDatasets::import(path)?;
let price = PriceDatasets::import(path, config)?;
let constant = ConstantDataset::import(path)?;
let constant = ConstantDataset::import(path, config)?;
let date_metadata = DateMetadataDataset::import(path)?;
let date_metadata = DateMetadataDataset::import(path, config)?;
let cointime = CointimeDataset::import(path)?;
let cointime = CointimeDataset::import(path, config)?;
let coindays = CoindaysDataset::import(path)?;
let coindays = CoindaysDataset::import(path, config)?;
let mining = MiningDataset::import(path)?;
let mining = MiningDataset::import(path, config)?;
let block_metadata = BlockMetadataDataset::import(path)?;
let block_metadata = BlockMetadataDataset::import(path, config)?;
let transaction = TransactionDataset::import(path)?;
let transaction = TransactionDataset::import(path, config)?;
let address = AddressDatasets::import(path)?;
let address = AddressDatasets::import(path, config)?;
let utxo = UTXODatasets::import(path)?;
let utxo = UTXODatasets::import(path, config)?;
let mut s = Self {
min_initial_states: MinInitialStates::default(),
@@ -141,7 +141,7 @@ impl AllDatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
s.export_meta_files()?;

32
parser/src/datasets/price/mod.rs
View File

@@ -11,7 +11,7 @@ pub use ohlc::*;
use crate::{
price::{Binance, Kibo, Kraken},
structs::{
Amount, AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height,
Amount, AnyBiMap, AnyDateMap, BiMap, Config, Date, DateMap, DateMapChunkId, Height,
HeightMapChunkId, MapKey, Timestamp,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
@@ -90,7 +90,7 @@ pub struct PriceDatasets {
}
impl PriceDatasets {
pub fn import(datasets_path: &str) -> color_eyre::Result<Self> {
pub fn import(datasets_path: &str, config: &Config) -> color_eyre::Result<Self> {
let price_path = "../price";
let f = |s: &str| format!("{datasets_path}/{s}");
@@ -113,31 +113,31 @@ impl PriceDatasets {
close: BiMap::new_bin(1, &f("close")),
market_cap: BiMap::new_bin(1, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, &f("price_1w_sma")),
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma")?,
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
price_1m_sma: BiMap::new_bin(1, &f("price_1m_sma")),
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma")?,
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
price_1y_sma: BiMap::new_bin(1, &f("price_1y_sma")),
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma")?,
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
price_2y_sma: BiMap::new_bin(1, &f("price_2y_sma")),
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma")?,
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
price_4y_sma: BiMap::new_bin(1, &f("price_4y_sma")),
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma")?,
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
price_8d_sma: BiMap::new_bin(1, &f("price_8d_sma")),
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma")?,
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
price_13d_sma: BiMap::new_bin(1, &f("price_13d_sma")),
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma")?,
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
price_21d_sma: BiMap::new_bin(1, &f("price_21d_sma")),
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma")?,
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
price_34d_sma: BiMap::new_bin(1, &f("price_34d_sma")),
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma")?,
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
price_55d_sma: BiMap::new_bin(1, &f("price_55d_sma")),
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma")?,
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
price_89d_sma: BiMap::new_bin(1, &f("price_89d_sma")),
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma")?,
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
price_144d_sma: BiMap::new_bin(1, &f("price_144d_sma")),
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma")?,
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
price_200w_sma: BiMap::new_bin(1, &f("price_200w_sma")),
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma")?,
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
price_1d_total_return: DateMap::new_bin(1, &f("price_1d_total_return")),
price_1m_total_return: DateMap::new_bin(1, &f("price_1m_total_return")),
price_6m_total_return: DateMap::new_bin(1, &f("price_6m_total_return")),
@@ -169,7 +169,7 @@ impl PriceDatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

11
parser/src/datasets/subs/capitalization.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::CapitalizationState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
utils::ONE_MONTH_IN_DAYS,
};
@@ -23,7 +23,11 @@ pub struct CapitalizationDataset {
}
impl CapitalizationDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -44,11 +48,12 @@ impl CapitalizationDataset {
"{}realized_price",
name.as_ref().map_or("".to_owned(), |n| format!("{n}-"))
),
config,
)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/input.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::InputState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config},
DateMap, HeightMap,
};
@@ -20,7 +20,11 @@ pub struct InputSubDataset {
}
impl InputSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -38,7 +42,7 @@ impl InputSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

22
parser/src/datasets/subs/mod.rs
View File

@@ -20,7 +20,7 @@ pub use supply::*;
pub use unrealized::*;
pub use utxo::*;
use crate::datasets::AnyDataset;
use crate::{datasets::AnyDataset, structs::Config};
use super::AnyDatasetGroup;
@@ -37,16 +37,20 @@ pub struct SubDataset {
}
impl SubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let s = Self {
capitalization: CapitalizationDataset::import(parent_path, name)?,
input: InputSubDataset::import(parent_path, name)?,
capitalization: CapitalizationDataset::import(parent_path, name, config)?,
input: InputSubDataset::import(parent_path, name, config)?,
// output: OutputSubDataset::import(parent_path)?,
price_paid: PricePaidSubDataset::import(parent_path, name)?,
realized: RealizedSubDataset::import(parent_path, name)?,
supply: SupplySubDataset::import(parent_path, name)?,
unrealized: UnrealizedSubDataset::import(parent_path, name)?,
utxo: UTXOSubDataset::import(parent_path, name)?,
price_paid: PricePaidSubDataset::import(parent_path, name, config)?,
realized: RealizedSubDataset::import(parent_path, name, config)?,
supply: SupplySubDataset::import(parent_path, name, config)?,
unrealized: UnrealizedSubDataset::import(parent_path, name, config)?,
utxo: UTXOSubDataset::import(parent_path, name, config)?,
};
Ok(s)

10
parser/src/datasets/subs/price_paid.rs
View File

@@ -4,7 +4,7 @@ use itertools::Itertools;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::PricePaidState,
structs::{AnyBiMap, BiMap, Date, Height},
structs::{AnyBiMap, BiMap, Config, Date, Height},
};
#[derive(Default, Allocative)]
@@ -34,7 +34,11 @@ pub struct PricePaidSubDataset {
}
impl PricePaidSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -68,7 +72,7 @@ impl PricePaidSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/subs/ratio.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
@@ -31,7 +31,7 @@ pub struct RatioDataset {
}
impl RatioDataset {
pub fn import(parent_path: &str, name: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, name: &str, config: &Config) -> color_eyre::Result<Self> {
let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
@@ -61,7 +61,7 @@ impl RatioDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/realized.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::RealizedState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Price},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Price},
utils::ONE_MONTH_IN_DAYS,
DateMap, HeightMap,
};
@@ -46,7 +46,11 @@ pub struct RealizedSubDataset {
}
impl RealizedSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -114,7 +118,7 @@ impl RealizedSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/supply.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::SupplyState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -20,7 +20,11 @@ pub struct SupplySubDataset {
}
impl SupplySubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -45,7 +49,7 @@ impl SupplySubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/unrealized.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::UnrealizedState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -27,7 +27,11 @@ pub struct UnrealizedSubDataset {
}
impl UnrealizedSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -68,7 +72,7 @@ impl UnrealizedSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/utxo.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::UTXOState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -15,7 +15,11 @@ pub struct UTXOSubDataset {
}
impl UTXOSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -31,7 +35,7 @@ impl UTXOSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/transaction.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::InsertData,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, HeightMap},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, HeightMap},
utils::{
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
},
@@ -54,7 +54,7 @@ pub struct TransactionDataset {
}
impl TransactionDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -105,7 +105,7 @@ impl TransactionDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

12
parser/src/datasets/utxo/dataset.rs
View File

@@ -6,7 +6,7 @@ use crate::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
states::UTXOCohortId,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Date, Height},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
};
#[derive(Default, Allocative)]
@@ -19,17 +19,21 @@ pub struct UTXODataset {
}
impl UTXODataset {
pub fn import(parent_path: &str, id: UTXOCohortId) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
id: UTXOCohortId,
config: &Config,
) -> color_eyre::Result<Self> {
let name = id.name().to_owned();
let mut s = Self {
min_initial_states: MinInitialStates::default(),
id,
subs: SubDataset::import(parent_path, &Some(name))?,
subs: SubDataset::import(parent_path, &Some(name), config)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

8
parser/src/datasets/utxo/mod.rs
View File

@@ -9,7 +9,7 @@ use itertools::Itertools;
use crate::{
datasets::AnyDatasets,
states::{SplitByUTXOCohort, UTXOCohortId},
structs::{BiMap, Date, Height},
structs::{BiMap, Config, Date, Height},
};
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates};
@@ -22,13 +22,13 @@ pub struct UTXODatasets {
}
impl UTXODatasets {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByUTXOCohort::<UTXODataset>::default();
cohorts
.as_vec()
.into_par_iter()
.map(|(_, id)| (id, UTXODataset::import(parent_path, id)))
.map(|(_, id)| (id, UTXODataset::import(parent_path, id, config)))
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
@@ -43,7 +43,7 @@ impl UTXODatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
Ok(s)
}

2
parser/src/io/binary.rs
View File

@@ -69,6 +69,8 @@ impl Binary {
where
T: Debug + Encode,
{
// log(&format!("Exporting: {:?}", path));
match Self::type_from_path(path) {
BinaryType::Compressed => Self::export_compressed(path, value),
BinaryType::Raw => Self::export_raw(path, value),

2
parser/src/io/serialization.rs
View File

@@ -95,7 +95,7 @@ impl Serialization {
let path = path.to_str().unwrap();
let res = Binary::export(
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path,)),
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path)),
value,
);

2
parser/src/structs/any_map.rs
View File

@@ -23,4 +23,6 @@ pub trait AnyMap {
fn pre_export(&mut self);
fn export(&self) -> color_eyre::Result<()>;
fn post_export(&mut self);
fn delete_files(&self);
}

48
parser/src/structs/config.rs
View File

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use crate::log;
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)]
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[command(version, about, long_about = None)]
pub struct Config {
/// Bitcoin data directory path, saved
@@ -32,6 +32,10 @@ pub struct Config {
#[arg(long, value_name = "SECONDS")]
pub delay: Option<u64>,
/// Maximum ram you want the program to use in GB, default: 75% of total, not saved
#[arg(long, value_name = "GB")]
pub max_ram: Option<f64>,
/// Start a dry run, default: false, not saved
#[arg(long, value_name = "BOOL")]
dry_run: Option<bool>,
@@ -39,6 +43,10 @@ pub struct Config {
/// Record ram usage, default: false, not saved
#[arg(long, value_name = "BOOL")]
record_ram_usage: Option<bool>,
/// Recompute all computed datasets, default: false, not saved
#[arg(long, value_name = "BOOL")]
recompute_computed: Option<bool>,
}
impl Config {
@@ -50,40 +58,48 @@ impl Config {
toml::from_str(&contents).unwrap_or_default()
});
let config_args = Config::parse();
let mut config_args = Config::parse();
if let Some(datadir) = config_args.datadir {
if let Some(datadir) = config_args.datadir.take() {
config_saved.datadir = Some(datadir);
}
if let Some(rpcconnect) = config_args.rpcconnect {
if let Some(rpcconnect) = config_args.rpcconnect.take() {
config_saved.rpcconnect = Some(rpcconnect);
}
if let Some(rpcport) = config_args.rpcport {
if let Some(rpcport) = config_args.rpcport.take() {
config_saved.rpcport = Some(rpcport);
}
if let Some(rpcuser) = config_args.rpcuser {
if let Some(rpcuser) = config_args.rpcuser.take() {
config_saved.rpcuser = Some(rpcuser);
}
if let Some(rpcpassword) = config_args.rpcpassword {
if let Some(rpcpassword) = config_args.rpcpassword.take() {
config_saved.rpcpassword = Some(rpcpassword);
}
if let Some(delay) = config_args.delay {
if let Some(delay) = config_args.delay.take() {
config_saved.delay = Some(delay);
}
if let Some(max_ram) = config_args.max_ram.take() {
config_saved.max_ram = Some(max_ram);
}
// Done importing
let config = config_saved;
let mut config = config_saved;
config.check();
config.write()?;
config.dry_run = config_args.dry_run.take();
config.record_ram_usage = config_args.record_ram_usage.take();
config.recompute_computed = config_args.recompute_computed.take();
log("---");
log("Configuration:");
log(&format!("datadir: {:?}", config.datadir));
@@ -92,10 +108,20 @@ impl Config {
log(&format!("rpcuser: {:?}", config.rpcuser));
log(&format!("rpcpassword: {:?}", config.rpcpassword));
log(&format!("delay: {:?}", config.delay));
log(&format!("max_ram: {:?}", config.max_ram));
log(&format!("dry_run: {:?}", config.dry_run));
log(&format!("record_ram_usage: {:?}", config.record_ram_usage));
log(&format!(
"recompute_computed: {:?}",
config.recompute_computed
));
log("---");
if config_args != Config::default() {
dbg!(config_args);
panic!("Didn't consume the full config")
}
Ok(config)
}
@@ -132,4 +158,8 @@ impl Config {
pub fn record_ram_usage(&self) -> bool {
self.record_ram_usage.is_some_and(|b| b)
}
pub fn recompute_computed(&self) -> bool {
self.recompute_computed.is_some_and(|b| b)
}
}

4
parser/src/structs/date.rs
View File

@@ -73,6 +73,10 @@ impl Date {
Ordering::Greater => unreachable!("0.0 but shouldn't be called"),
}
}
pub fn is_new_year(&self) -> bool {
self.day() == 1 && self.month() == 1
}
}
impl MapKey<DateMapChunkId> for Date {

68
parser/src/structs/generic_map.rs
View File

@@ -156,9 +156,7 @@ where
if serialized.version() == s.version {
s.imported.insert(chunk_start, serialized);
} else {
s.read_dir()
.iter()
.for_each(|(_, path)| fs::remove_file(path).unwrap())
s.delete_files();
}
}
});
@@ -204,12 +202,10 @@ where
}
pub fn insert(&mut self, key: Key, value: Value) -> Value {
if !self.is_key_safe(key) {
self.to_insert
.entry(key.to_chunk_id())
.or_default()
.insert(key.to_serialized_key(), value);
}
self.to_insert
.entry(key.to_chunk_id())
.or_default()
.insert(key.to_serialized_key(), value);
value
}
@@ -341,8 +337,10 @@ where
});
let path = self.path_all.join(chunk_id.to_name());
self.serialization.export(Path::new(&path), serialized)?;
// Export last
if index == len - 1 {
if let Some(path_last) = self.path_last.as_ref() {
self.serialization
@@ -370,6 +368,12 @@ where
self.to_insert.clear();
}
fn delete_files(&self) {
self.read_dir()
.iter()
.for_each(|(_, path)| fs::remove_file(path).unwrap())
}
}
impl<Key, Value, ChunkId, Serialized> GenericMap<Key, Value, ChunkId, Serialized>
@@ -379,14 +383,16 @@ where
Key: MapKey<ChunkId>,
Serialized: MapSerialized<Key, Value, ChunkId>,
{
pub fn sum_keys(&self, keys: &[Key]) -> Value
pub fn sum_keys(&mut self, keys: &[Key]) -> Value
where
Value: Sum,
{
keys.iter().flat_map(|key| self.get(key)).sum::<Value>()
keys.iter()
.map(|key| self.get_or_import(key).unwrap())
.sum::<Value>()
}
pub fn average_keys(&self, keys: &[Key]) -> f32
pub fn average_keys(&mut self, keys: &[Key]) -> f32
where
Value: Sum,
f32: LossyFrom<Value>,
@@ -654,7 +660,7 @@ where
let previous_average: f32 = average
.unwrap_or_else(|| {
key.checked_sub(1)
.and_then(|previous_average_key| self.get(&previous_average_key))
.and_then(|previous_average_key| self.get_or_import(&previous_average_key))
.unwrap_or_default()
})
.into();
@@ -832,7 +838,7 @@ where
keys.iter().for_each(|key| {
if previous_max.is_none() {
key.checked_sub(1)
.and_then(|previous_max_key| self.get(&previous_max_key))
.and_then(|previous_max_key| self.get_or_import(&previous_max_key))
.and_then(|v| previous_max.replace(v));
}
@@ -850,4 +856,38 @@ where
self.insert(*key, previous_max.unwrap());
});
}
pub fn multi_insert_min(&mut self, keys: &[Key], source: &mut Self, min_excluded: Value)
where
Value: Default + PartialOrd,
{
let mut previous_min = None;
keys.iter().for_each(|key| {
if previous_min.is_none() {
if let Some(value) = key
.checked_sub(1)
.and_then(|previous_min_key| self.get_or_import(&previous_min_key))
{
if value > min_excluded {
previous_min.replace(value);
}
}
}
let last_value = source.get_or_import(key).unwrap_or_else(|| {
dbg!(key);
panic!()
});
if last_value > min_excluded
&& (previous_min.is_none()
|| previous_min.is_some_and(|previous_min| previous_min > last_value))
{
previous_min.replace(last_value);
}
self.insert(*key, previous_min.unwrap_or_default());
});
}
}

2
parser/src/structs/mod.rs
View File

@@ -26,6 +26,7 @@ mod liquidity;
mod map_value;
mod partial_txout_data;
mod price;
mod ram;
mod sent_data;
mod serialized_btreemap;
mod serialized_vec;
@@ -61,6 +62,7 @@ pub use liquidity::*;
pub use map_value::*;
pub use partial_txout_data::*;
pub use price::*;
pub use ram::*;
pub use sent_data::*;
pub use serialized_btreemap::*;
pub use serialized_vec::*;

26
parser/src/structs/ram.rs Normal file
View File

@@ -0,0 +1,26 @@
use derive_deref::Deref;
use memory_stats::memory_stats;
use sysinfo::System;
use crate::structs::Config;
#[allow(clippy::upper_case_acronyms)]
#[derive(Deref)]
pub struct RAM(System);
impl RAM {
pub fn new() -> Self {
Self(System::new_all())
}
pub fn max_exceeded(&self, config: &Config) -> bool {
let ram_used = memory_stats().unwrap().physical_mem as f64;
if let Some(max_ram) = config.max_ram {
(ram_used / 1_000_000_000.0) > max_ram
} else {
let ram_total = self.total_memory() as f64;
ram_used / ram_total > 0.75
}
}
}