Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-04 11:19:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
k
2024-10-20 18:31:43 +02:00
parent ffa4871035
commit 5b9d599e83
47 changed files with 666 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -22,6 +22,8 @@
- Added `Drawdown`
- Added `Adjusted Value Created`, `Adjusted Value Destroyed` and `Adjusted Spent Output Profit Ratio` to all entities
- Added `Realized Profit To Loss Ratio` to all entities
- Added `Hash Price Min`
- Added `Hash Price Rebound`
- Removed datasets split by liquidity for all datasets already split by any address kind, while fun to have, they took time to compute, ram, and space to store and no one was actually checking them
- Fixed a lot of values in split by liquidity datasets
@@ -42,6 +44,9 @@
- Added config print at the start of the program
- Compressed `empty_address_data` struct to save space (should shave of between up to 50% of the `address_index_to_empty_address_data` database)
- Doubled the number of `txid_to_tx_data` databases from 4096 to 8192
- Added `--recompute_computed true` argument, to allow recomputation of computed datasets in case of a bug
- Fixed not saved arguments, not being processed properly
- Fixed bug in `generic_map.multi_insert_simple_average`
## Server

83
parser/Cargo.lock generated
View File

@@ -920,7 +920,7 @@ dependencies = [
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
"windows-core 0.52.0",
]
[[package]]
@@ -1184,6 +1184,15 @@ dependencies = [
"libc",
]
[[package]]
name = "ntapi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
dependencies = [
"winapi",
]
[[package]]
name = "num-format"
version = "0.4.4"
@@ -1337,6 +1346,7 @@ dependencies = [
"sanakirja",
"serde",
"serde_json",
"sysinfo",
"toml",
"zstd",
]
@@ -1829,6 +1839,20 @@ dependencies = [
"futures-core",
]
[[package]]
name = "sysinfo"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
"windows",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
@@ -2232,6 +2256,16 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
"windows-core 0.57.0",
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.52.0"
@@ -2241,17 +2275,60 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.1.2",
"windows-targets",
]
[[package]]
name = "windows-implement"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-interface"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-strings",
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.2.0"
@@ -2267,7 +2344,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-targets",
]

1
parser/Cargo.toml
View File

@@ -26,5 +26,6 @@ reqwest = { version = "0.12.8", features = ["blocking", "json"] }
sanakirja = "1.4.2"
serde = { version = "1.0.210", features = ["derive"] }
serde_json = "1.0.128"
sysinfo = "0.32.0"
toml = "0.8.19"
zstd = "0.13.2"

20
parser/src/actions/iter_blocks.rs
View File

@@ -1,6 +1,5 @@
use std::{collections::BTreeSet, time::Instant};
use chrono::Datelike;
use export::ExportedData;
use itertools::Itertools;
@@ -13,9 +12,9 @@ use crate::{
datasets::{AllDatasets, ComputeData},
io::OUTPUTS_FOLDER_PATH,
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{DateData, MapKey, Timestamp},
structs::{DateData, MapKey, Timestamp, RAM},
utils::{generate_allocation_files, log, time},
Config, Exit, Height,
Config, Date, Exit, Height,
};
pub fn iter_blocks(
@@ -26,7 +25,7 @@ pub fn iter_blocks(
) -> color_eyre::Result<()> {
log("Starting...");
let mut datasets = AllDatasets::import()?;
let mut datasets = AllDatasets::import(config)?;
log("Imported datasets");
@@ -58,6 +57,8 @@ pub fn iter_blocks(
let mut block_iter = block_receiver.iter();
let ram = RAM::new();
'parsing: loop {
let instant = Instant::now();
@@ -174,10 +175,12 @@ pub fn iter_blocks(
if is_date_last_block {
height += blocks_loop_i;
let is_new_month = next_block_date
.map_or(true, |next_block_date| next_block_date.day() == 1);
let is_new_year = next_block_date.as_ref().map_or(true, Date::is_new_year);
if is_new_month || height.is_close_to_end(approx_block_count) {
if is_new_year
|| ram.max_exceeded(config)
|| height.is_close_to_end(approx_block_count)
{
break 'days;
}
@@ -193,11 +196,12 @@ pub fn iter_blocks(
let last_height = height - 1_u32;
log(&format!(
"Parsing month took {} seconds (last height: {last_height})\n",
"Parsing group took {} seconds (last height: {last_height})\n",
instant.elapsed().as_secs_f32(),
));
if first_unsafe_heights.computed <= last_height {
log("Computing datasets...");
time("Computing datasets", || {
let dates = processed_dates.into_iter().collect_vec();

2
parser/src/actions/min_height.rs
View File

@@ -87,7 +87,7 @@ pub fn find_first_inserted_unsafe_height(
let computed = datasets_min_initial_states.computed.last_date.and_then(
|last_date| datasets.date_metadata
.last_height
.get(&last_date)
.get_or_import(&last_date)
.and_then(|last_date_height| {
if datasets_min_initial_states.computed.last_height.map_or(true, |last_height| {
last_height < last_date_height

9
parser/src/datasets/_traits/any_dataset.rs
View File

@@ -283,4 +283,13 @@ pub trait AnyDataset {
.for_each(|map| map.post_export())
});
}
fn reset_computed(&self) {
self.to_all_computed_date_map_vec()
.iter()
.for_each(|map| map.delete_files());
self.to_all_computed_height_map_vec()
.iter()
.for_each(|map| map.delete_files());
}
}

28
parser/src/datasets/_traits/min_initial_state.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use crate::structs::{AnyDateMap, AnyHeightMap, Date, Height};
use crate::structs::{AnyDateMap, AnyHeightMap, Config, Date, Height};
use super::{AnyDataset, AnyDatasets};
@@ -16,17 +16,17 @@ impl MinInitialStates {
self.computed = other.computed;
}
pub fn compute_from_dataset(dataset: &dyn AnyDataset) -> Self {
pub fn compute_from_dataset(dataset: &dyn AnyDataset, config: &Config) -> Self {
Self {
inserted: MinInitialState::compute_from_dataset(dataset, Mode::Inserted),
computed: MinInitialState::compute_from_dataset(dataset, Mode::Computed),
inserted: MinInitialState::compute_from_dataset(dataset, Mode::Inserted, config),
computed: MinInitialState::compute_from_dataset(dataset, Mode::Computed, config),
}
}
pub fn compute_from_datasets(datasets: &dyn AnyDatasets) -> Self {
pub fn compute_from_datasets(datasets: &dyn AnyDatasets, config: &Config) -> Self {
Self {
inserted: MinInitialState::compute_from_datasets(datasets, Mode::Inserted),
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed),
inserted: MinInitialState::compute_from_datasets(datasets, Mode::Inserted, config),
computed: MinInitialState::compute_from_datasets(datasets, Mode::Computed, config),
}
}
}
@@ -52,7 +52,7 @@ impl MinInitialState {
// self.last_height = other.last_height;
// }
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode) -> Self {
fn compute_from_datasets(datasets: &dyn AnyDatasets, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {
let contains_date_maps = |dataset: &&(dyn AnyDataset + Sync + Send)| {
@@ -111,6 +111,11 @@ impl MinInitialState {
}
}
Mode::Computed => {
if config.recompute_computed() {
// datasets.reset_computed();
return Self::default();
}
let contains_date_maps = |dataset: &&(dyn AnyDataset + Sync + Send)| {
!dataset.to_all_computed_date_map_vec().is_empty()
};
@@ -197,7 +202,7 @@ impl MinInitialState {
)
}
fn compute_from_dataset(dataset: &dyn AnyDataset, mode: Mode) -> Self {
fn compute_from_dataset(dataset: &dyn AnyDataset, mode: Mode, config: &Config) -> Self {
match mode {
Mode::Inserted => {
let date_vec = dataset.to_all_inserted_date_map_vec();
@@ -215,6 +220,11 @@ impl MinInitialState {
}
}
Mode::Computed => {
if config.recompute_computed() {
dataset.reset_computed();
return Self::default();
}
let date_vec = dataset.to_all_computed_date_map_vec();
let height_vec = dataset.to_all_computed_height_map_vec();

6
parser/src/datasets/address/all_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Allocative)]
@@ -18,7 +18,7 @@ pub struct AllAddressesMetadataDataset {
}
impl AllAddressesMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -30,7 +30,7 @@ impl AllAddressesMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

15
parser/src/datasets/address/cohort.rs
View File

@@ -1,4 +1,5 @@
use allocative::Allocative;
use itertools::Itertools;
use crate::{
@@ -6,7 +7,7 @@ use crate::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
states::{AddressCohortId, DurableStates},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Date, Height},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
};
use super::cohort_metadata::MetadataDataset;
@@ -23,19 +24,23 @@ pub struct CohortDataset {
}
impl CohortDataset {
pub fn import(parent_path: &str, id: AddressCohortId) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
id: AddressCohortId,
config: &Config,
) -> color_eyre::Result<Self> {
let name = id.as_name().map(|s| s.to_owned());
let split = id.as_split();
let mut s = Self {
min_initial_states: MinInitialStates::default(),
split,
metadata: MetadataDataset::import(parent_path, &name)?,
subs: SubDataset::import(parent_path, &name)?,
metadata: MetadataDataset::import(parent_path, &name, config)?,
subs: SubDataset::import(parent_path, &name, config)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/address/cohort_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -18,7 +18,11 @@ pub struct MetadataDataset {
}
impl MetadataDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -35,7 +39,7 @@ impl MetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/address/mod.rs
View File

@@ -8,7 +8,7 @@ use rayon::prelude::*;
use crate::{
states::SplitByAddressCohort,
structs::{BiMap, Height},
structs::{BiMap, Config, Height},
Date,
};
@@ -26,13 +26,13 @@ pub struct AddressDatasets {
}
impl AddressDatasets {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByAddressCohort::<CohortDataset>::default();
cohorts
.as_vec()
.into_par_iter()
.map(|(_, id)| (id, CohortDataset::import(parent_path, id)))
.map(|(_, id)| (id, CohortDataset::import(parent_path, id, config)))
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
@@ -43,13 +43,13 @@ impl AddressDatasets {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
metadata: AllAddressesMetadataDataset::import(parent_path)?,
metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
cohorts,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
Ok(s)
}

6
parser/src/datasets/block_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyHeightMap, Date, HeightMap, Timestamp},
structs::{AnyHeightMap, Config, Date, HeightMap, Timestamp},
};
use super::{InsertData, MinInitialStates};
@@ -17,7 +17,7 @@ pub struct BlockMetadataDataset {
}
impl BlockMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -28,7 +28,7 @@ impl BlockMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/coindays.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, AnyHeightMap},
structs::{AnyDateMap, AnyHeightMap, Config},
DateMap, HeightMap,
};
@@ -18,7 +18,7 @@ pub struct CoindaysDataset {
}
impl CoindaysDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -29,7 +29,7 @@ impl CoindaysDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

14
parser/src/datasets/cointime.rs
View File

@@ -1,7 +1,7 @@
use allocative::Allocative;
use crate::{
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, DateMap, Height},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height},
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
HeightMap,
};
@@ -71,7 +71,7 @@ pub struct CointimeDataset {
}
impl CointimeDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -79,7 +79,7 @@ impl CointimeDataset {
active_cap: BiMap::new_bin(1, &f("active_cap")),
active_price: BiMap::new_bin(1, &f("active_price")),
active_price_ratio: RatioDataset::import(parent_path, "active_price")?,
active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
active_supply: BiMap::new_bin(1, &f("active_supply")),
active_supply_3m_net_change: BiMap::new_bin(1, &f("active_supply_3m_net_change")),
active_supply_net_change: BiMap::new_bin(1, &f("active_supply_net_change")),
@@ -101,7 +101,7 @@ impl CointimeDataset {
),
cointime_cap: BiMap::new_bin(1, &f("cointime_cap")),
cointime_price: BiMap::new_bin(1, &f("cointime_price")),
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price")?,
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
cointime_value_created: HeightMap::new_bin(1, &f("cointime_value_created")),
cointime_value_created_1d_sum: DateMap::new_bin(1, &f("cointime_value_created_1d_sum")),
cointime_value_destroyed: HeightMap::new_bin(1, &f("cointime_value_destroyed")),
@@ -141,14 +141,14 @@ impl CointimeDataset {
total_cointime_value_stored: BiMap::new_bin(1, &f("total_cointime_value_stored")),
true_market_deviation: BiMap::new_bin(1, &f("true_market_deviation")),
true_market_mean: BiMap::new_bin(1, &f("true_market_mean")),
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean")?,
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
1,
&f("true_market_net_unrealized_profit_and_loss"),
),
vaulted_cap: BiMap::new_bin(1, &f("vaulted_cap")),
vaulted_price: BiMap::new_bin(1, &f("vaulted_price")),
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price")?,
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
vaulted_supply: BiMap::new_bin(1, &f("vaulted_supply")),
vaulted_supply_3m_net_change: BiMap::new_bin(1, &f("vaulted_supply_3m_net_change")),
vaulted_supply_net_change: BiMap::new_bin(1, &f("vaulted_supply_net_change")),
@@ -157,7 +157,7 @@ impl CointimeDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/constant.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use crate::structs::{AnyBiMap, BiMap};
use crate::structs::{AnyBiMap, BiMap, Config};
use super::{AnyDataset, ComputeData, MinInitialStates};
@@ -16,7 +16,7 @@ pub struct ConstantDataset {
}
impl ConstantDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -29,7 +29,7 @@ impl ConstantDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/date_metadata.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, DateMap, Height},
structs::{AnyDateMap, Config, DateMap, Height},
};
use super::{InsertData, MinInitialStates};
@@ -17,7 +17,7 @@ pub struct DateMetadataDataset {
}
impl DateMetadataDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -28,7 +28,7 @@ impl DateMetadataDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

47
parser/src/datasets/mining.rs
View File

@@ -6,7 +6,7 @@ use crate::{
datasets::AnyDataset,
structs::{
date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap,
AnyDateMap, AnyHeightMap, BiMap, DateMap, Height, HeightMap, MapKey,
AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height, HeightMap, MapKey,
},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
@@ -80,6 +80,7 @@ pub struct MiningDataset {
pub blocks_mined_1y_sum: DateMap<usize>,
pub blocks_mined_1y_target: DateMap<usize>,
pub cumulative_block_size: BiMap<f32>,
pub cumulative_block_size_gigabytes: BiMap<f32>,
pub subsidy_1y_sum: DateMap<f64>,
pub subsidy_in_dollars_1y_sum: DateMap<f64>,
pub cumulative_subsidy: BiMap<f64>,
@@ -104,8 +105,8 @@ pub struct MiningDataset {
pub hash_rate_1m_sma: DateMap<f32>,
pub hash_rate_2m_sma: DateMap<f32>,
pub hash_price: DateMap<f64>,
// pub hash_price_min: DateMap<f64>,
// pub hash_price_rebound: DateMap<f64>,
pub hash_price_min: DateMap<f64>,
pub hash_price_rebound: DateMap<f64>,
pub difficulty_adjustment: DateMap<f64>,
pub block_size_recap: DateRecapDataset<f32>, // in MB
pub block_weight_recap: DateRecapDataset<f32>, // in MB
@@ -124,7 +125,7 @@ pub struct MiningDataset {
}
impl MiningDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -206,6 +207,10 @@ impl MiningDataset {
.add_min(),
)?,
cumulative_block_size: BiMap::new_bin(1, &f("cumulative_block_size")),
cumulative_block_size_gigabytes: BiMap::new_bin(
1,
&f("cumulative_block_size_gigabytes"),
),
block_weight: HeightMap::new_bin(1, &f("block_weight")),
block_weight_recap: RecapDataset::import(
&f("block_weight_1d"),
@@ -251,11 +256,13 @@ impl MiningDataset {
hash_rate_1m_sma: DateMap::new_bin(1, &f("hash_rate_1m_sma")),
hash_rate_2m_sma: DateMap::new_bin(1, &f("hash_rate_2m_sma")),
hash_price: DateMap::new_bin(1, &f("hash_price")),
hash_price_min: DateMap::new_bin(1, &f("hash_price_min")),
hash_price_rebound: DateMap::new_bin(1, &f("hash_price_rebound")),
puell_multiple: DateMap::new_bin(1, &f("puell_multiple")),
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}
@@ -536,13 +543,22 @@ impl MiningDataset {
last_height,
);
self.cumulative_block_size
.height
.multi_insert_cumulative(heights, &mut self.block_size);
self.cumulative_block_size_gigabytes
.multi_insert_simple_transform(heights, dates, &mut self.cumulative_block_size, &|v| {
v / 1000.0
});
// https://hashrateindex.com/blog/what-is-bitcoins-hashrate/
self.hash_rate.multi_insert(dates, |date| {
let blocks_mined = self.blocks_mined.get_or_import(date).unwrap();
let difficulty = self.difficulty.date.get_or_import(date).unwrap();
((blocks_mined as f64 / date.get_day_completion() * TARGET_BLOCKS_PER_DAY as f64)
(blocks_mined as f64 / (date.get_day_completion() * TARGET_BLOCKS_PER_DAY as f64)
* difficulty
* 2.0_f64.powi(32))
/ 600.0
@@ -570,11 +586,20 @@ impl MiningDataset {
self.hash_price.multi_insert(dates, |date| {
let coinbase_in_dollars = self.coinbase_in_dollars_1d_sum.get_or_import(date).unwrap();
let hashrate = self.hash_rate.get_or_import(date).unwrap();
let hash_rate = self.hash_rate.get_or_import(date).unwrap();
coinbase_in_dollars as f64 / hashrate / 1_000.0
coinbase_in_dollars as f64 / hash_rate / 1_000.0
});
self.hash_price_min
.multi_insert_min(dates, &mut self.hash_price, 0.0);
self.hash_price_rebound.multi_insert_percentage(
dates,
&mut self.hash_price,
&mut self.hash_price_min,
);
self.puell_multiple.multi_insert_divide(
dates,
&mut self.coinbase_in_dollars_1d_sum,
@@ -721,6 +746,7 @@ impl AnyDataset for MiningDataset {
&self.cumulative_subsidy,
&self.cumulative_subsidy_in_dollars,
&self.cumulative_block_size,
&self.cumulative_block_size_gigabytes,
]
}
@@ -733,6 +759,7 @@ impl AnyDataset for MiningDataset {
&mut self.cumulative_subsidy,
&mut self.cumulative_subsidy_in_dollars,
&mut self.cumulative_block_size,
&mut self.cumulative_block_size_gigabytes,
]
}
@@ -778,6 +805,8 @@ impl AnyDataset for MiningDataset {
&self.hash_rate_1m_sma,
&self.hash_rate_2m_sma,
&self.hash_price,
&self.hash_price_min,
&self.hash_price_rebound,
&self.puell_multiple,
&self.difficulty_adjustment,
]
@@ -825,6 +854,8 @@ impl AnyDataset for MiningDataset {
&mut self.hash_rate_1m_sma,
&mut self.hash_rate_2m_sma,
&mut self.hash_price,
&mut self.hash_price_min,
&mut self.hash_price_rebound,
&mut self.puell_multiple,
&mut self.difficulty_adjustment,
]

26
parser/src/datasets/mod.rs
View File

@@ -45,7 +45,7 @@ use crate::{
// UTXOCohortsReceivedStates,
UTXOCohortsSentStates,
},
structs::{Amount, Date, Height, Price, Timestamp},
structs::{Amount, Config, Date, Height, Price, Timestamp},
};
pub struct InsertData<'a> {
@@ -102,28 +102,28 @@ pub struct AllDatasets {
const DATASETS_PATH: &str = "../datasets";
impl AllDatasets {
pub fn import() -> color_eyre::Result<Self> {
pub fn import(config: &Config) -> color_eyre::Result<Self> {
let path = DATASETS_PATH;
let price = PriceDatasets::import(path)?;
let price = PriceDatasets::import(path, config)?;
let constant = ConstantDataset::import(path)?;
let constant = ConstantDataset::import(path, config)?;
let date_metadata = DateMetadataDataset::import(path)?;
let date_metadata = DateMetadataDataset::import(path, config)?;
let cointime = CointimeDataset::import(path)?;
let cointime = CointimeDataset::import(path, config)?;
let coindays = CoindaysDataset::import(path)?;
let coindays = CoindaysDataset::import(path, config)?;
let mining = MiningDataset::import(path)?;
let mining = MiningDataset::import(path, config)?;
let block_metadata = BlockMetadataDataset::import(path)?;
let block_metadata = BlockMetadataDataset::import(path, config)?;
let transaction = TransactionDataset::import(path)?;
let transaction = TransactionDataset::import(path, config)?;
let address = AddressDatasets::import(path)?;
let address = AddressDatasets::import(path, config)?;
let utxo = UTXODatasets::import(path)?;
let utxo = UTXODatasets::import(path, config)?;
let mut s = Self {
min_initial_states: MinInitialStates::default(),
@@ -141,7 +141,7 @@ impl AllDatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
s.export_meta_files()?;

32
parser/src/datasets/price/mod.rs
View File

@@ -11,7 +11,7 @@ pub use ohlc::*;
use crate::{
price::{Binance, Kibo, Kraken},
structs::{
Amount, AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height,
Amount, AnyBiMap, AnyDateMap, BiMap, Config, Date, DateMap, DateMapChunkId, Height,
HeightMapChunkId, MapKey, Timestamp,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
@@ -90,7 +90,7 @@ pub struct PriceDatasets {
}
impl PriceDatasets {
pub fn import(datasets_path: &str) -> color_eyre::Result<Self> {
pub fn import(datasets_path: &str, config: &Config) -> color_eyre::Result<Self> {
let price_path = "../price";
let f = |s: &str| format!("{datasets_path}/{s}");
@@ -113,31 +113,31 @@ impl PriceDatasets {
close: BiMap::new_bin(1, &f("close")),
market_cap: BiMap::new_bin(1, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, &f("price_1w_sma")),
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma")?,
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
price_1m_sma: BiMap::new_bin(1, &f("price_1m_sma")),
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma")?,
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
price_1y_sma: BiMap::new_bin(1, &f("price_1y_sma")),
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma")?,
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
price_2y_sma: BiMap::new_bin(1, &f("price_2y_sma")),
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma")?,
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
price_4y_sma: BiMap::new_bin(1, &f("price_4y_sma")),
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma")?,
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
price_8d_sma: BiMap::new_bin(1, &f("price_8d_sma")),
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma")?,
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
price_13d_sma: BiMap::new_bin(1, &f("price_13d_sma")),
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma")?,
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
price_21d_sma: BiMap::new_bin(1, &f("price_21d_sma")),
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma")?,
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
price_34d_sma: BiMap::new_bin(1, &f("price_34d_sma")),
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma")?,
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
price_55d_sma: BiMap::new_bin(1, &f("price_55d_sma")),
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma")?,
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
price_89d_sma: BiMap::new_bin(1, &f("price_89d_sma")),
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma")?,
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
price_144d_sma: BiMap::new_bin(1, &f("price_144d_sma")),
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma")?,
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
price_200w_sma: BiMap::new_bin(1, &f("price_200w_sma")),
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma")?,
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
price_1d_total_return: DateMap::new_bin(1, &f("price_1d_total_return")),
price_1m_total_return: DateMap::new_bin(1, &f("price_1m_total_return")),
price_6m_total_return: DateMap::new_bin(1, &f("price_6m_total_return")),
@@ -169,7 +169,7 @@ impl PriceDatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

11
parser/src/datasets/subs/capitalization.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::CapitalizationState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
utils::ONE_MONTH_IN_DAYS,
};
@@ -23,7 +23,11 @@ pub struct CapitalizationDataset {
}
impl CapitalizationDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -44,11 +48,12 @@ impl CapitalizationDataset {
"{}realized_price",
name.as_ref().map_or("".to_owned(), |n| format!("{n}-"))
),
config,
)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/input.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::InputState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config},
DateMap, HeightMap,
};
@@ -20,7 +20,11 @@ pub struct InputSubDataset {
}
impl InputSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -38,7 +42,7 @@ impl InputSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

22
parser/src/datasets/subs/mod.rs
View File

@@ -20,7 +20,7 @@ pub use supply::*;
pub use unrealized::*;
pub use utxo::*;
use crate::datasets::AnyDataset;
use crate::{datasets::AnyDataset, structs::Config};
use super::AnyDatasetGroup;
@@ -37,16 +37,20 @@ pub struct SubDataset {
}
impl SubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let s = Self {
capitalization: CapitalizationDataset::import(parent_path, name)?,
input: InputSubDataset::import(parent_path, name)?,
capitalization: CapitalizationDataset::import(parent_path, name, config)?,
input: InputSubDataset::import(parent_path, name, config)?,
// output: OutputSubDataset::import(parent_path)?,
price_paid: PricePaidSubDataset::import(parent_path, name)?,
realized: RealizedSubDataset::import(parent_path, name)?,
supply: SupplySubDataset::import(parent_path, name)?,
unrealized: UnrealizedSubDataset::import(parent_path, name)?,
utxo: UTXOSubDataset::import(parent_path, name)?,
price_paid: PricePaidSubDataset::import(parent_path, name, config)?,
realized: RealizedSubDataset::import(parent_path, name, config)?,
supply: SupplySubDataset::import(parent_path, name, config)?,
unrealized: UnrealizedSubDataset::import(parent_path, name, config)?,
utxo: UTXOSubDataset::import(parent_path, name, config)?,
};
Ok(s)

10
parser/src/datasets/subs/price_paid.rs
View File

@@ -4,7 +4,7 @@ use itertools::Itertools;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::PricePaidState,
structs::{AnyBiMap, BiMap, Date, Height},
structs::{AnyBiMap, BiMap, Config, Date, Height},
};
#[derive(Default, Allocative)]
@@ -34,7 +34,11 @@ pub struct PricePaidSubDataset {
}
impl PricePaidSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -68,7 +72,7 @@ impl PricePaidSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/subs/ratio.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, MinInitialStates},
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
@@ -31,7 +31,7 @@ pub struct RatioDataset {
}
impl RatioDataset {
pub fn import(parent_path: &str, name: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, name: &str, config: &Config) -> color_eyre::Result<Self> {
let f_ratio = |s: &str| format!("{parent_path}/market_price_to_{name}_{s}");
let f_price = |s: &str| format!("{parent_path}/{name}_{s}");
@@ -61,7 +61,7 @@ impl RatioDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/realized.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::RealizedState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Price},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Price},
utils::ONE_MONTH_IN_DAYS,
DateMap, HeightMap,
};
@@ -46,7 +46,11 @@ pub struct RealizedSubDataset {
}
impl RealizedSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -114,7 +118,7 @@ impl RealizedSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/supply.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::SupplyState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -20,7 +20,11 @@ pub struct SupplySubDataset {
}
impl SupplySubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -45,7 +49,7 @@ impl SupplySubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/unrealized.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::UnrealizedState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -27,7 +27,11 @@ pub struct UnrealizedSubDataset {
}
impl UnrealizedSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -68,7 +72,7 @@ impl UnrealizedSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

10
parser/src/datasets/subs/utxo.rs
View File

@@ -3,7 +3,7 @@ use allocative::Allocative;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::UTXOState,
structs::{AnyBiMap, BiMap},
structs::{AnyBiMap, BiMap, Config},
};
#[derive(Default, Allocative)]
@@ -15,7 +15,11 @@ pub struct UTXOSubDataset {
}
impl UTXOSubDataset {
pub fn import(parent_path: &str, name: &Option<String>) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
name: &Option<String>,
config: &Config,
) -> color_eyre::Result<Self> {
let f = |s: &str| {
if let Some(name) = name {
format!("{parent_path}/{name}/{s}")
@@ -31,7 +35,7 @@ impl UTXOSubDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

6
parser/src/datasets/transaction.rs
View File

@@ -2,7 +2,7 @@ use allocative::Allocative;
use crate::{
datasets::InsertData,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, HeightMap},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, HeightMap},
utils::{
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
},
@@ -54,7 +54,7 @@ pub struct TransactionDataset {
}
impl TransactionDataset {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let f = |s: &str| format!("{parent_path}/{s}");
let mut s = Self {
@@ -105,7 +105,7 @@ impl TransactionDataset {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

12
parser/src/datasets/utxo/dataset.rs
View File

@@ -6,7 +6,7 @@ use crate::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
states::UTXOCohortId,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Date, Height},
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
};
#[derive(Default, Allocative)]
@@ -19,17 +19,21 @@ pub struct UTXODataset {
}
impl UTXODataset {
pub fn import(parent_path: &str, id: UTXOCohortId) -> color_eyre::Result<Self> {
pub fn import(
parent_path: &str,
id: UTXOCohortId,
config: &Config,
) -> color_eyre::Result<Self> {
let name = id.name().to_owned();
let mut s = Self {
min_initial_states: MinInitialStates::default(),
id,
subs: SubDataset::import(parent_path, &Some(name))?,
subs: SubDataset::import(parent_path, &Some(name), config)?,
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s));
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}

8
parser/src/datasets/utxo/mod.rs
View File

@@ -9,7 +9,7 @@ use itertools::Itertools;
use crate::{
datasets::AnyDatasets,
states::{SplitByUTXOCohort, UTXOCohortId},
structs::{BiMap, Date, Height},
structs::{BiMap, Config, Date, Height},
};
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates};
@@ -22,13 +22,13 @@ pub struct UTXODatasets {
}
impl UTXODatasets {
pub fn import(parent_path: &str) -> color_eyre::Result<Self> {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByUTXOCohort::<UTXODataset>::default();
cohorts
.as_vec()
.into_par_iter()
.map(|(_, id)| (id, UTXODataset::import(parent_path, id)))
.map(|(_, id)| (id, UTXODataset::import(parent_path, id, config)))
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
@@ -43,7 +43,7 @@ impl UTXODatasets {
};
s.min_initial_states
.consume(MinInitialStates::compute_from_datasets(&s));
.consume(MinInitialStates::compute_from_datasets(&s, config));
Ok(s)
}

2
parser/src/io/binary.rs
View File

@@ -69,6 +69,8 @@ impl Binary {
where
T: Debug + Encode,
{
// log(&format!("Exporting: {:?}", path));
match Self::type_from_path(path) {
BinaryType::Compressed => Self::export_compressed(path, value),
BinaryType::Raw => Self::export_raw(path, value),

2
parser/src/io/serialization.rs
View File

@@ -95,7 +95,7 @@ impl Serialization {
let path = path.to_str().unwrap();
let res = Binary::export(
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path,)),
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path)),
value,
);

2
parser/src/structs/any_map.rs
View File

@@ -23,4 +23,6 @@ pub trait AnyMap {
fn pre_export(&mut self);
fn export(&self) -> color_eyre::Result<()>;
fn post_export(&mut self);
fn delete_files(&self);
}

48
parser/src/structs/config.rs
View File

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use crate::log;
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)]
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[command(version, about, long_about = None)]
pub struct Config {
/// Bitcoin data directory path, saved
@@ -32,6 +32,10 @@ pub struct Config {
#[arg(long, value_name = "SECONDS")]
pub delay: Option<u64>,
/// Maximum ram you want the program to use in GB, default: 75% of total, not saved
#[arg(long, value_name = "GB")]
pub max_ram: Option<f64>,
/// Start a dry run, default: false, not saved
#[arg(long, value_name = "BOOL")]
dry_run: Option<bool>,
@@ -39,6 +43,10 @@ pub struct Config {
/// Record ram usage, default: false, not saved
#[arg(long, value_name = "BOOL")]
record_ram_usage: Option<bool>,
/// Recompute all computed datasets, default: false, not saved
#[arg(long, value_name = "BOOL")]
recompute_computed: Option<bool>,
}
impl Config {
@@ -50,40 +58,48 @@ impl Config {
toml::from_str(&contents).unwrap_or_default()
});
let config_args = Config::parse();
let mut config_args = Config::parse();
if let Some(datadir) = config_args.datadir {
if let Some(datadir) = config_args.datadir.take() {
config_saved.datadir = Some(datadir);
}
if let Some(rpcconnect) = config_args.rpcconnect {
if let Some(rpcconnect) = config_args.rpcconnect.take() {
config_saved.rpcconnect = Some(rpcconnect);
}
if let Some(rpcport) = config_args.rpcport {
if let Some(rpcport) = config_args.rpcport.take() {
config_saved.rpcport = Some(rpcport);
}
if let Some(rpcuser) = config_args.rpcuser {
if let Some(rpcuser) = config_args.rpcuser.take() {
config_saved.rpcuser = Some(rpcuser);
}
if let Some(rpcpassword) = config_args.rpcpassword {
if let Some(rpcpassword) = config_args.rpcpassword.take() {
config_saved.rpcpassword = Some(rpcpassword);
}
if let Some(delay) = config_args.delay {
if let Some(delay) = config_args.delay.take() {
config_saved.delay = Some(delay);
}
if let Some(max_ram) = config_args.max_ram.take() {
config_saved.max_ram = Some(max_ram);
}
// Done importing
let config = config_saved;
let mut config = config_saved;
config.check();
config.write()?;
config.dry_run = config_args.dry_run.take();
config.record_ram_usage = config_args.record_ram_usage.take();
config.recompute_computed = config_args.recompute_computed.take();
log("---");
log("Configuration:");
log(&format!("datadir: {:?}", config.datadir));
@@ -92,10 +108,20 @@ impl Config {
log(&format!("rpcuser: {:?}", config.rpcuser));
log(&format!("rpcpassword: {:?}", config.rpcpassword));
log(&format!("delay: {:?}", config.delay));
log(&format!("max_ram: {:?}", config.max_ram));
log(&format!("dry_run: {:?}", config.dry_run));
log(&format!("record_ram_usage: {:?}", config.record_ram_usage));
log(&format!(
"recompute_computed: {:?}",
config.recompute_computed
));
log("---");
if config_args != Config::default() {
dbg!(config_args);
panic!("Didn't consume the full config")
}
Ok(config)
}
@@ -132,4 +158,8 @@ impl Config {
pub fn record_ram_usage(&self) -> bool {
self.record_ram_usage.is_some_and(|b| b)
}
pub fn recompute_computed(&self) -> bool {
self.recompute_computed.is_some_and(|b| b)
}
}

4
parser/src/structs/date.rs
View File

@@ -73,6 +73,10 @@ impl Date {
Ordering::Greater => unreachable!("0.0 but shouldn't be called"),
}
}
pub fn is_new_year(&self) -> bool {
self.day() == 1 && self.month() == 1
}
}
impl MapKey<DateMapChunkId> for Date {

68
parser/src/structs/generic_map.rs
View File

@@ -156,9 +156,7 @@ where
if serialized.version() == s.version {
s.imported.insert(chunk_start, serialized);
} else {
s.read_dir()
.iter()
.for_each(|(_, path)| fs::remove_file(path).unwrap())
s.delete_files();
}
}
});
@@ -204,12 +202,10 @@ where
}
pub fn insert(&mut self, key: Key, value: Value) -> Value {
if !self.is_key_safe(key) {
self.to_insert
.entry(key.to_chunk_id())
.or_default()
.insert(key.to_serialized_key(), value);
}
self.to_insert
.entry(key.to_chunk_id())
.or_default()
.insert(key.to_serialized_key(), value);
value
}
@@ -341,8 +337,10 @@ where
});
let path = self.path_all.join(chunk_id.to_name());
self.serialization.export(Path::new(&path), serialized)?;
// Export last
if index == len - 1 {
if let Some(path_last) = self.path_last.as_ref() {
self.serialization
@@ -370,6 +368,12 @@ where
self.to_insert.clear();
}
fn delete_files(&self) {
self.read_dir()
.iter()
.for_each(|(_, path)| fs::remove_file(path).unwrap())
}
}
impl<Key, Value, ChunkId, Serialized> GenericMap<Key, Value, ChunkId, Serialized>
@@ -379,14 +383,16 @@ where
Key: MapKey<ChunkId>,
Serialized: MapSerialized<Key, Value, ChunkId>,
{
pub fn sum_keys(&self, keys: &[Key]) -> Value
pub fn sum_keys(&mut self, keys: &[Key]) -> Value
where
Value: Sum,
{
keys.iter().flat_map(|key| self.get(key)).sum::<Value>()
keys.iter()
.map(|key| self.get_or_import(key).unwrap())
.sum::<Value>()
}
pub fn average_keys(&self, keys: &[Key]) -> f32
pub fn average_keys(&mut self, keys: &[Key]) -> f32
where
Value: Sum,
f32: LossyFrom<Value>,
@@ -654,7 +660,7 @@ where
let previous_average: f32 = average
.unwrap_or_else(|| {
key.checked_sub(1)
.and_then(|previous_average_key| self.get(&previous_average_key))
.and_then(|previous_average_key| self.get_or_import(&previous_average_key))
.unwrap_or_default()
})
.into();
@@ -832,7 +838,7 @@ where
keys.iter().for_each(|key| {
if previous_max.is_none() {
key.checked_sub(1)
.and_then(|previous_max_key| self.get(&previous_max_key))
.and_then(|previous_max_key| self.get_or_import(&previous_max_key))
.and_then(|v| previous_max.replace(v));
}
@@ -850,4 +856,38 @@ where
self.insert(*key, previous_max.unwrap());
});
}
pub fn multi_insert_min(&mut self, keys: &[Key], source: &mut Self, min_excluded: Value)
where
Value: Default + PartialOrd,
{
let mut previous_min = None;
keys.iter().for_each(|key| {
if previous_min.is_none() {
if let Some(value) = key
.checked_sub(1)
.and_then(|previous_min_key| self.get_or_import(&previous_min_key))
{
if value > min_excluded {
previous_min.replace(value);
}
}
}
let last_value = source.get_or_import(key).unwrap_or_else(|| {
dbg!(key);
panic!()
});
if last_value > min_excluded
&& (previous_min.is_none()
|| previous_min.is_some_and(|previous_min| previous_min > last_value))
{
previous_min.replace(last_value);
}
self.insert(*key, previous_min.unwrap_or_default());
});
}
}

2
parser/src/structs/mod.rs
View File

@@ -26,6 +26,7 @@ mod liquidity;
mod map_value;
mod partial_txout_data;
mod price;
mod ram;
mod sent_data;
mod serialized_btreemap;
mod serialized_vec;
@@ -61,6 +62,7 @@ pub use liquidity::*;
pub use map_value::*;
pub use partial_txout_data::*;
pub use price::*;
pub use ram::*;
pub use sent_data::*;
pub use serialized_btreemap::*;
pub use serialized_vec::*;

26
parser/src/structs/ram.rs Normal file
View File

@@ -0,0 +1,26 @@
use derive_deref::Deref;
use memory_stats::memory_stats;
use sysinfo::System;
use crate::structs::Config;
#[allow(clippy::upper_case_acronyms)]
#[derive(Deref)]
pub struct RAM(System);
impl RAM {
pub fn new() -> Self {
Self(System::new_all())
}
pub fn max_exceeded(&self, config: &Config) -> bool {
let ram_used = memory_stats().unwrap().physical_mem as f64;
if let Some(max_ram) = config.max_ram {
(ram_used / 1_000_000_000.0) > max_ram
} else {
let ram_total = self.total_memory() as f64;
ram_used / ram_total > 0.75
}
}
}

83
server/Cargo.lock generated
View File

@@ -1309,7 +1309,7 @@ dependencies = [
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
"windows-core 0.52.0",
]
[[package]]
@@ -1681,6 +1681,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "ntapi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
dependencies = [
"winapi",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
@@ -1886,6 +1895,7 @@ dependencies = [
"sanakirja",
"serde",
"serde_json",
"sysinfo",
"toml",
"zstd",
]
@@ -3695,6 +3705,20 @@ dependencies = [
"futures-core",
]
[[package]]
name = "sysinfo"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
"windows",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
@@ -4235,6 +4259,16 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
"windows-core 0.57.0",
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.52.0"
@@ -4244,17 +4278,60 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.1.2",
"windows-targets",
]
[[package]]
name = "windows-implement"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-interface"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-strings",
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.2.0"
@@ -4270,7 +4347,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-targets",
]

2
server/src/header_map.rs
View File

@@ -149,7 +149,7 @@ impl HeaderMapUtils for HeaderMap {
"json" => self.insert_content_type_application_json(),
"html" => self.insert_content_type_text_html(),
"css" => self.insert_content_type_text_css(),
"txt" => self.insert_content_type_text_plain(),
"toml" | "txt" => self.insert_content_type_text_plain(),
"pdf" => self.insert_content_type_application_pdf(),
"woff2" => self.insert_content_type_font_woff2(),
"ico" => self.insert_content_type_image_icon(),

7
server/src/website/handlers/file.rs
View File

@@ -19,7 +19,12 @@ use super::minify_js;
const WEBSITE_PATH: &str = "../website/";
pub async fn file_handler(headers: HeaderMap, path: extract::Path<String>) -> Response {
let path = path.0.replace("..", "").replace("\\", "");
let mut path = path.0.replace("..", "").replace("\\", "");
if path.ends_with("Cargo.toml") {
path = "../server/Cargo.toml".to_owned();
}
let mut path = str_to_path(&path);
if !path.exists() {

2
website/scripts/chart.js
View File

@@ -740,7 +740,7 @@ export function initChartsElement({
// Don't trigger reactivity by design
activeDatasets().add(dataset);
const title = "Price";
const title = "BTC Price";
/** @type {SeriesBlueprint} */
let seriesBlueprint;

204
website/scripts/options.js
View File

@@ -1558,21 +1558,38 @@ function createPartialOptions(colors) {
{
scale,
name: "Size",
tree: createRecapOptions({
scale,
title: "Block Size",
color: colors.off,
unit: "Megabytes",
keySum: "date-to-block-size-1d-sum",
keyAverage: "date-to-block-size-1d-average",
keyMax: "date-to-block-size-1d-max",
key90p: "date-to-block-size-1d-90p",
key75p: "date-to-block-size-1d-75p",
keyMedian: "date-to-block-size-1d-median",
key25p: "date-to-block-size-1d-25p",
key10p: "date-to-block-size-1d-10p",
keyMin: "date-to-block-size-1d-min",
}),
tree: [
{
scale,
icon: "📏",
name: "Cumulative",
title: "Cumulative Block Size",
description: "",
unit: "Gigabytes",
bottom: [
{
title: "Size",
color: colors.off,
datasetPath: `${scale}-to-cumulative-block-size-gigabytes`,
},
],
},
...createRecapOptions({
scale,
title: "Block Size",
color: colors.off,
unit: "Megabytes",
keySum: "date-to-block-size-1d-sum",
keyAverage: "date-to-block-size-1d-average",
keyMax: "date-to-block-size-1d-max",
key90p: "date-to-block-size-1d-90p",
key75p: "date-to-block-size-1d-75p",
keyMedian: "date-to-block-size-1d-median",
key25p: "date-to-block-size-1d-25p",
key10p: "date-to-block-size-1d-10p",
keyMin: "date-to-block-size-1d-min",
}),
],
},
{
scale,
@@ -1691,21 +1708,6 @@ function createPartialOptions(colors) {
],
},
])),
{
scale,
icon: "📏",
name: "Cumulative Size",
title: "Cumulative Block Size",
description: "",
unit: "Megabytes",
bottom: [
{
title: "Size",
color: colors.off,
datasetPath: `${scale}-to-cumulative-block-size`,
},
],
},
],
};
}
@@ -2224,62 +2226,102 @@ function createPartialOptions(colors) {
},
{
scale,
icon: "⛏️",
name: "Hash Rate",
title: "Hash Rate",
description: "",
unit: "ExaHash / Second",
bottom: [
name: "Hash",
tree: [
{
title: "1M SMA",
color: colors.momentumYellow,
datasetPath: `date-to-hash-rate-1m-sma`,
scale,
icon: "⛏️",
name: "Rate",
title: "Hash Rate",
description: "",
unit: "ExaHash / Second",
bottom: [
{
title: "1M SMA",
color: colors.momentumYellow,
datasetPath: `date-to-hash-rate-1m-sma`,
},
{
title: "1W SMA",
color: colors.bitcoin,
datasetPath: `date-to-hash-rate-1w-sma`,
},
{
title: "Rate",
color: colors.darkBitcoin,
datasetPath: `date-to-hash-rate`,
},
],
},
{
title: "1W SMA",
color: colors.bitcoin,
datasetPath: `date-to-hash-rate-1w-sma`,
scale,
icon: "🎗️",
name: "Ribbon",
title: "Hash Ribbon",
description: "",
unit: "ExaHash / Second",
bottom: [
{
title: "1M SMA",
color: colors.profit,
datasetPath: `date-to-hash-rate-1m-sma`,
},
{
title: "2M SMA",
color: colors.loss,
datasetPath: `date-to-hash-rate-2m-sma`,
},
],
},
{
title: "Rate",
color: colors.darkBitcoin,
datasetPath: `date-to-hash-rate`,
},
],
},
{
scale,
icon: "🎗️",
name: "Hash Ribbon",
title: "Hash Ribbon",
description: "",
unit: "ExaHash / Second",
bottom: [
{
title: "1M SMA",
color: colors.profit,
datasetPath: `date-to-hash-rate-1m-sma`,
},
{
title: "2M SMA",
color: colors.loss,
datasetPath: `date-to-hash-rate-2m-sma`,
},
],
},
{
scale,
icon: "🏷️",
name: "Hash Price",
title: "Hash Price",
description: "",
unit: "Dollars / (PetaHash / Second)",
bottom: [
{
title: "Price",
color: colors.dollars,
datasetPath: `date-to-hash-price`,
name: "Price",
tree: [
{
scale,
icon: "🏷️",
name: "Price",
title: "Hash Price",
description: "",
unit: "Dollars / (PetaHash / Second)",
bottom: [
{
title: "Hash Price",
color: colors.dollars,
datasetPath: `date-to-hash-price`,
},
],
},
{
scale,
icon: "😢",
name: "Min",
title: "Min Hash Price",
description: "",
unit: "Dollars / (PetaHash / Second)",
bottom: [
{
title: "Min Hash Price",
color: colors.dollars,
datasetPath: `date-to-hash-price-min`,
},
],
},
{
scale,
icon: "🤞",
name: "Rebound",
title: "Hash Price Rebound",
description: "",
unit: "Percentage",
bottom: [
{
title: "Rebound",
color: colors.yellow,
datasetPath: `date-to-hash-price-rebound`,
},
],
},
],
},
],
},

1
website/scripts/service-worker.js
View File

@@ -14,6 +14,7 @@ self.addEventListener("install", (_event) => {
"/index.html",
"/assets/fonts/satoshi/2024-09/font.var.woff2",
"/scripts/main.js",
"/scripts/options.js",
"/scripts/chart.js",
"/styles/chart.css",
"/scripts/packages/lean-qr/v2.3.4/script.js",

6
website/scripts/types/paths.d.ts vendored
View File

File diff suppressed because one or more lines are too long

1
website/scripts/types/self.d.ts vendored
View File

@@ -89,6 +89,7 @@ type Unit =
| "Dollars / (PetaHash / Second)"
| "ExaHash / Second"
| "Height"
| "Gigabytes"
| "Megabytes"
| "Percentage"
| "Ratio"