Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 07:09:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: AnyDataset DX improvements

Browse Source
This commit is contained in:
k
2024-10-28 16:48:27 +01:00
parent 36ad0b3014
commit 48a8aad20e
43 changed files with 1688 additions and 1870 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
parser/Cargo.lock generated
View File

@@ -48,7 +48,7 @@ checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -390,7 +390,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -557,7 +557,7 @@ checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -920,7 +920,7 @@ dependencies = [
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core 0.52.0",
"windows-core",
]
[[package]]
@@ -1107,16 +1107,6 @@ dependencies = [
"libc",
]
[[package]]
name = "memory-stats"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -1184,15 +1174,6 @@ dependencies = [
"libc",
]
[[package]]
name = "ntapi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
dependencies = [
"winapi",
]
[[package]]
name = "num-format"
version = "0.4.4"
@@ -1250,7 +1231,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -1339,14 +1320,13 @@ dependencies = [
"derive_deref",
"inferno",
"itertools",
"memory-stats",
"ordered-float",
"rayon",
"reqwest",
"sanakirja",
"serde",
"serde_json",
"sysinfo",
"struct_iterable",
"toml",
"zstd",
]
@@ -1386,9 +1366,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.87"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a"
checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
dependencies = [
"unicode-ident",
]
@@ -1708,7 +1688,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -1802,6 +1782,28 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "struct_iterable"
version = "0.1.1"
dependencies = [
"struct_iterable_derive",
"struct_iterable_internal",
]
[[package]]
name = "struct_iterable_derive"
version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"struct_iterable_internal",
"syn 2.0.85",
]
[[package]]
name = "struct_iterable_internal"
version = "0.1.1"
[[package]]
name = "subtle"
version = "2.6.1"
@@ -1821,9 +1823,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.79"
version = "2.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
dependencies = [
"proc-macro2",
"quote",
@@ -1839,20 +1841,6 @@ dependencies = [
"futures-core",
]
[[package]]
name = "sysinfo"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
"windows",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
@@ -1904,7 +1892,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]
@@ -2179,7 +2167,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
"wasm-bindgen-shared",
]
@@ -2213,7 +2201,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -2256,16 +2244,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
"windows-core 0.57.0",
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.52.0"
@@ -2275,60 +2253,17 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "windows-core"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.1.2",
"windows-targets",
]
[[package]]
name = "windows-implement"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-interface"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result 0.2.0",
"windows-result",
"windows-strings",
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.2.0"
@@ -2344,7 +2279,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result 0.2.0",
"windows-result",
"windows-targets",
]
@@ -2457,7 +2392,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.79",
"syn 2.0.85",
]
[[package]]

5
parser/Cargo.toml
View File

@@ -19,13 +19,14 @@ ctrlc = { version = "3.4.5", features = ["termination"] }
derive_deref = "1.1.1"
inferno = "0.11.21"
itertools = "0.13.0"
memory-stats = "1.2.0"
ordered-float = "4.3.0"
rayon = "1.10.0"
reqwest = { version = "0.12.8", features = ["blocking", "json"] }
sanakirja = "1.4.2"
serde = { version = "1.0.210", features = ["derive"] }
serde_json = "1.0.128"
sysinfo = "0.32.0"
struct_iterable = { path = "../iterable" }
toml = "0.8.19"
zstd = "0.13.2"
# memory-stats = "1.2.0"
# sysinfo = "0.32.0"

624
parser/src/datasets/_traits/any_dataset.rs
View File

@@ -1,14 +1,21 @@
use itertools::Itertools;
use rayon::prelude::*;
use struct_iterable::Iterable;
use crate::{
datasets::ComputeData,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, Date, Height},
datasets::{
cohort_metadata::AddressCohortMetadataDataset, ComputeData, DateRecapDataset, RatioDataset,
SubDataset,
},
structs::{
AnyBiMap, AnyDateMap, AnyHeightMap, AnyMap, BiMap, Date, Height, MapKind, Timestamp, OHLC,
},
DateMap, HeightMap,
};
use super::MinInitialStates;
use super::{AnyDatasetGroup, MinInitialStates};
pub trait AnyDataset {
pub trait AnyDataset: Iterable {
fn get_min_initial_states(&self) -> &MinInitialStates;
fn needs_insert(&self, height: Height, date: Date) -> bool {
@@ -38,28 +45,613 @@ pub trait AnyDataset {
})
}
fn to_kind_bi_map_vec(&self, kind: MapKind) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
let mut v = vec![];
self.iter().for_each(|(_, any)| {
if let Some(map) = any.downcast_ref::<BiMap<u8>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<u16>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<u32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<u64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<usize>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<f32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<f64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<Date>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<Height>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<BiMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyBiMap + Send + Sync))
}
} else if let Some(dataset) = any.downcast_ref::<RatioDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_bi_map_vec(),
MapKind::Computed => dataset.to_computed_bi_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
} else if let Some(dataset) = any.downcast_ref::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_bi_map_vec(),
MapKind::Computed => dataset.to_computed_bi_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
} else if let Some(dataset) = any.downcast_ref::<SubDataset>() {
dataset.as_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_bi_map_vec(kind));
});
}
});
v
}
fn to_kind_mut_bi_map_vec(&mut self, kind: MapKind) -> Vec<&mut dyn AnyBiMap> {
let mut v = vec![];
self.iter_mut().for_each(|(_, any)| match any {
any if any.is::<BiMap<u8>>() => {
if let Some(map) = any.downcast_mut::<BiMap<u8>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<u16>>() => {
if let Some(map) = any.downcast_mut::<BiMap<u16>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<u32>>() => {
if let Some(map) = any.downcast_mut::<BiMap<u32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<u64>>() => {
if let Some(map) = any.downcast_mut::<BiMap<u64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<usize>>() => {
if let Some(map) = any.downcast_mut::<BiMap<usize>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<f32>>() => {
if let Some(map) = any.downcast_mut::<BiMap<f32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<f64>>() => {
if let Some(map) = any.downcast_mut::<BiMap<f64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<OHLC>>() => {
if let Some(map) = any.downcast_mut::<BiMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<Date>>() => {
if let Some(map) = any.downcast_mut::<BiMap<Date>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<Height>>() => {
if let Some(map) = any.downcast_mut::<BiMap<Height>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<BiMap<Timestamp>>() => {
if let Some(map) = any.downcast_mut::<BiMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyBiMap);
}
}
}
any if any.is::<RatioDataset>() => {
if let Some(dataset) = any.downcast_mut::<RatioDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_mut_bi_map_vec(),
MapKind::Computed => dataset.to_computed_mut_bi_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
}
any if any.is::<AddressCohortMetadataDataset>() => {
if let Some(dataset) = any.downcast_mut::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_mut_bi_map_vec(),
MapKind::Computed => dataset.to_computed_mut_bi_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
}
any if any.is::<SubDataset>() => {
if let Some(dataset) = any.downcast_mut::<SubDataset>() {
dataset.as_mut_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_mut_bi_map_vec(kind));
});
}
}
_ => {}
});
v
}
fn to_kind_date_map_vec(&self, kind: MapKind) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
let mut v = vec![];
self.iter().for_each(|(_, any)| {
if let Some(map) = any.downcast_ref::<DateMap<u8>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<u16>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<u32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<u64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<usize>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<f32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<f64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<Date>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<Height>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<DateMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
} else if let Some(dataset) = any.downcast_ref::<DateRecapDataset<u32>>() {
dataset.as_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
});
} else if let Some(dataset) = any.downcast_ref::<DateRecapDataset<u64>>() {
dataset.as_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
});
} else if let Some(dataset) = any.downcast_ref::<DateRecapDataset<f32>>() {
dataset.as_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &(dyn AnyDateMap + Send + Sync))
}
});
} else if let Some(dataset) = any.downcast_ref::<SubDataset>() {
dataset.as_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_date_map_vec(kind));
});
} else if let Some(dataset) = any.downcast_ref::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_date_map_vec(),
MapKind::Computed => dataset.to_computed_date_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
});
v
}
fn to_kind_mut_date_map_vec(&mut self, kind: MapKind) -> Vec<&mut dyn AnyDateMap> {
let mut v = vec![];
self.iter_mut().for_each(|(_, any)| match any {
any if any.is::<DateMap<u8>>() => {
if let Some(map) = any.downcast_mut::<DateMap<u8>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<u16>>() => {
if let Some(map) = any.downcast_mut::<DateMap<u16>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<u32>>() => {
if let Some(map) = any.downcast_mut::<DateMap<u32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<u64>>() => {
if let Some(map) = any.downcast_mut::<DateMap<u64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<usize>>() => {
if let Some(map) = any.downcast_mut::<DateMap<usize>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<f32>>() => {
if let Some(map) = any.downcast_mut::<DateMap<f32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<f64>>() => {
if let Some(map) = any.downcast_mut::<DateMap<f64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<OHLC>>() => {
if let Some(map) = any.downcast_mut::<DateMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<Date>>() => {
if let Some(map) = any.downcast_mut::<DateMap<Date>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<Height>>() => {
if let Some(map) = any.downcast_mut::<DateMap<Height>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateMap<Timestamp>>() => {
if let Some(map) = any.downcast_mut::<DateMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
}
}
any if any.is::<DateRecapDataset<u32>>() => {
if let Some(dataset) = any.downcast_mut::<DateRecapDataset<u32>>() {
dataset.as_mut_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
});
}
}
any if any.is::<DateRecapDataset<u64>>() => {
if let Some(dataset) = any.downcast_mut::<DateRecapDataset<u64>>() {
dataset.as_mut_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
});
}
}
any if any.is::<DateRecapDataset<f32>>() => {
if let Some(dataset) = any.downcast_mut::<DateRecapDataset<f32>>() {
dataset.as_mut_vec().into_iter().for_each(|map| {
if map.kind() == kind {
v.push(map as &mut dyn AnyDateMap);
}
});
}
}
any if any.is::<SubDataset>() => {
if let Some(dataset) = any.downcast_mut::<SubDataset>() {
dataset.as_mut_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_mut_date_map_vec(kind));
});
}
}
any if any.is::<AddressCohortMetadataDataset>() => {
if let Some(dataset) = any.downcast_mut::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_mut_date_map_vec(),
MapKind::Computed => dataset.to_computed_mut_date_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
}
_ => {}
});
v
}
fn to_kind_height_map_vec(&self, kind: MapKind) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
let mut v = vec![];
self.iter().for_each(|(_, any)| {
if let Some(map) = any.downcast_ref::<HeightMap<u8>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<u16>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<u32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<u64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<usize>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<f32>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<f64>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<Date>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<Height>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(map) = any.downcast_ref::<HeightMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &(dyn AnyHeightMap + Send + Sync))
}
} else if let Some(dataset) = any.downcast_ref::<SubDataset>() {
dataset.as_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_height_map_vec(kind));
});
} else if let Some(dataset) = any.downcast_ref::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_height_map_vec(),
MapKind::Computed => dataset.to_computed_height_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
});
v
}
fn to_kind_mut_height_map_vec(&mut self, kind: MapKind) -> Vec<&mut dyn AnyHeightMap> {
let mut v = vec![];
self.iter_mut().for_each(|(_, any)| match any {
any if any.is::<HeightMap<u8>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<u8>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<u16>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<u16>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<u32>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<u32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<u64>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<u64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<usize>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<usize>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<f32>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<f32>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<f64>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<f64>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<OHLC>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<OHLC>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<Date>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<Date>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<Height>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<Height>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<HeightMap<Timestamp>>() => {
if let Some(map) = any.downcast_mut::<HeightMap<Timestamp>>() {
if map.kind() == kind {
v.push(map as &mut dyn AnyHeightMap);
}
}
}
any if any.is::<SubDataset>() => {
if let Some(dataset) = any.downcast_mut::<SubDataset>() {
dataset.as_mut_vec().into_iter().for_each(|dataset| {
v.append(&mut dataset.to_kind_mut_height_map_vec(kind));
});
}
}
any if any.is::<AddressCohortMetadataDataset>() => {
if let Some(dataset) = any.downcast_mut::<AddressCohortMetadataDataset>() {
match kind {
MapKind::Inserted => dataset.to_inserted_mut_height_map_vec(),
MapKind::Computed => dataset.to_computed_mut_height_map_vec(),
}
.into_iter()
.for_each(|map| {
v.push(map);
});
}
}
_ => {}
});
v
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![]
self.to_kind_bi_map_vec(MapKind::Inserted)
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![]
self.to_kind_height_map_vec(MapKind::Inserted)
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![]
self.to_kind_date_map_vec(MapKind::Inserted)
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![]
self.to_kind_mut_bi_map_vec(MapKind::Inserted)
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![]
self.to_kind_mut_height_map_vec(MapKind::Inserted)
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![]
self.to_kind_mut_date_map_vec(MapKind::Inserted)
}
fn to_all_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
@@ -140,27 +732,27 @@ pub trait AnyDataset {
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![]
self.to_kind_bi_map_vec(MapKind::Computed)
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![]
self.to_kind_height_map_vec(MapKind::Computed)
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![]
self.to_kind_date_map_vec(MapKind::Computed)
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![]
self.to_kind_mut_bi_map_vec(MapKind::Computed)
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![]
self.to_kind_mut_height_map_vec(MapKind::Computed)
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![]
self.to_kind_mut_date_map_vec(MapKind::Computed)
}
fn to_all_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {

34
parser/src/datasets/address/all_metadata.rs
View File

@@ -1,19 +1,16 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct AllAddressesMetadataDataset {
min_initial_states: MinInitialStates,
// Inserted
created_addreses: BiMap<u32>,
empty_addresses: BiMap<u32>,
// Computed
new_addresses: BiMap<u32>,
}
@@ -24,9 +21,12 @@ impl AllAddressesMetadataDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
created_addreses: BiMap::new_bin(1, &f("created_addresses")),
empty_addresses: BiMap::new_bin(1, &f("empty_addresses")),
new_addresses: BiMap::new_bin(1, &f("new_addresses")),
// Inserted
created_addreses: BiMap::new_bin(1, MapKind::Inserted, &f("created_addresses")),
empty_addresses: BiMap::new_bin(1, MapKind::Inserted, &f("empty_addresses")),
// Computed
new_addresses: BiMap::new_bin(1, MapKind::Computed, &f("new_addresses")),
};
s.min_initial_states
@@ -71,20 +71,4 @@ impl AnyDataset for AllAddressesMetadataDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.created_addreses, &self.empty_addresses]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.created_addreses, &mut self.empty_addresses]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.new_addresses]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.new_addresses]
}
}

114
parser/src/datasets/address/cohort.rs
View File

@@ -1,24 +1,22 @@
use allocative::Allocative;
use itertools::Itertools;
use struct_iterable::Iterable;
use crate::{
datasets::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
states::{AddressCohortId, DurableStates},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
structs::{AddressSplit, BiMap, Config, Date, Height},
};
use super::cohort_metadata::MetadataDataset;
use super::cohort_metadata::AddressCohortMetadataDataset;
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct CohortDataset {
min_initial_states: MinInitialStates,
split: AddressSplit,
metadata: MetadataDataset,
metadata: AddressCohortMetadataDataset,
pub subs: SubDataset,
}
@@ -35,7 +33,7 @@ impl CohortDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
split,
metadata: MetadataDataset::import(parent_path, &name, config)?,
metadata: AddressCohortMetadataDataset::import(parent_path, &name, config)?,
subs: SubDataset::import(parent_path, &name, config)?,
};
@@ -202,20 +200,6 @@ impl CohortDataset {
// .insert(insert_data, &state.highly_liquid);
// }
fn as_vec(&self) -> Vec<&(dyn AnyDataset + Send + Sync)> {
vec![self.subs.as_vec(), vec![&self.metadata]]
.into_iter()
.flatten()
.collect_vec()
}
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyDataset> {
vec![self.subs.as_mut_vec(), vec![&mut self.metadata]]
.into_iter()
.flatten()
.collect_vec()
}
pub fn insert(&mut self, insert_data: &InsertData) {
if !insert_data.compute_addresses {
return;
@@ -389,90 +373,6 @@ impl CohortDataset {
}
impl AnyDataset for CohortDataset {
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_height_map_vec())
.collect_vec()
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_date_map_vec())
.collect_vec()
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_bi_map_vec())
.collect_vec()
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_height_map_vec())
.collect_vec()
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_date_map_vec())
.collect_vec()
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_bi_map_vec())
.collect_vec()
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_height_map_vec())
.collect_vec()
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_date_map_vec())
.collect_vec()
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
self.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_bi_map_vec())
.collect_vec()
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_height_map_vec())
.collect_vec()
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_date_map_vec())
.collect_vec()
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
self.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_bi_map_vec())
.collect_vec()
}
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}

23
parser/src/datasets/address/cohort_metadata.rs
View File

@@ -1,15 +1,15 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
};
#[derive(Default, Allocative)]
pub struct MetadataDataset {
#[derive(Allocative, Iterable)]
pub struct AddressCohortMetadataDataset {
min_initial_states: MinInitialStates,
// Inserted
address_count: BiMap<f64>,
// pub output: OutputSubDataset,
// Sending addresses
@@ -17,7 +17,7 @@ pub struct MetadataDataset {
// Active addresses (Unique(Sending + Receiving))
}
impl MetadataDataset {
impl AddressCohortMetadataDataset {
pub fn import(
parent_path: &str,
name: &Option<String>,
@@ -34,7 +34,8 @@ impl MetadataDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
address_count: BiMap::new_bin(1, &f("address_count")),
// Inserted
address_count: BiMap::new_bin(1, MapKind::Inserted, &f("address_count")),
// output: OutputSubDataset::import(parent_path)?,
};
@@ -62,16 +63,8 @@ impl MetadataDataset {
}
}
impl AnyDataset for MetadataDataset {
impl AnyDataset for AddressCohortMetadataDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.address_count]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.address_count]
}
}

8
parser/src/datasets/address/mod.rs
View File

@@ -1,6 +1,6 @@
mod all_metadata;
mod cohort;
mod cohort_metadata;
pub mod cohort_metadata;
use allocative::Allocative;
use itertools::Itertools;
@@ -27,7 +27,7 @@ pub struct AddressDatasets {
impl AddressDatasets {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByAddressCohort::<CohortDataset>::default();
let mut cohorts = SplitByAddressCohort::<Option<CohortDataset>>::default();
cohorts
.as_vec()
@@ -36,7 +36,7 @@ impl AddressDatasets {
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
*cohorts.get_mut_from_id(&id) = dataset?;
cohorts.get_mut_from_id(&id).replace(dataset?);
Ok(())
})?;
@@ -45,7 +45,7 @@ impl AddressDatasets {
metadata: AllAddressesMetadataDataset::import(parent_path, config)?,
cohorts,
cohorts: cohorts.unwrap(),
};
s.min_initial_states

21
parser/src/datasets/block_metadata.rs
View File

@@ -1,17 +1,16 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::AnyDataset,
structs::{AnyHeightMap, Config, Date, HeightMap, Timestamp},
structs::{Config, Date, HeightMap, MapKind, Timestamp},
};
use super::{InsertData, MinInitialStates};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct BlockMetadataDataset {
min_initial_states: MinInitialStates,
// Inserted
pub date: HeightMap<Date>,
pub timestamp: HeightMap<Timestamp>,
}
@@ -22,9 +21,9 @@ impl BlockMetadataDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
date: HeightMap::new_bin(1, &f("date")),
timestamp: HeightMap::new_bin(1, &f("timestamp")),
// Inserted
date: HeightMap::new_bin(1, MapKind::Inserted, &f("date")),
timestamp: HeightMap::new_bin(1, MapKind::Inserted, &f("timestamp")),
};
s.min_initial_states
@@ -49,12 +48,4 @@ impl AnyDataset for BlockMetadataDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![&self.date, &self.timestamp]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![&mut self.date, &mut self.timestamp]
}
}

32
parser/src/datasets/coindays.rs
View File

@@ -1,18 +1,17 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, AnyHeightMap, Config},
structs::{Config, MapKind},
DateMap, HeightMap,
};
use super::{InsertData, MinInitialStates};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct CoindaysDataset {
min_initial_states: MinInitialStates,
// Inserted
pub coindays_destroyed: HeightMap<f32>,
pub coindays_destroyed_1d_sum: DateMap<f32>,
}
@@ -24,8 +23,13 @@ impl CoindaysDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
coindays_destroyed: HeightMap::new_bin(1, &f("coindays_destroyed")),
coindays_destroyed_1d_sum: DateMap::new_bin(1, &f("coindays_destroyed_1d_sum")),
// Inserted
coindays_destroyed: HeightMap::new_bin(1, MapKind::Inserted, &f("coindays_destroyed")),
coindays_destroyed_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("coindays_destroyed_1d_sum"),
),
};
s.min_initial_states
@@ -56,22 +60,6 @@ impl CoindaysDataset {
}
impl AnyDataset for CoindaysDataset {
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![&self.coindays_destroyed]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![&self.coindays_destroyed_1d_sum]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![&mut self.coindays_destroyed]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![&mut self.coindays_destroyed_1d_sum]
}
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}

345
parser/src/datasets/cointime.rs
View File

@@ -1,14 +1,15 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height},
structs::{BiMap, Config, DateMap, Height, MapKind},
utils::{ONE_DAY_IN_DAYS, ONE_YEAR_IN_DAYS, THREE_MONTHS_IN_DAYS, TWO_WEEK_IN_DAYS},
HeightMap,
};
use super::{AnyDataset, ComputeData, InsertData, MinInitialStates, RatioDataset};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct CointimeDataset {
min_initial_states: MinInitialStates,
@@ -77,83 +78,187 @@ impl CointimeDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
active_cap: BiMap::new_bin(1, &f("active_cap")),
active_price: BiMap::new_bin(1, &f("active_price")),
// Inserted
coinblocks_destroyed: HeightMap::new_bin(
1,
MapKind::Inserted,
&f("coinblocks_destroyed"),
),
coinblocks_destroyed_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("coinblocks_destroyed_1d_sum"),
),
// Computed
active_cap: BiMap::new_bin(1, MapKind::Computed, &f("active_cap")),
active_price: BiMap::new_bin(1, MapKind::Computed, &f("active_price")),
active_price_ratio: RatioDataset::import(parent_path, "active_price", config)?,
active_supply: BiMap::new_bin(1, &f("active_supply")),
active_supply_3m_net_change: BiMap::new_bin(1, &f("active_supply_3m_net_change")),
active_supply_net_change: BiMap::new_bin(1, &f("active_supply_net_change")),
activity_to_vaultedness_ratio: BiMap::new_bin(2, &f("activity_to_vaultedness_ratio")),
coinblocks_created: HeightMap::new_bin(1, &f("coinblocks_created")),
coinblocks_created_1d_sum: DateMap::new_bin(1, &f("coinblocks_created_1d_sum")),
coinblocks_destroyed: HeightMap::new_bin(1, &f("coinblocks_destroyed")),
coinblocks_destroyed_1d_sum: DateMap::new_bin(1, &f("coinblocks_destroyed_1d_sum")),
coinblocks_stored: HeightMap::new_bin(1, &f("coinblocks_stored")),
coinblocks_stored_1d_sum: DateMap::new_bin(1, &f("coinblocks_stored_1d_sum")),
cointime_adjusted_velocity: DateMap::new_bin(1, &f("cointime_adjusted_velocity")),
active_supply: BiMap::new_bin(1, MapKind::Computed, &f("active_supply")),
active_supply_3m_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("active_supply_3m_net_change"),
),
active_supply_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("active_supply_net_change"),
),
activity_to_vaultedness_ratio: BiMap::new_bin(
2,
MapKind::Computed,
&f("activity_to_vaultedness_ratio"),
),
coinblocks_created: HeightMap::new_bin(1, MapKind::Computed, &f("coinblocks_created")),
coinblocks_created_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("coinblocks_created_1d_sum"),
),
coinblocks_stored: HeightMap::new_bin(1, MapKind::Computed, &f("coinblocks_stored")),
coinblocks_stored_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("coinblocks_stored_1d_sum"),
),
cointime_adjusted_velocity: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_adjusted_velocity"),
),
cointime_adjusted_inflation_rate: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_adjusted_inflation_rate"),
),
cointime_adjusted_yearly_inflation_rate: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_adjusted_yearly_inflation_rate"),
),
cointime_cap: BiMap::new_bin(1, &f("cointime_cap")),
cointime_price: BiMap::new_bin(1, &f("cointime_price")),
cointime_cap: BiMap::new_bin(1, MapKind::Computed, &f("cointime_cap")),
cointime_price: BiMap::new_bin(1, MapKind::Computed, &f("cointime_price")),
cointime_price_ratio: RatioDataset::import(parent_path, "cointime_price", config)?,
cointime_value_created: HeightMap::new_bin(1, &f("cointime_value_created")),
cointime_value_created_1d_sum: DateMap::new_bin(1, &f("cointime_value_created_1d_sum")),
cointime_value_destroyed: HeightMap::new_bin(1, &f("cointime_value_destroyed")),
cointime_value_created: HeightMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_created"),
),
cointime_value_created_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_created_1d_sum"),
),
cointime_value_destroyed: HeightMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_destroyed"),
),
cointime_value_destroyed_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_destroyed_1d_sum"),
),
cointime_value_stored: HeightMap::new_bin(1, &f("cointime_value_stored")),
cointime_value_stored_1d_sum: DateMap::new_bin(1, &f("cointime_value_stored_1d_sum")),
concurrent_liveliness: DateMap::new_bin(1, &f("concurrent_liveliness")),
cointime_value_stored: HeightMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_stored"),
),
cointime_value_stored_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("cointime_value_stored_1d_sum"),
),
concurrent_liveliness: DateMap::new_bin(
1,
MapKind::Computed,
&f("concurrent_liveliness"),
),
concurrent_liveliness_2w_median: DateMap::new_bin(
2,
MapKind::Computed,
&f("concurrent_liveliness_2w_median"),
),
cumulative_coinblocks_created: BiMap::new_bin(1, &f("cumulative_coinblocks_created")),
cumulative_coinblocks_created: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_coinblocks_created"),
),
cumulative_coinblocks_destroyed: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_coinblocks_destroyed"),
),
cumulative_coinblocks_stored: BiMap::new_bin(1, &f("cumulative_coinblocks_stored")),
investor_cap: BiMap::new_bin(1, &f("investor_cap")),
investorness: BiMap::new_bin(1, &f("investorness")),
liveliness: BiMap::new_bin(1, &f("liveliness")),
liveliness_net_change: BiMap::new_bin(1, &f("liveliness_net_change")),
cumulative_coinblocks_stored: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_coinblocks_stored"),
),
investor_cap: BiMap::new_bin(1, MapKind::Computed, &f("investor_cap")),
investorness: BiMap::new_bin(1, MapKind::Computed, &f("investorness")),
liveliness: BiMap::new_bin(1, MapKind::Computed, &f("liveliness")),
liveliness_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("liveliness_net_change"),
),
liveliness_net_change_2w_median: BiMap::new_bin(
3,
MapKind::Computed,
&f("liveliness_net_change_2w_median"),
),
producerness: BiMap::new_bin(1, &f("producerness")),
thermo_cap: BiMap::new_bin(1, &f("thermo_cap")),
producerness: BiMap::new_bin(1, MapKind::Computed, &f("producerness")),
thermo_cap: BiMap::new_bin(1, MapKind::Computed, &f("thermo_cap")),
thermo_cap_to_investor_cap_ratio: BiMap::new_bin(
2,
MapKind::Computed,
&f("thermo_cap_to_investor_cap_ratio"),
),
total_cointime_value_created: BiMap::new_bin(1, &f("total_cointime_value_created")),
total_cointime_value_destroyed: BiMap::new_bin(1, &f("total_cointime_value_destroyed")),
total_cointime_value_stored: BiMap::new_bin(1, &f("total_cointime_value_stored")),
true_market_deviation: BiMap::new_bin(1, &f("true_market_deviation")),
true_market_mean: BiMap::new_bin(1, &f("true_market_mean")),
total_cointime_value_created: BiMap::new_bin(
1,
MapKind::Computed,
&f("total_cointime_value_created"),
),
total_cointime_value_destroyed: BiMap::new_bin(
1,
MapKind::Computed,
&f("total_cointime_value_destroyed"),
),
total_cointime_value_stored: BiMap::new_bin(
1,
MapKind::Computed,
&f("total_cointime_value_stored"),
),
true_market_deviation: BiMap::new_bin(
1,
MapKind::Computed,
&f("true_market_deviation"),
),
true_market_mean: BiMap::new_bin(1, MapKind::Computed, &f("true_market_mean")),
true_market_mean_ratio: RatioDataset::import(parent_path, "true_market_mean", config)?,
true_market_net_unrealized_profit_and_loss: BiMap::new_bin(
1,
MapKind::Computed,
&f("true_market_net_unrealized_profit_and_loss"),
),
vaulted_cap: BiMap::new_bin(1, &f("vaulted_cap")),
vaulted_price: BiMap::new_bin(1, &f("vaulted_price")),
vaulted_cap: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_cap")),
vaulted_price: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_price")),
vaulted_price_ratio: RatioDataset::import(parent_path, "vaulted_price", config)?,
vaulted_supply: BiMap::new_bin(1, &f("vaulted_supply")),
vaulted_supply_3m_net_change: BiMap::new_bin(1, &f("vaulted_supply_3m_net_change")),
vaulted_supply_net_change: BiMap::new_bin(1, &f("vaulted_supply_net_change")),
vaultedness: BiMap::new_bin(1, &f("vaultedness")),
vaulting_rate: BiMap::new_bin(1, &f("vaulting_rate")),
vaulted_supply: BiMap::new_bin(1, MapKind::Computed, &f("vaulted_supply")),
vaulted_supply_3m_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("vaulted_supply_3m_net_change"),
),
vaulted_supply_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("vaulted_supply_net_change"),
),
vaultedness: BiMap::new_bin(1, MapKind::Computed, &f("vaultedness")),
vaulting_rate: BiMap::new_bin(1, MapKind::Computed, &f("vaulting_rate")),
};
s.min_initial_states
@@ -544,160 +649,6 @@ impl CointimeDataset {
}
impl AnyDataset for CointimeDataset {
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![&self.coinblocks_destroyed]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![&mut self.coinblocks_destroyed]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![&self.coinblocks_destroyed_1d_sum]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![&mut self.coinblocks_destroyed_1d_sum]
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.coinblocks_created,
&self.coinblocks_stored,
&self.cointime_value_created,
&self.cointime_value_destroyed,
&self.cointime_value_stored,
]
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.coinblocks_created,
&mut self.coinblocks_stored,
&mut self.cointime_value_created,
&mut self.cointime_value_destroyed,
&mut self.cointime_value_stored,
]
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.coinblocks_created_1d_sum,
&self.coinblocks_stored_1d_sum,
&self.concurrent_liveliness,
&self.concurrent_liveliness_2w_median,
&self.cointime_adjusted_velocity,
&self.cointime_value_created_1d_sum,
&self.cointime_value_destroyed_1d_sum,
&self.cointime_value_stored_1d_sum,
&self.cointime_adjusted_inflation_rate,
&self.cointime_adjusted_yearly_inflation_rate,
]
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.coinblocks_created_1d_sum,
&mut self.coinblocks_stored_1d_sum,
&mut self.concurrent_liveliness,
&mut self.concurrent_liveliness_2w_median,
&mut self.cointime_adjusted_velocity,
&mut self.cointime_value_created_1d_sum,
&mut self.cointime_value_destroyed_1d_sum,
&mut self.cointime_value_stored_1d_sum,
&mut self.cointime_adjusted_inflation_rate,
&mut self.cointime_adjusted_yearly_inflation_rate,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
let mut v = vec![
&self.active_cap as &(dyn AnyBiMap + Send + Sync),
&self.active_price,
&self.active_supply,
&self.active_supply_3m_net_change,
&self.active_supply_net_change,
&self.activity_to_vaultedness_ratio,
&self.cointime_cap,
&self.cointime_price,
&self.cumulative_coinblocks_created,
&self.cumulative_coinblocks_destroyed,
&self.cumulative_coinblocks_stored,
&self.investor_cap,
&self.investorness,
&self.liveliness,
&self.liveliness_net_change,
&self.liveliness_net_change_2w_median,
&self.producerness,
&self.thermo_cap,
&self.thermo_cap_to_investor_cap_ratio,
&self.total_cointime_value_created,
&self.total_cointime_value_destroyed,
&self.total_cointime_value_stored,
&self.true_market_deviation,
&self.true_market_mean,
&self.true_market_net_unrealized_profit_and_loss,
&self.vaulted_cap,
&self.vaulted_price,
&self.vaulted_supply,
&self.vaulted_supply_net_change,
&self.vaulted_supply_3m_net_change,
&self.vaultedness,
&self.vaulting_rate,
];
v.append(&mut self.active_price_ratio.to_computed_bi_map_vec());
v.append(&mut self.cointime_price_ratio.to_computed_bi_map_vec());
v.append(&mut self.true_market_mean_ratio.to_computed_bi_map_vec());
v.append(&mut self.vaulted_price_ratio.to_computed_bi_map_vec());
v
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
let mut v = vec![
&mut self.active_cap as &mut dyn AnyBiMap,
&mut self.active_price,
&mut self.active_supply,
&mut self.active_supply_3m_net_change,
&mut self.active_supply_net_change,
&mut self.activity_to_vaultedness_ratio,
&mut self.cointime_cap,
&mut self.cointime_price,
&mut self.cumulative_coinblocks_created,
&mut self.cumulative_coinblocks_destroyed,
&mut self.cumulative_coinblocks_stored,
&mut self.investor_cap,
&mut self.investorness,
&mut self.liveliness,
&mut self.liveliness_net_change,
&mut self.liveliness_net_change_2w_median,
&mut self.producerness,
&mut self.thermo_cap,
&mut self.thermo_cap_to_investor_cap_ratio,
&mut self.total_cointime_value_created,
&mut self.total_cointime_value_destroyed,
&mut self.total_cointime_value_stored,
&mut self.true_market_deviation,
&mut self.true_market_mean,
&mut self.true_market_net_unrealized_profit_and_loss,
&mut self.vaulted_cap,
&mut self.vaulted_price,
&mut self.vaulted_supply,
&mut self.vaulted_supply_net_change,
&mut self.vaulted_supply_3m_net_change,
&mut self.vaultedness,
&mut self.vaulting_rate,
];
v.append(&mut self.active_price_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.cointime_price_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.true_market_mean_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.vaulted_price_ratio.to_computed_mut_bi_map_vec());
v
}
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}

23
parser/src/datasets/constant.rs
View File

@@ -1,14 +1,14 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::structs::{AnyBiMap, BiMap, Config};
use crate::structs::{BiMap, Config, MapKind};
use super::{AnyDataset, ComputeData, MinInitialStates};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct ConstantDataset {
min_initial_states: MinInitialStates,
// Computed
pub _0: BiMap<u16>,
pub _1: BiMap<u16>,
pub _50: BiMap<u16>,
@@ -22,10 +22,11 @@ impl ConstantDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
_0: BiMap::new_bin(1, &f("0")),
_1: BiMap::new_bin(1, &f("1")),
_50: BiMap::new_bin(1, &f("50")),
_100: BiMap::new_bin(1, &f("100")),
// Computed
_0: BiMap::new_bin(1, MapKind::Computed, &f("0")),
_1: BiMap::new_bin(1, MapKind::Computed, &f("1")),
_50: BiMap::new_bin(1, MapKind::Computed, &f("50")),
_100: BiMap::new_bin(1, MapKind::Computed, &f("100")),
};
s.min_initial_states
@@ -46,12 +47,4 @@ impl AnyDataset for ConstantDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self._0, &self._1, &self._50, &self._100]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self._0, &mut self._1, &mut self._50, &mut self._100]
}
}

19
parser/src/datasets/date_metadata.rs
View File

@@ -1,17 +1,17 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::AnyDataset,
structs::{AnyDateMap, Config, DateMap, Height},
structs::{Config, DateMap, Height, MapKind},
};
use super::{InsertData, MinInitialStates};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct DateMetadataDataset {
min_initial_states: MinInitialStates,
// Inserted
pub first_height: DateMap<Height>,
pub last_height: DateMap<Height>,
}
@@ -23,8 +23,9 @@ impl DateMetadataDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
first_height: DateMap::new_bin(1, &f("first_height")),
last_height: DateMap::new_bin(1, &f("last_height")),
// Inserted
first_height: DateMap::new_bin(1, MapKind::Inserted, &f("first_height")),
last_height: DateMap::new_bin(1, MapKind::Inserted, &f("last_height")),
};
s.min_initial_states
@@ -49,14 +50,6 @@ impl DateMetadataDataset {
}
impl AnyDataset for DateMetadataDataset {
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![&self.first_height, &self.last_height]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![&mut self.first_height, &mut self.last_height]
}
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}

459
parser/src/datasets/mining.rs
View File

@@ -1,13 +1,11 @@
use allocative::Allocative;
use itertools::Itertools;
use ordered_float::OrderedFloat;
use struct_iterable::Iterable;
use crate::{
datasets::AnyDataset,
structs::{
date_map_vec_to_any_date_map_vec, date_map_vec_to_mut_any_date_map_vec, Amount, AnyBiMap,
AnyDateMap, AnyHeightMap, BiMap, Config, DateMap, Height, HeightMap, MapKey,
},
structs::{Amount, BiMap, Config, DateMap, Height, HeightMap, MapKey, MapKind},
utils::{
BYTES_IN_MB, ONE_DAY_IN_DAYS, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS,
TARGET_BLOCKS_PER_DAY,
@@ -18,7 +16,7 @@ use super::{
ComputeData, DateRecapDataset, InsertData, MinInitialStates, RecapDataset, RecapOptions,
};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct MiningDataset {
min_initial_states: MinInitialStates,
@@ -131,68 +129,166 @@ impl MiningDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
total_blocks_mined: DateMap::new_bin(1, &f("total_blocks_mined")),
blocks_mined: DateMap::new_bin(1, &f("blocks_mined")),
coinbase: HeightMap::new_bin(1, &f("coinbase")),
coinbase_1d_sum: DateMap::new_bin(1, &f("coinbase_1d_sum")),
coinbase_in_dollars: HeightMap::new_bin(1, &f("coinbase_in_dollars")),
coinbase_in_dollars_1d_sum: DateMap::new_bin(1, &f("coinbase_in_dollars_1d_sum")),
coinbase_1y_sum: DateMap::new_bin(1, &f("coinbase_1y_sum")),
coinbase_in_dollars_1y_sum: DateMap::new_bin(1, &f("coinbase_in_dollars_1y_sum")),
// ---
// Inserted
// ---
total_blocks_mined: DateMap::new_bin(1, MapKind::Inserted, &f("total_blocks_mined")),
blocks_mined: DateMap::new_bin(1, MapKind::Inserted, &f("blocks_mined")),
coinbase: HeightMap::new_bin(1, MapKind::Inserted, &f("coinbase")),
coinbase_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("coinbase_1d_sum")),
coinbase_in_dollars: HeightMap::new_bin(
1,
MapKind::Inserted,
&f("coinbase_in_dollars"),
),
coinbase_in_dollars_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("coinbase_in_dollars_1d_sum"),
),
fees: HeightMap::new_bin(1, MapKind::Inserted, &f("fees")),
fees_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("fees_1d_sum")),
fees_in_dollars: HeightMap::new_bin(1, MapKind::Inserted, &f("fees_in_dollars")),
fees_in_dollars_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("fees_in_dollars_1d_sum"),
),
subsidy: HeightMap::new_bin(1, MapKind::Inserted, &f("subsidy")),
subsidy_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("subsidy_1d_sum")),
subsidy_in_dollars: HeightMap::new_bin(1, MapKind::Inserted, &f("subsidy_in_dollars")),
subsidy_in_dollars_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("subsidy_in_dollars_1d_sum"),
),
last_subsidy: DateMap::new_bin(1, MapKind::Inserted, &f("last_subsidy")),
last_subsidy_in_dollars: DateMap::new_bin(
1,
MapKind::Inserted,
&f("last_subsidy_in_dollars"),
),
last_coinbase: DateMap::new_bin(1, MapKind::Inserted, &f("last_coinbase")),
last_coinbase_in_dollars: DateMap::new_bin(
1,
MapKind::Inserted,
&f("last_coinbase_in_dollars"),
),
last_fees: DateMap::new_bin(1, MapKind::Inserted, &f("last_fees")),
last_fees_in_dollars: DateMap::new_bin(
1,
MapKind::Inserted,
&f("last_fees_in_dollars"),
),
difficulty: BiMap::new_bin(1, MapKind::Inserted, &f("difficulty")),
block_size: HeightMap::new_bin(1, MapKind::Inserted, &f("block_size")),
block_weight: HeightMap::new_bin(1, MapKind::Inserted, &f("block_weight")),
block_vbytes: HeightMap::new_bin(1, MapKind::Inserted, &f("block_vbytes")),
block_interval: HeightMap::new_bin(2, MapKind::Inserted, &f("block_interval")),
// ---
// Computed
// ---
coinbase_1y_sum: DateMap::new_bin(1, MapKind::Computed, &f("coinbase_1y_sum")),
coinbase_in_dollars_1y_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("coinbase_in_dollars_1y_sum"),
),
coinbase_in_dollars_1d_sum_1y_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("coinbase_in_dollars_1d_sum_1y_sma"),
),
cumulative_coinbase: BiMap::new_bin(1, &f("cumulative_coinbase")),
cumulative_coinbase_in_dollars: BiMap::new_bin(1, &f("cumulative_coinbase_in_dollars")),
fees: HeightMap::new_bin(1, &f("fees")),
fees_1d_sum: DateMap::new_bin(1, &f("fees_1d_sum")),
fees_in_dollars: HeightMap::new_bin(1, &f("fees_in_dollars")),
fees_in_dollars_1d_sum: DateMap::new_bin(1, &f("fees_in_dollars_1d_sum")),
fees_1y_sum: DateMap::new_bin(1, &f("fees_1y_sum")),
fees_in_dollars_1y_sum: DateMap::new_bin(1, &f("fees_in_dollars_1y_sum")),
cumulative_fees: BiMap::new_bin(1, &f("cumulative_fees")),
cumulative_fees_in_dollars: BiMap::new_bin(1, &f("cumulative_fees_in_dollars")),
subsidy: HeightMap::new_bin(1, &f("subsidy")),
subsidy_1d_sum: DateMap::new_bin(1, &f("subsidy_1d_sum")),
subsidy_in_dollars: HeightMap::new_bin(1, &f("subsidy_in_dollars")),
subsidy_in_dollars_1d_sum: DateMap::new_bin(1, &f("subsidy_in_dollars_1d_sum")),
subsidy_1y_sum: DateMap::new_bin(1, &f("subsidy_1y_sum")),
subsidy_in_dollars_1y_sum: DateMap::new_bin(1, &f("subsidy_in_dollars_1y_sum")),
cumulative_subsidy: BiMap::new_bin(1, &f("cumulative_subsidy")),
cumulative_subsidy_in_dollars: BiMap::new_bin(1, &f("cumulative_subsidy_in_dollars")),
cumulative_coinbase: BiMap::new_bin(1, MapKind::Computed, &f("cumulative_coinbase")),
cumulative_coinbase_in_dollars: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_coinbase_in_dollars"),
),
subsidy_to_coinbase_ratio: HeightMap::new_bin(1, &f("subsidy_to_coinbase_ratio")),
subsidy_to_coinbase_1d_ratio: DateMap::new_bin(1, &f("subsidy_to_coinbase_1d_ratio")),
fees_to_coinbase_ratio: HeightMap::new_bin(1, &f("fees_to_coinbase_ratio")),
fees_to_coinbase_1d_ratio: DateMap::new_bin(1, &f("fees_to_coinbase_1d_ratio")),
fees_1y_sum: DateMap::new_bin(1, MapKind::Computed, &f("fees_1y_sum")),
fees_in_dollars_1y_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("fees_in_dollars_1y_sum"),
),
cumulative_fees: BiMap::new_bin(1, MapKind::Computed, &f("cumulative_fees")),
cumulative_fees_in_dollars: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_fees_in_dollars"),
),
subsidy_1y_sum: DateMap::new_bin(1, MapKind::Computed, &f("subsidy_1y_sum")),
subsidy_in_dollars_1y_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("subsidy_in_dollars_1y_sum"),
),
cumulative_subsidy: BiMap::new_bin(1, MapKind::Computed, &f("cumulative_subsidy")),
cumulative_subsidy_in_dollars: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_subsidy_in_dollars"),
),
annualized_issuance: DateMap::new_bin(1, &f("annualized_issuance")),
inflation_rate: DateMap::new_bin(2, &f("inflation_rate")),
yearly_inflation_rate: DateMap::new_bin(1, &f("yearly_inflation_rate")),
last_subsidy: DateMap::new_bin(1, &f("last_subsidy")),
last_subsidy_in_dollars: DateMap::new_bin(1, &f("last_subsidy_in_dollars")),
last_coinbase: DateMap::new_bin(1, &f("last_coinbase")),
last_coinbase_in_dollars: DateMap::new_bin(1, &f("last_coinbase_in_dollars")),
last_fees: DateMap::new_bin(1, &f("last_fees")),
last_fees_in_dollars: DateMap::new_bin(1, &f("last_fees_in_dollars")),
blocks_mined_1d_target: DateMap::new_bin(1, &f("blocks_mined_1d_target")),
blocks_mined_1w_sma: DateMap::new_bin(1, &f("blocks_mined_1w_sma")),
blocks_mined_1m_sma: DateMap::new_bin(1, &f("blocks_mined_1m_sma")),
blocks_mined_1w_sum: DateMap::new_bin(1, &f("blocks_mined_1w_sum")),
blocks_mined_1m_sum: DateMap::new_bin(1, &f("blocks_mined_1m_sum")),
blocks_mined_1y_sum: DateMap::new_bin(1, &f("blocks_mined_1y_sum")),
blocks_mined_1w_target: DateMap::new_bin(1, &f("blocks_mined_1w_target")),
blocks_mined_1m_target: DateMap::new_bin(1, &f("blocks_mined_1m_target")),
blocks_mined_1y_target: DateMap::new_bin(1, &f("blocks_mined_1y_target")),
difficulty: BiMap::new_bin(1, &f("difficulty")),
difficulty_adjustment: DateMap::new_bin(1, &f("difficulty_adjustment")),
block_size: HeightMap::new_bin(1, &f("block_size")),
subsidy_to_coinbase_ratio: HeightMap::new_bin(
1,
MapKind::Computed,
&f("subsidy_to_coinbase_ratio"),
),
subsidy_to_coinbase_1d_ratio: DateMap::new_bin(
1,
MapKind::Computed,
&f("subsidy_to_coinbase_1d_ratio"),
),
fees_to_coinbase_ratio: HeightMap::new_bin(
1,
MapKind::Computed,
&f("fees_to_coinbase_ratio"),
),
fees_to_coinbase_1d_ratio: DateMap::new_bin(
1,
MapKind::Computed,
&f("fees_to_coinbase_1d_ratio"),
),
annualized_issuance: DateMap::new_bin(1, MapKind::Computed, &f("annualized_issuance")),
inflation_rate: DateMap::new_bin(2, MapKind::Computed, &f("inflation_rate")),
yearly_inflation_rate: DateMap::new_bin(
1,
MapKind::Computed,
&f("yearly_inflation_rate"),
),
blocks_mined_1d_target: DateMap::new_bin(
1,
MapKind::Computed,
&f("blocks_mined_1d_target"),
),
blocks_mined_1w_sma: DateMap::new_bin(1, MapKind::Computed, &f("blocks_mined_1w_sma")),
blocks_mined_1m_sma: DateMap::new_bin(1, MapKind::Computed, &f("blocks_mined_1m_sma")),
blocks_mined_1w_sum: DateMap::new_bin(1, MapKind::Computed, &f("blocks_mined_1w_sum")),
blocks_mined_1m_sum: DateMap::new_bin(1, MapKind::Computed, &f("blocks_mined_1m_sum")),
blocks_mined_1y_sum: DateMap::new_bin(1, MapKind::Computed, &f("blocks_mined_1y_sum")),
blocks_mined_1w_target: DateMap::new_bin(
1,
MapKind::Computed,
&f("blocks_mined_1w_target"),
),
blocks_mined_1m_target: DateMap::new_bin(
1,
MapKind::Computed,
&f("blocks_mined_1m_target"),
),
blocks_mined_1y_target: DateMap::new_bin(
1,
MapKind::Computed,
&f("blocks_mined_1y_target"),
),
difficulty_adjustment: DateMap::new_bin(
1,
MapKind::Computed,
&f("difficulty_adjustment"),
),
block_size_recap: RecapDataset::import(
&f("block_size_1d"),
RecapOptions::default()
@@ -206,12 +302,16 @@ impl MiningDataset {
.add_10p()
.add_min(),
)?,
cumulative_block_size: BiMap::new_bin(1, &f("cumulative_block_size")),
cumulative_block_size: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_block_size"),
),
cumulative_block_size_gigabytes: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_block_size_gigabytes"),
),
block_weight: HeightMap::new_bin(1, &f("block_weight")),
block_weight_recap: RecapDataset::import(
&f("block_weight_1d"),
RecapOptions::default()
@@ -224,7 +324,6 @@ impl MiningDataset {
.add_10p()
.add_min(),
)?,
block_vbytes: HeightMap::new_bin(1, &f("block_vbytes")),
block_vbytes_recap: RecapDataset::import(
&f("block_vbytes_1d"),
RecapOptions::default()
@@ -237,8 +336,6 @@ impl MiningDataset {
.add_10p()
.add_min(),
)?,
// block_vbytes_1d_sma: HeightMap::new_bin(1, &f("block_vbytes")),
block_interval: HeightMap::new_bin(2, &f("block_interval")),
block_interval_recap: RecapDataset::import(
&f("block_interval_1d"),
RecapOptions::default()
@@ -251,14 +348,14 @@ impl MiningDataset {
.add_10p()
.add_min(),
)?,
hash_rate: DateMap::new_bin(1, &f("hash_rate")),
hash_rate_1w_sma: DateMap::new_bin(1, &f("hash_rate_1w_sma")),
hash_rate_1m_sma: DateMap::new_bin(1, &f("hash_rate_1m_sma")),
hash_rate_2m_sma: DateMap::new_bin(1, &f("hash_rate_2m_sma")),
hash_price: DateMap::new_bin(1, &f("hash_price")),
hash_price_min: DateMap::new_bin(1, &f("hash_price_min")),
hash_price_rebound: DateMap::new_bin(1, &f("hash_price_rebound")),
puell_multiple: DateMap::new_bin(1, &f("puell_multiple")),
hash_rate: DateMap::new_bin(1, MapKind::Computed, &f("hash_rate")),
hash_rate_1w_sma: DateMap::new_bin(1, MapKind::Computed, &f("hash_rate_1w_sma")),
hash_rate_1m_sma: DateMap::new_bin(1, MapKind::Computed, &f("hash_rate_1m_sma")),
hash_rate_2m_sma: DateMap::new_bin(1, MapKind::Computed, &f("hash_rate_2m_sma")),
hash_price: DateMap::new_bin(1, MapKind::Computed, &f("hash_price")),
hash_price_min: DateMap::new_bin(1, MapKind::Computed, &f("hash_price_min")),
hash_price_rebound: DateMap::new_bin(1, MapKind::Computed, &f("hash_price_rebound")),
puell_multiple: DateMap::new_bin(1, MapKind::Computed, &f("puell_multiple")),
};
s.min_initial_states
@@ -660,218 +757,4 @@ impl AnyDataset for MiningDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.difficulty]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.difficulty]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.coinbase_1d_sum,
&self.coinbase_in_dollars_1d_sum,
&self.fees_1d_sum,
&self.fees_in_dollars_1d_sum,
&self.subsidy_1d_sum,
&self.subsidy_in_dollars_1d_sum,
&self.total_blocks_mined,
&self.blocks_mined,
&self.last_subsidy,
&self.last_subsidy_in_dollars,
&self.last_coinbase,
&self.last_coinbase_in_dollars,
&self.last_fees,
&self.last_fees_in_dollars,
]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.coinbase_1d_sum,
&mut self.coinbase_in_dollars_1d_sum,
&mut self.fees_1d_sum,
&mut self.fees_in_dollars_1d_sum,
&mut self.subsidy_1d_sum,
&mut self.subsidy_in_dollars_1d_sum,
&mut self.total_blocks_mined,
&mut self.blocks_mined,
&mut self.last_subsidy,
&mut self.last_subsidy_in_dollars,
&mut self.last_coinbase,
&mut self.last_coinbase_in_dollars,
&mut self.last_fees,
&mut self.last_fees_in_dollars,
]
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.coinbase,
&self.coinbase_in_dollars,
&self.fees,
&self.fees_in_dollars,
&self.subsidy,
&self.subsidy_in_dollars,
&self.block_size,
&self.block_weight,
&self.block_vbytes,
&self.block_interval,
]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.coinbase,
&mut self.coinbase_in_dollars,
&mut self.fees,
&mut self.fees_in_dollars,
&mut self.subsidy,
&mut self.subsidy_in_dollars,
&mut self.block_size,
&mut self.block_weight,
&mut self.block_vbytes,
&mut self.block_interval,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.cumulative_coinbase,
&self.cumulative_coinbase_in_dollars,
&self.cumulative_fees,
&self.cumulative_fees_in_dollars,
&self.cumulative_subsidy,
&self.cumulative_subsidy_in_dollars,
&self.cumulative_block_size,
&self.cumulative_block_size_gigabytes,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.cumulative_coinbase,
&mut self.cumulative_coinbase_in_dollars,
&mut self.cumulative_fees,
&mut self.cumulative_fees_in_dollars,
&mut self.cumulative_subsidy,
&mut self.cumulative_subsidy_in_dollars,
&mut self.cumulative_block_size,
&mut self.cumulative_block_size_gigabytes,
]
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.subsidy_to_coinbase_ratio,
&self.fees_to_coinbase_ratio,
]
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.subsidy_to_coinbase_ratio,
&mut self.fees_to_coinbase_ratio,
]
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
[
&self.blocks_mined_1d_target as &(dyn AnyDateMap + Send + Sync),
&self.blocks_mined_1w_sma,
&self.blocks_mined_1m_sma,
&self.blocks_mined_1w_sum,
&self.blocks_mined_1m_sum,
&self.blocks_mined_1y_sum,
&self.blocks_mined_1w_target,
&self.blocks_mined_1m_target,
&self.blocks_mined_1y_target,
&self.subsidy_1y_sum,
&self.subsidy_in_dollars_1y_sum,
&self.coinbase_1y_sum,
&self.coinbase_in_dollars_1y_sum,
&self.coinbase_in_dollars_1d_sum_1y_sma,
&self.fees_to_coinbase_1d_ratio,
&self.annualized_issuance,
&self.fees_1y_sum,
&self.fees_in_dollars_1y_sum,
&self.inflation_rate,
&self.yearly_inflation_rate,
&self.subsidy_to_coinbase_1d_ratio,
&self.hash_rate,
&self.hash_rate_1w_sma,
&self.hash_rate_1m_sma,
&self.hash_rate_2m_sma,
&self.hash_price,
&self.hash_price_min,
&self.hash_price_rebound,
&self.puell_multiple,
&self.difficulty_adjustment,
]
.into_iter()
.chain(date_map_vec_to_any_date_map_vec(
self.block_size_recap.as_vec(),
))
.chain(date_map_vec_to_any_date_map_vec(
self.block_vbytes_recap.as_vec(),
))
.chain(date_map_vec_to_any_date_map_vec(
self.block_weight_recap.as_vec(),
))
.chain(date_map_vec_to_any_date_map_vec(
self.block_interval_recap.as_vec(),
))
.collect_vec()
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
[
&mut self.blocks_mined_1d_target as &mut dyn AnyDateMap,
&mut self.blocks_mined_1w_sma,
&mut self.blocks_mined_1m_sma,
&mut self.blocks_mined_1w_sum,
&mut self.blocks_mined_1m_sum,
&mut self.blocks_mined_1y_sum,
&mut self.blocks_mined_1w_target,
&mut self.blocks_mined_1m_target,
&mut self.blocks_mined_1y_target,
&mut self.annualized_issuance,
&mut self.subsidy_1y_sum,
&mut self.subsidy_in_dollars_1y_sum,
&mut self.fees_to_coinbase_1d_ratio,
&mut self.inflation_rate,
&mut self.yearly_inflation_rate,
&mut self.subsidy_to_coinbase_1d_ratio,
&mut self.coinbase_1y_sum,
&mut self.coinbase_in_dollars_1y_sum,
&mut self.coinbase_in_dollars_1d_sum_1y_sma,
&mut self.fees_1y_sum,
&mut self.fees_in_dollars_1y_sum,
&mut self.hash_rate,
&mut self.hash_rate_1w_sma,
&mut self.hash_rate_1m_sma,
&mut self.hash_rate_2m_sma,
&mut self.hash_price,
&mut self.hash_price_min,
&mut self.hash_price_rebound,
&mut self.puell_multiple,
&mut self.difficulty_adjustment,
]
.into_iter()
.chain(date_map_vec_to_mut_any_date_map_vec(
self.block_size_recap.as_mut_vec(),
))
.chain(date_map_vec_to_mut_any_date_map_vec(
self.block_vbytes_recap.as_mut_vec(),
))
.chain(date_map_vec_to_mut_any_date_map_vec(
self.block_weight_recap.as_mut_vec(),
))
.chain(date_map_vec_to_mut_any_date_map_vec(
self.block_interval_recap.as_mut_vec(),
))
.collect_vec()
}
}

275
parser/src/datasets/price/mod.rs
View File

@@ -1,25 +1,23 @@
mod ohlc;
use std::collections::BTreeMap;
use allocative::Allocative;
use chrono::Days;
use color_eyre::eyre::Error;
pub use ohlc::*;
use struct_iterable::Iterable;
use crate::{
price::{Binance, Kibo, Kraken},
structs::{
Amount, AnyBiMap, AnyDateMap, BiMap, Config, Date, DateMap, DateMapChunkId, Height,
HeightMapChunkId, MapKey, Timestamp,
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey,
MapKind, Timestamp, OHLC,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
use super::{AnyDataset, ComputeData, MinInitialStates, RatioDataset};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct PriceDatasets {
min_initial_states: MinInitialStates,
@@ -31,10 +29,7 @@ pub struct PriceDatasets {
kibo_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
kibo_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
// Inserted
pub ohlc: BiMap<OHLC>,
// Computed
pub open: BiMap<f32>,
pub high: BiMap<f32>,
pub low: BiMap<f32>,
@@ -106,66 +101,124 @@ impl PriceDatasets {
kibo_by_height: BTreeMap::default(),
kibo_by_date: BTreeMap::default(),
ohlc: BiMap::new_json(1, price_path),
open: BiMap::new_bin(1, &f("open")),
high: BiMap::new_bin(1, &f("high")),
low: BiMap::new_bin(1, &f("low")),
close: BiMap::new_bin(1, &f("close")),
market_cap: BiMap::new_bin(1, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, &f("price_1w_sma")),
// ---
// Inserted
// ---
ohlc: BiMap::new_json(1, MapKind::Inserted, price_path),
// ---
// Computed
// ---
open: BiMap::new_bin(1, MapKind::Computed, &f("open")),
high: BiMap::new_bin(1, MapKind::Computed, &f("high")),
low: BiMap::new_bin(1, MapKind::Computed, &f("low")),
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")),
price_1w_sma_ratio: RatioDataset::import(datasets_path, "price_1w_sma", config)?,
price_1m_sma: BiMap::new_bin(1, &f("price_1m_sma")),
price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")),
price_1m_sma_ratio: RatioDataset::import(datasets_path, "price_1m_sma", config)?,
price_1y_sma: BiMap::new_bin(1, &f("price_1y_sma")),
price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")),
price_1y_sma_ratio: RatioDataset::import(datasets_path, "price_1y_sma", config)?,
price_2y_sma: BiMap::new_bin(1, &f("price_2y_sma")),
price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")),
price_2y_sma_ratio: RatioDataset::import(datasets_path, "price_2y_sma", config)?,
price_4y_sma: BiMap::new_bin(1, &f("price_4y_sma")),
price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")),
price_4y_sma_ratio: RatioDataset::import(datasets_path, "price_4y_sma", config)?,
price_8d_sma: BiMap::new_bin(1, &f("price_8d_sma")),
price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")),
price_8d_sma_ratio: RatioDataset::import(datasets_path, "price_8d_sma", config)?,
price_13d_sma: BiMap::new_bin(1, &f("price_13d_sma")),
price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")),
price_13d_sma_ratio: RatioDataset::import(datasets_path, "price_13d_sma", config)?,
price_21d_sma: BiMap::new_bin(1, &f("price_21d_sma")),
price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")),
price_21d_sma_ratio: RatioDataset::import(datasets_path, "price_21d_sma", config)?,
price_34d_sma: BiMap::new_bin(1, &f("price_34d_sma")),
price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")),
price_34d_sma_ratio: RatioDataset::import(datasets_path, "price_34d_sma", config)?,
price_55d_sma: BiMap::new_bin(1, &f("price_55d_sma")),
price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")),
price_55d_sma_ratio: RatioDataset::import(datasets_path, "price_55d_sma", config)?,
price_89d_sma: BiMap::new_bin(1, &f("price_89d_sma")),
price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")),
price_89d_sma_ratio: RatioDataset::import(datasets_path, "price_89d_sma", config)?,
price_144d_sma: BiMap::new_bin(1, &f("price_144d_sma")),
price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")),
price_144d_sma_ratio: RatioDataset::import(datasets_path, "price_144d_sma", config)?,
price_200w_sma: BiMap::new_bin(1, &f("price_200w_sma")),
price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")),
price_200w_sma_ratio: RatioDataset::import(datasets_path, "price_200w_sma", config)?,
price_1d_total_return: DateMap::new_bin(1, &f("price_1d_total_return")),
price_1m_total_return: DateMap::new_bin(1, &f("price_1m_total_return")),
price_6m_total_return: DateMap::new_bin(1, &f("price_6m_total_return")),
price_1y_total_return: DateMap::new_bin(1, &f("price_1y_total_return")),
price_2y_total_return: DateMap::new_bin(1, &f("price_2y_total_return")),
price_3y_total_return: DateMap::new_bin(1, &f("price_3y_total_return")),
price_4y_total_return: DateMap::new_bin(1, &f("price_4y_total_return")),
price_6y_total_return: DateMap::new_bin(1, &f("price_6y_total_return")),
price_8y_total_return: DateMap::new_bin(1, &f("price_8y_total_return")),
price_10y_total_return: DateMap::new_bin(1, &f("price_10y_total_return")),
price_4y_compound_return: DateMap::new_bin(1, &f("price_4y_compound_return")),
all_time_high: BiMap::new_bin(1, &f("all_time_high")),
all_time_high_date: DateMap::new_bin(1, &f("all_time_high_date")),
days_since_all_time_high: DateMap::new_bin(1, &f("days_since_all_time_high")),
price_1d_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1d_total_return"),
),
price_1m_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1m_total_return"),
),
price_6m_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_6m_total_return"),
),
price_1y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1y_total_return"),
),
price_2y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_2y_total_return"),
),
price_3y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_3y_total_return"),
),
price_4y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_4y_total_return"),
),
price_6y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_6y_total_return"),
),
price_8y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_8y_total_return"),
),
price_10y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_10y_total_return"),
),
price_4y_compound_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_4y_compound_return"),
),
all_time_high: BiMap::new_bin(1, MapKind::Computed, &f("all_time_high")),
all_time_high_date: DateMap::new_bin(1, MapKind::Computed, &f("all_time_high_date")),
days_since_all_time_high: DateMap::new_bin(
1,
MapKind::Computed,
&f("days_since_all_time_high"),
),
max_days_between_all_time_highs: DateMap::new_bin(
1,
MapKind::Computed,
&f("max_days_between_all_time_highs"),
),
max_years_between_all_time_highs: DateMap::new_bin(
2,
MapKind::Computed,
&f("max_years_between_all_time_highs"),
),
market_price_to_all_time_high_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("market_price_to_all_time_high_ratio"),
),
drawdown: BiMap::new_bin(1, &f("drawdown")),
sats_per_dollar: BiMap::new_bin(1, &f("sats_per_dollar")),
drawdown: BiMap::new_bin(1, MapKind::Computed, &f("drawdown")),
sats_per_dollar: BiMap::new_bin(1, MapKind::Computed, &f("sats_per_dollar")),
};
s.min_initial_states
@@ -676,138 +729,4 @@ impl AnyDataset for PriceDatasets {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.ohlc]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.ohlc]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
let mut v = vec![
&self.open as &(dyn AnyBiMap + Send + Sync),
&self.high,
&self.low,
&self.close,
&self.market_cap,
&self.price_1w_sma,
&self.price_1m_sma,
&self.price_1y_sma,
&self.price_2y_sma,
&self.price_4y_sma,
&self.price_8d_sma,
&self.price_13d_sma,
&self.price_21d_sma,
&self.price_34d_sma,
&self.price_55d_sma,
&self.price_89d_sma,
&self.price_144d_sma,
&self.price_200w_sma,
&self.all_time_high,
&self.market_price_to_all_time_high_ratio,
&self.drawdown,
&self.sats_per_dollar,
];
v.append(&mut self.price_1w_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_1m_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_1y_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_2y_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_4y_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_8d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_13d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_21d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_34d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_55d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_89d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_144d_sma_ratio.to_computed_bi_map_vec());
v.append(&mut self.price_200w_sma_ratio.to_computed_bi_map_vec());
v
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
let mut v = vec![
&mut self.open as &mut dyn AnyBiMap,
&mut self.high,
&mut self.low,
&mut self.close,
&mut self.market_cap,
&mut self.price_1w_sma,
&mut self.price_1m_sma,
&mut self.price_1y_sma,
&mut self.price_2y_sma,
&mut self.price_4y_sma,
&mut self.price_8d_sma,
&mut self.price_13d_sma,
&mut self.price_21d_sma,
&mut self.price_34d_sma,
&mut self.price_55d_sma,
&mut self.price_89d_sma,
&mut self.price_144d_sma,
&mut self.price_200w_sma,
&mut self.all_time_high,
&mut self.market_price_to_all_time_high_ratio,
&mut self.drawdown,
&mut self.sats_per_dollar,
];
v.append(&mut self.price_1w_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_1m_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_1y_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_2y_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_4y_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_8d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_13d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_21d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_34d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_55d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_89d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_144d_sma_ratio.to_computed_mut_bi_map_vec());
v.append(&mut self.price_200w_sma_ratio.to_computed_mut_bi_map_vec());
v
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.price_1d_total_return,
&self.price_1m_total_return,
&self.price_6m_total_return,
&self.price_1y_total_return,
&self.price_2y_total_return,
&self.price_3y_total_return,
&self.price_4y_total_return,
&self.price_6y_total_return,
&self.price_8y_total_return,
&self.price_10y_total_return,
&self.price_4y_compound_return,
&self.all_time_high_date,
&self.days_since_all_time_high,
&self.max_days_between_all_time_highs,
&self.max_years_between_all_time_highs,
]
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.price_1d_total_return,
&mut self.price_1m_total_return,
&mut self.price_6m_total_return,
&mut self.price_1y_total_return,
&mut self.price_2y_total_return,
&mut self.price_3y_total_return,
&mut self.price_4y_total_return,
&mut self.price_6y_total_return,
&mut self.price_8y_total_return,
&mut self.price_10y_total_return,
&mut self.price_4y_compound_return,
&mut self.all_time_high_date,
&mut self.days_since_all_time_high,
&mut self.max_days_between_all_time_highs,
&mut self.max_years_between_all_time_highs,
]
}
}

51
parser/src/datasets/subs/capitalization.rs
View File

@@ -1,22 +1,20 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::CapitalizationState,
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
utils::ONE_MONTH_IN_DAYS,
};
use super::RatioDataset;
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct CapitalizationDataset {
min_initial_states: MinInitialStates,
// Inserted
pub realized_cap: BiMap<f32>,
// Computed
pub realized_price: BiMap<f32>,
realized_cap_1m_net_change: BiMap<f32>,
realized_price_ratio: RatioDataset,
@@ -39,9 +37,20 @@ impl CapitalizationDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
realized_cap: BiMap::new_bin(1, &f("realized_cap")),
realized_cap_1m_net_change: BiMap::new_bin(1, &f("realized_cap_1m_net_change")),
realized_price: BiMap::new_bin(1, &f("realized_price")),
// ---
// Inserted
// ---
realized_cap: BiMap::new_bin(1, MapKind::Inserted, &f("realized_cap")),
// ---
// Computed
// ---
realized_cap_1m_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("realized_cap_1m_net_change"),
),
realized_price: BiMap::new_bin(1, MapKind::Computed, &f("realized_price")),
realized_price_ratio: RatioDataset::import(
parent_path,
&format!(
@@ -109,30 +118,4 @@ impl AnyDataset for CapitalizationDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.realized_cap]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.realized_cap]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
let mut v = vec![
&self.realized_price as &(dyn AnyBiMap + Send + Sync),
&self.realized_cap_1m_net_change,
];
v.append(&mut self.realized_price_ratio.to_computed_bi_map_vec());
v
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
let mut v = vec![
&mut self.realized_price as &mut dyn AnyBiMap,
&mut self.realized_cap_1m_net_change,
];
v.append(&mut self.realized_price_ratio.to_computed_mut_bi_map_vec());
v
}
}

38
parser/src/datasets/subs/input.rs
View File

@@ -1,13 +1,14 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::InputState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
DateMap, HeightMap,
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct InputSubDataset {
min_initial_states: MinInitialStates,
@@ -36,9 +37,12 @@ impl InputSubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
count: BiMap::new_bin(1, &f("input_count")),
volume: HeightMap::new_bin(1, &f("input_volume")),
volume_1d_sum: DateMap::new_bin(1, &f("input_volume_1d_sum")),
// ---
// Inserted
// ---
count: BiMap::new_bin(1, MapKind::Inserted, &f("input_count")),
volume: HeightMap::new_bin(1, MapKind::Inserted, &f("input_volume")),
volume_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("input_volume_1d_sum")),
};
s.min_initial_states
@@ -78,28 +82,4 @@ impl AnyDataset for InputSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.count]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.count]
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![&self.volume]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![&mut self.volume]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![&self.volume_1d_sum]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![&mut self.volume_1d_sum]
}
}

3
parser/src/datasets/subs/mod.rs
View File

@@ -16,6 +16,7 @@ pub use price_paid::*;
pub use ratio::*;
pub use realized::*;
pub use recap::*;
use struct_iterable::Iterable;
pub use supply::*;
pub use unrealized::*;
pub use utxo::*;
@@ -24,7 +25,7 @@ use crate::{datasets::AnyDataset, structs::Config};
use super::AnyDatasetGroup;
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct SubDataset {
pub capitalization: CapitalizationDataset,
pub input: InputSubDataset,

16
parser/src/datasets/subs/output.rs
View File

@@ -84,20 +84,4 @@ impl AnyDataset for OutputSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.count, &self.volume]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.count, &mut self.volume]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.annualized_volume, &self.velocity]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.annualized_volume, &mut self.velocity]
}
}

86
parser/src/datasets/subs/price_paid.rs
View File

@@ -1,17 +1,16 @@
use allocative::Allocative;
use itertools::Itertools;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::PricePaidState,
structs::{AnyBiMap, BiMap, Config, Date, Height},
structs::{BiMap, Config, Date, Height, MapKind},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct PricePaidSubDataset {
min_initial_states: MinInitialStates,
// Inserted
pp_median: BiMap<f32>,
pp_95p: BiMap<f32>,
pp_90p: BiMap<f32>,
@@ -50,25 +49,28 @@ impl PricePaidSubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
pp_median: BiMap::new_bin(1, &f("median_price_paid")),
pp_95p: BiMap::new_bin(1, &f("95p_price_paid")),
pp_90p: BiMap::new_bin(1, &f("90p_price_paid")),
pp_85p: BiMap::new_bin(1, &f("85p_price_paid")),
pp_80p: BiMap::new_bin(1, &f("80p_price_paid")),
pp_75p: BiMap::new_bin(1, &f("75p_price_paid")),
pp_70p: BiMap::new_bin(1, &f("70p_price_paid")),
pp_65p: BiMap::new_bin(1, &f("65p_price_paid")),
pp_60p: BiMap::new_bin(1, &f("60p_price_paid")),
pp_55p: BiMap::new_bin(1, &f("55p_price_paid")),
pp_45p: BiMap::new_bin(1, &f("45p_price_paid")),
pp_40p: BiMap::new_bin(1, &f("40p_price_paid")),
pp_35p: BiMap::new_bin(1, &f("35p_price_paid")),
pp_30p: BiMap::new_bin(1, &f("30p_price_paid")),
pp_25p: BiMap::new_bin(1, &f("25p_price_paid")),
pp_20p: BiMap::new_bin(1, &f("20p_price_paid")),
pp_15p: BiMap::new_bin(1, &f("15p_price_paid")),
pp_10p: BiMap::new_bin(1, &f("10p_price_paid")),
pp_05p: BiMap::new_bin(1, &f("05p_price_paid")),
// ---
// Inserted
// ---
pp_median: BiMap::new_bin(1, MapKind::Inserted, &f("median_price_paid")),
pp_95p: BiMap::new_bin(1, MapKind::Inserted, &f("95p_price_paid")),
pp_90p: BiMap::new_bin(1, MapKind::Inserted, &f("90p_price_paid")),
pp_85p: BiMap::new_bin(1, MapKind::Inserted, &f("85p_price_paid")),
pp_80p: BiMap::new_bin(1, MapKind::Inserted, &f("80p_price_paid")),
pp_75p: BiMap::new_bin(1, MapKind::Inserted, &f("75p_price_paid")),
pp_70p: BiMap::new_bin(1, MapKind::Inserted, &f("70p_price_paid")),
pp_65p: BiMap::new_bin(1, MapKind::Inserted, &f("65p_price_paid")),
pp_60p: BiMap::new_bin(1, MapKind::Inserted, &f("60p_price_paid")),
pp_55p: BiMap::new_bin(1, MapKind::Inserted, &f("55p_price_paid")),
pp_45p: BiMap::new_bin(1, MapKind::Inserted, &f("45p_price_paid")),
pp_40p: BiMap::new_bin(1, MapKind::Inserted, &f("40p_price_paid")),
pp_35p: BiMap::new_bin(1, MapKind::Inserted, &f("35p_price_paid")),
pp_30p: BiMap::new_bin(1, MapKind::Inserted, &f("30p_price_paid")),
pp_25p: BiMap::new_bin(1, MapKind::Inserted, &f("25p_price_paid")),
pp_20p: BiMap::new_bin(1, MapKind::Inserted, &f("20p_price_paid")),
pp_15p: BiMap::new_bin(1, MapKind::Inserted, &f("15p_price_paid")),
pp_10p: BiMap::new_bin(1, MapKind::Inserted, &f("10p_price_paid")),
pp_05p: BiMap::new_bin(1, MapKind::Inserted, &f("05p_price_paid")),
};
s.min_initial_states
@@ -230,30 +232,6 @@ impl PricePaidSubDataset {
})
}
pub fn inserted_as_vec(&self) -> Vec<&BiMap<f32>> {
vec![
&self.pp_95p,
&self.pp_90p,
&self.pp_85p,
&self.pp_80p,
&self.pp_75p,
&self.pp_70p,
&self.pp_65p,
&self.pp_60p,
&self.pp_55p,
&self.pp_median,
&self.pp_45p,
&self.pp_40p,
&self.pp_35p,
&self.pp_30p,
&self.pp_25p,
&self.pp_20p,
&self.pp_15p,
&self.pp_10p,
&self.pp_05p,
]
}
pub fn inserted_as_mut_vec(&mut self) -> Vec<&mut BiMap<f32>> {
vec![
&mut self.pp_95p,
@@ -283,18 +261,4 @@ impl AnyDataset for PricePaidSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
self.inserted_as_vec()
.into_iter()
.map(|dataset| dataset as &(dyn AnyBiMap + Send + Sync))
.collect_vec()
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
self.inserted_as_mut_vec()
.into_iter()
.map(|dataset| dataset as &mut dyn AnyBiMap)
.collect_vec()
}
}

86
parser/src/datasets/subs/ratio.rs
View File

@@ -1,16 +1,16 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, MinInitialStates},
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct RatioDataset {
min_initial_states: MinInitialStates,
// Computed
ratio: BiMap<f32>,
ratio_1w_sma: BiMap<f32>,
ratio_1m_sma: BiMap<f32>,
@@ -38,26 +38,30 @@ impl RatioDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
ratio: BiMap::new_bin(1, &f_ratio("ratio")),
ratio_1w_sma: BiMap::new_bin(2, &f_ratio("ratio_1w_sma")),
ratio_1m_sma: BiMap::new_bin(2, &f_ratio("ratio_1m_sma")),
ratio_1y_sma: BiMap::new_bin(2, &f_ratio("ratio_1y_sma")),
// ---
// Computed
// ---
ratio: BiMap::new_bin(1, MapKind::Computed, &f_ratio("ratio")),
ratio_1w_sma: BiMap::new_bin(2, MapKind::Computed, &f_ratio("ratio_1w_sma")),
ratio_1m_sma: BiMap::new_bin(2, MapKind::Computed, &f_ratio("ratio_1m_sma")),
ratio_1y_sma: BiMap::new_bin(2, MapKind::Computed, &f_ratio("ratio_1y_sma")),
ratio_1y_sma_momentum_oscillator: BiMap::new_bin(
2,
MapKind::Computed,
&f_ratio("ratio_1y_sma_momentum_oscillator"),
),
ratio_99p: BiMap::new_bin(3, &f_ratio("ratio_99p")),
ratio_99_5p: BiMap::new_bin(3, &f_ratio("ratio_99_5p")),
ratio_99_9p: BiMap::new_bin(3, &f_ratio("ratio_99_9p")),
ratio_1p: BiMap::new_bin(3, &f_ratio("ratio_1p")),
ratio_0_5p: BiMap::new_bin(3, &f_ratio("ratio_0_5p")),
ratio_0_1p: BiMap::new_bin(3, &f_ratio("ratio_0_1p")),
price_99p: BiMap::new_bin(4, &f_price("99p")),
price_99_5p: BiMap::new_bin(4, &f_price("99_5p")),
price_99_9p: BiMap::new_bin(4, &f_price("99_9p")),
price_1p: BiMap::new_bin(4, &f_price("1p")),
price_0_5p: BiMap::new_bin(4, &f_price("0_5p")),
price_0_1p: BiMap::new_bin(4, &f_price("0_1p")),
ratio_99p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_99p")),
ratio_99_5p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_99_5p")),
ratio_99_9p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_99_9p")),
ratio_1p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_1p")),
ratio_0_5p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_0_5p")),
ratio_0_1p: BiMap::new_bin(3, MapKind::Computed, &f_ratio("ratio_0_1p")),
price_99p: BiMap::new_bin(4, MapKind::Computed, &f_price("99p")),
price_99_5p: BiMap::new_bin(4, MapKind::Computed, &f_price("99_5p")),
price_99_9p: BiMap::new_bin(4, MapKind::Computed, &f_price("99_9p")),
price_1p: BiMap::new_bin(4, MapKind::Computed, &f_price("1p")),
price_0_5p: BiMap::new_bin(4, MapKind::Computed, &f_price("0_5p")),
price_0_1p: BiMap::new_bin(4, MapKind::Computed, &f_price("0_1p")),
};
s.min_initial_states
@@ -164,48 +168,4 @@ impl AnyDataset for RatioDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.ratio,
&self.ratio_1w_sma,
&self.ratio_1m_sma,
&self.ratio_1y_sma,
&self.ratio_1y_sma_momentum_oscillator,
&self.ratio_99p,
&self.ratio_99_5p,
&self.ratio_99_9p,
&self.ratio_1p,
&self.ratio_0_5p,
&self.ratio_0_1p,
&self.price_99p,
&self.price_99_5p,
&self.price_99_9p,
&self.price_1p,
&self.price_0_5p,
&self.price_0_1p,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.ratio,
&mut self.ratio_1w_sma,
&mut self.ratio_1m_sma,
&mut self.ratio_1y_sma,
&mut self.ratio_1y_sma_momentum_oscillator,
&mut self.ratio_99p,
&mut self.ratio_99_5p,
&mut self.ratio_99_9p,
&mut self.ratio_1p,
&mut self.ratio_0_5p,
&mut self.ratio_0_1p,
&mut self.price_99p,
&mut self.price_99_5p,
&mut self.price_99_9p,
&mut self.price_1p,
&mut self.price_0_5p,
&mut self.price_0_1p,
]
}
}

238
parser/src/datasets/subs/realized.rs
View File

@@ -1,18 +1,18 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::RealizedState,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Price},
structs::{BiMap, Config, MapKind, Price},
utils::ONE_MONTH_IN_DAYS,
DateMap, HeightMap,
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct RealizedSubDataset {
min_initial_states: MinInitialStates,
// Inserted
realized_profit: HeightMap<f32>,
realized_loss: HeightMap<f32>,
value_created: HeightMap<f32>,
@@ -27,8 +27,6 @@ pub struct RealizedSubDataset {
adjusted_value_destroyed_1d_sum: DateMap<f32>,
spent_output_profit_ratio: BiMap<f32>,
adjusted_spent_output_profit_ratio: BiMap<f32>,
// Computed
negative_realized_loss: HeightMap<f32>,
negative_realized_loss_1d_sum: DateMap<f32>,
net_realized_profit_and_loss: HeightMap<f32>,
@@ -62,57 +60,131 @@ impl RealizedSubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
realized_profit: HeightMap::new_bin(1, &f("realized_profit")),
realized_loss: HeightMap::new_bin(1, &f("realized_loss")),
value_created: HeightMap::new_bin(1, &f("value_created")),
adjusted_value_created: HeightMap::new_bin(1, &f("adjusted_value_created")),
value_destroyed: HeightMap::new_bin(1, &f("value_destroyed")),
adjusted_value_destroyed: HeightMap::new_bin(1, &f("adjusted_value_destroyed")),
realized_profit_1d_sum: DateMap::new_bin(1, &f("realized_profit_1d_sum")),
realized_loss_1d_sum: DateMap::new_bin(1, &f("realized_loss_1d_sum")),
value_created_1d_sum: DateMap::new_bin(1, &f("value_created_1d_sum")),
adjusted_value_created_1d_sum: DateMap::new_bin(1, &f("adjusted_value_created_1d_sum")),
value_destroyed_1d_sum: DateMap::new_bin(1, &f("value_destroyed_1d_sum")),
// ---
// Inserted
// ---
realized_profit: HeightMap::new_bin(1, MapKind::Inserted, &f("realized_profit")),
realized_loss: HeightMap::new_bin(1, MapKind::Inserted, &f("realized_loss")),
value_created: HeightMap::new_bin(1, MapKind::Inserted, &f("value_created")),
adjusted_value_created: HeightMap::new_bin(
1,
MapKind::Inserted,
&f("adjusted_value_created"),
),
value_destroyed: HeightMap::new_bin(1, MapKind::Inserted, &f("value_destroyed")),
adjusted_value_destroyed: HeightMap::new_bin(
1,
MapKind::Inserted,
&f("adjusted_value_destroyed"),
),
realized_profit_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("realized_profit_1d_sum"),
),
realized_loss_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("realized_loss_1d_sum"),
),
value_created_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("value_created_1d_sum"),
),
adjusted_value_created_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("adjusted_value_created_1d_sum"),
),
value_destroyed_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("value_destroyed_1d_sum"),
),
adjusted_value_destroyed_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("adjusted_value_destroyed_1d_sum"),
),
spent_output_profit_ratio: BiMap::new_bin(2, &f("spent_output_profit_ratio")),
spent_output_profit_ratio: BiMap::new_bin(
2,
MapKind::Inserted,
&f("spent_output_profit_ratio"),
),
adjusted_spent_output_profit_ratio: BiMap::new_bin(
2,
MapKind::Inserted,
&f("adjusted_spent_output_profit_ratio"),
),
negative_realized_loss: HeightMap::new_bin(2, &f("negative_realized_loss")),
negative_realized_loss_1d_sum: DateMap::new_bin(2, &f("negative_realized_loss_1d_sum")),
net_realized_profit_and_loss: HeightMap::new_bin(1, &f("net_realized_profit_and_loss")),
// ---
// Computed
// ---
negative_realized_loss: HeightMap::new_bin(
2,
MapKind::Computed,
&f("negative_realized_loss"),
),
negative_realized_loss_1d_sum: DateMap::new_bin(
2,
MapKind::Computed,
&f("negative_realized_loss_1d_sum"),
),
net_realized_profit_and_loss: HeightMap::new_bin(
1,
MapKind::Computed,
&f("net_realized_profit_and_loss"),
),
net_realized_profit_and_loss_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("net_realized_profit_and_loss_1d_sum"),
),
net_realized_profit_and_loss_1d_sum_to_market_cap_ratio: DateMap::new_bin(
2,
MapKind::Computed,
&f("net_realized_profit_and_loss_to_market_cap_ratio"),
),
cumulative_realized_profit: BiMap::new_bin(1, &f("cumulative_realized_profit")),
cumulative_realized_loss: BiMap::new_bin(1, &f("cumulative_realized_loss")),
cumulative_realized_profit: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_realized_profit"),
),
cumulative_realized_loss: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_realized_loss"),
),
cumulative_net_realized_profit_and_loss: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_net_realized_profit_and_loss"),
),
cumulative_net_realized_profit_and_loss_1m_net_change: BiMap::new_bin(
1,
MapKind::Computed,
&f("cumulative_net_realized_profit_and_loss_1m_net_change"),
),
realized_value: HeightMap::new_bin(1, &f("realized_value")),
realized_value_1d_sum: DateMap::new_bin(1, &f("realized_value_1d_sum")),
sell_side_risk_ratio: DateMap::new_bin(1, &f("sell_side_risk_ratio")),
realized_value: HeightMap::new_bin(1, MapKind::Computed, &f("realized_value")),
realized_value_1d_sum: DateMap::new_bin(
1,
MapKind::Computed,
&f("realized_value_1d_sum"),
),
sell_side_risk_ratio: DateMap::new_bin(
1,
MapKind::Computed,
&f("sell_side_risk_ratio"),
),
realized_profit_to_loss_ratio: HeightMap::new_bin(
1,
MapKind::Computed,
&f("realized_profit_to_loss_ratio"),
),
realized_profit_to_loss_1d_sum_ratio: DateMap::new_bin(
1,
MapKind::Computed,
&f("realized_profit_to_loss_1d_sum_ratio"),
),
};
@@ -312,120 +384,4 @@ impl AnyDataset for RealizedSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.spent_output_profit_ratio,
&self.adjusted_spent_output_profit_ratio,
]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.spent_output_profit_ratio,
&mut self.adjusted_spent_output_profit_ratio,
]
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.realized_loss,
&self.realized_profit,
&self.value_created,
&self.adjusted_value_created,
&self.value_destroyed,
&self.adjusted_value_destroyed,
]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.realized_loss,
&mut self.realized_profit,
&mut self.value_created,
&mut self.adjusted_value_created,
&mut self.value_destroyed,
&mut self.adjusted_value_destroyed,
]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.realized_loss_1d_sum,
&self.realized_profit_1d_sum,
&self.value_created_1d_sum,
&self.adjusted_value_created_1d_sum,
&self.value_destroyed_1d_sum,
&self.adjusted_value_destroyed_1d_sum,
]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.realized_loss_1d_sum,
&mut self.realized_profit_1d_sum,
&mut self.value_created_1d_sum,
&mut self.adjusted_value_created_1d_sum,
&mut self.value_destroyed_1d_sum,
&mut self.adjusted_value_destroyed_1d_sum,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.cumulative_realized_profit,
&self.cumulative_realized_loss,
&self.cumulative_net_realized_profit_and_loss,
&self.cumulative_net_realized_profit_and_loss_1m_net_change,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.cumulative_realized_profit,
&mut self.cumulative_realized_loss,
&mut self.cumulative_net_realized_profit_and_loss,
&mut self.cumulative_net_realized_profit_and_loss_1m_net_change,
]
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.negative_realized_loss,
&self.net_realized_profit_and_loss,
&self.realized_value,
&self.realized_profit_to_loss_ratio,
]
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.negative_realized_loss,
&mut self.net_realized_profit_and_loss,
&mut self.realized_value,
&mut self.realized_profit_to_loss_ratio,
]
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.sell_side_risk_ratio,
&self.negative_realized_loss_1d_sum,
&self.net_realized_profit_and_loss_1d_sum,
&self.net_realized_profit_and_loss_1d_sum_to_market_cap_ratio,
&self.realized_value_1d_sum,
&self.realized_profit_to_loss_1d_sum_ratio,
]
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.sell_side_risk_ratio,
&mut self.negative_realized_loss_1d_sum,
&mut self.net_realized_profit_and_loss_1d_sum,
&mut self.net_realized_profit_and_loss_1d_sum_to_market_cap_ratio,
&mut self.realized_value_1d_sum,
&mut self.realized_profit_to_loss_1d_sum_ratio,
]
}
}

57
parser/src/datasets/subs/recap.rs
View File

@@ -3,7 +3,7 @@ use std::{iter::Sum, ops::Add};
use allocative::Allocative;
use crate::{
structs::{DateMapChunkId, GenericMap, MapKey, MapSerialized, MapValue},
structs::{DateMapChunkId, GenericMap, MapKey, MapKind, MapSerialized, MapValue},
utils::{get_percentile, LossyFrom},
Date, MapChunkId, SerializedBTreeMap,
};
@@ -95,17 +95,36 @@ where
let f = |s: &str| format!("{parent_path}/{s}");
let s = Self {
min: options.min.then(|| GenericMap::new_bin(1, &f("min"))),
max: options.max.then(|| GenericMap::new_bin(1, &f("max"))),
median: options.median.then(|| GenericMap::new_bin(1, &f("median"))),
// ---
// Computed
// ---
min: options
.min
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("min"))),
max: options
.max
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("max"))),
median: options
.median
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("median"))),
average: options
.average
.then(|| GenericMap::new_bin(1, &f("average"))),
sum: options.sum.then(|| GenericMap::new_bin(1, &f("sum"))),
_90p: options._90p.then(|| GenericMap::new_bin(1, &f("90p"))),
_75p: options._75p.then(|| GenericMap::new_bin(1, &f("75p"))),
_25p: options._25p.then(|| GenericMap::new_bin(1, &f("25p"))),
_10p: options._10p.then(|| GenericMap::new_bin(1, &f("10p"))),
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("average"))),
sum: options
.sum
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("sum"))),
_90p: options
._90p
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("90p"))),
_75p: options
._75p
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("75p"))),
_25p: options
._25p
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("25p"))),
_10p: options
._10p
.then(|| GenericMap::new_bin(1, MapKind::Computed, &f("10p"))),
};
Ok(s)
@@ -128,31 +147,31 @@ where
values.sort_unstable();
if let Some(max) = self.max.as_mut() {
max.insert(key, Value::lossy_from(*values.last().unwrap()));
max.insert_computed(key, Value::lossy_from(*values.last().unwrap()));
}
if let Some(_90p) = self._90p.as_mut() {
_90p.insert(key, Value::lossy_from(get_percentile(values, 0.90)));
_90p.insert_computed(key, Value::lossy_from(get_percentile(values, 0.90)));
}
if let Some(_75p) = self._75p.as_mut() {
_75p.insert(key, Value::lossy_from(get_percentile(values, 0.75)));
_75p.insert_computed(key, Value::lossy_from(get_percentile(values, 0.75)));
}
if let Some(median) = self.median.as_mut() {
median.insert(key, Value::lossy_from(get_percentile(values, 0.50)));
median.insert_computed(key, Value::lossy_from(get_percentile(values, 0.50)));
}
if let Some(_25p) = self._25p.as_mut() {
_25p.insert(key, Value::lossy_from(get_percentile(values, 0.25)));
_25p.insert_computed(key, Value::lossy_from(get_percentile(values, 0.25)));
}
if let Some(_10p) = self._10p.as_mut() {
_10p.insert(key, Value::lossy_from(get_percentile(values, 0.10)));
_10p.insert_computed(key, Value::lossy_from(get_percentile(values, 0.10)));
}
if let Some(min) = self.min.as_mut() {
min.insert(key, Value::lossy_from(*values.first().unwrap()));
min.insert_computed(key, Value::lossy_from(*values.first().unwrap()));
}
}
@@ -160,12 +179,12 @@ where
let sum = Value::lossy_from(values.iter().sum::<Value2>());
if let Some(sum_map) = self.sum.as_mut() {
sum_map.insert(key, sum);
sum_map.insert_computed(key, sum);
}
if let Some(average) = self.average.as_mut() {
let len = values.len() as f32;
average.insert(key, Value::lossy_from(f32::lossy_from(sum) / len));
average.insert_computed(key, Value::lossy_from(f32::lossy_from(sum) / len));
}
}
}

45
parser/src/datasets/subs/supply.rs
View File

@@ -1,19 +1,17 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::SupplyState,
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct SupplySubDataset {
min_initial_states: MinInitialStates,
// Inserted
pub supply: BiMap<f64>,
// Computed
pub supply_to_circulating_supply_ratio: BiMap<f64>,
pub halved_supply: BiMap<f64>,
pub halved_supply_to_circulating_supply_ratio: BiMap<f64>,
@@ -36,14 +34,23 @@ impl SupplySubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
supply: BiMap::new_bin(1, &f("supply")),
// ---
// Inserted
// ---
supply: BiMap::new_bin(1, MapKind::Inserted, &f("supply")),
// ---
// Computed,
// ---
supply_to_circulating_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("supply_to_circulating_supply_ratio"),
),
halved_supply: BiMap::new_bin(1, &f("halved_supply")),
halved_supply: BiMap::new_bin(1, MapKind::Computed, &f("halved_supply")),
halved_supply_to_circulating_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("halved_supply_to_circulating_supply_ratio"),
),
};
@@ -97,28 +104,4 @@ impl AnyDataset for SupplySubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.supply]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.supply]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.supply_to_circulating_supply_ratio,
&self.halved_supply,
&self.halved_supply_to_circulating_supply_ratio,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.supply_to_circulating_supply_ratio,
&mut self.halved_supply,
&mut self.halved_supply_to_circulating_supply_ratio,
]
}
}

82
parser/src/datasets/subs/unrealized.rs
View File

@@ -1,21 +1,19 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
states::UnrealizedState,
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct UnrealizedSubDataset {
min_initial_states: MinInitialStates,
// Inserted
supply_in_profit: BiMap<f64>,
unrealized_profit: BiMap<f32>,
unrealized_loss: BiMap<f32>,
// Computed
supply_in_loss: BiMap<f64>,
negative_unrealized_loss: BiMap<f32>,
net_unrealized_profit_and_loss: BiMap<f32>,
@@ -43,30 +41,50 @@ impl UnrealizedSubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
supply_in_profit: BiMap::new_bin(1, &f("supply_in_profit")),
supply_in_loss: BiMap::new_bin(1, &f("supply_in_loss")),
unrealized_profit: BiMap::new_bin(1, &f("unrealized_profit")),
unrealized_loss: BiMap::new_bin(1, &f("unrealized_loss")),
negative_unrealized_loss: BiMap::new_bin(1, &f("negative_unrealized_loss")),
net_unrealized_profit_and_loss: BiMap::new_bin(1, &f("net_unrealized_profit_and_loss")),
// ---
// Inserted
// ---
supply_in_profit: BiMap::new_bin(1, MapKind::Inserted, &f("supply_in_profit")),
unrealized_profit: BiMap::new_bin(1, MapKind::Inserted, &f("unrealized_profit")),
unrealized_loss: BiMap::new_bin(1, MapKind::Inserted, &f("unrealized_loss")),
// ---
// Inserted
// ---
supply_in_loss: BiMap::new_bin(1, MapKind::Computed, &f("supply_in_loss")),
negative_unrealized_loss: BiMap::new_bin(
1,
MapKind::Computed,
&f("negative_unrealized_loss"),
),
net_unrealized_profit_and_loss: BiMap::new_bin(
1,
MapKind::Computed,
&f("net_unrealized_profit_and_loss"),
),
net_unrealized_profit_and_loss_to_market_cap_ratio: BiMap::new_bin(
2,
MapKind::Computed,
&f("net_unrealized_profit_and_loss_to_market_cap_ratio"),
),
supply_in_profit_to_own_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("supply_in_profit_to_own_supply_ratio"),
),
supply_in_profit_to_circulating_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("supply_in_profit_to_circulating_supply_ratio"),
),
supply_in_loss_to_own_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("supply_in_loss_to_own_supply_ratio"),
),
supply_in_loss_to_circulating_supply_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("supply_in_loss_to_circulating_supply_ratio"),
),
};
@@ -176,46 +194,4 @@ impl AnyDataset for UnrealizedSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.supply_in_profit,
&self.unrealized_profit,
&self.unrealized_loss,
]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.supply_in_profit,
&mut self.unrealized_profit,
&mut self.unrealized_loss,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.supply_in_loss,
&self.negative_unrealized_loss,
&self.net_unrealized_profit_and_loss,
&self.net_unrealized_profit_and_loss_to_market_cap_ratio,
&self.supply_in_profit_to_own_supply_ratio,
&self.supply_in_profit_to_circulating_supply_ratio,
&self.supply_in_loss_to_own_supply_ratio,
&self.supply_in_loss_to_circulating_supply_ratio,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.supply_in_loss,
&mut self.negative_unrealized_loss,
&mut self.net_unrealized_profit_and_loss,
&mut self.net_unrealized_profit_and_loss_to_market_cap_ratio,
&mut self.supply_in_profit_to_own_supply_ratio,
&mut self.supply_in_profit_to_circulating_supply_ratio,
&mut self.supply_in_loss_to_own_supply_ratio,
&mut self.supply_in_loss_to_circulating_supply_ratio,
]
}
}

19
parser/src/datasets/subs/utxo.rs
View File

@@ -1,16 +1,16 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::{AnyDataset, InsertData, MinInitialStates},
states::UTXOState,
structs::{AnyBiMap, BiMap, Config},
structs::{BiMap, Config, MapKind},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct UTXOSubDataset {
min_initial_states: MinInitialStates,
// Inserted
count: BiMap<f64>,
}
@@ -31,7 +31,10 @@ impl UTXOSubDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
count: BiMap::new_bin(1, &f("utxo_count")),
// ---
// Inserted
// ---
count: BiMap::new_bin(1, MapKind::Inserted, &f("utxo_count")),
};
s.min_initial_states
@@ -62,12 +65,4 @@ impl AnyDataset for UTXOSubDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![&self.count]
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![&mut self.count]
}
}

195
parser/src/datasets/transaction.rs
View File

@@ -1,8 +1,9 @@
use allocative::Allocative;
use struct_iterable::Iterable;
use crate::{
datasets::InsertData,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, HeightMap},
structs::{BiMap, Config, HeightMap, MapKind},
utils::{
ONE_DAY_IN_S, ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS, TARGET_BLOCKS_PER_DAY,
},
@@ -11,11 +12,10 @@ use crate::{
use super::{AnyDataset, ComputeData, MinInitialStates};
#[derive(Allocative)]
#[derive(Allocative, Iterable)]
pub struct TransactionDataset {
min_initial_states: MinInitialStates,
// Inserted
pub count: HeightMap<usize>,
pub count_1d_sum: DateMap<usize>,
pub volume: HeightMap<f64>,
@@ -31,8 +31,6 @@ pub struct TransactionDataset {
// 10th 25th 75th 90th percentiles
// type
// version
// Computed
pub count_1w_sma: HeightMap<f32>,
pub count_1d_sum_1w_sma: DateMap<f32>,
pub count_1m_sma: HeightMap<f32>,
@@ -60,48 +58,105 @@ impl TransactionDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
count: HeightMap::new_bin(1, &f("transaction_count")),
count_1d_sum: DateMap::new_bin(1, &f("transaction_count_1d_sum")),
count_1w_sma: HeightMap::new_bin(1, &f("transaction_count_1w_sma")),
count_1d_sum_1w_sma: DateMap::new_bin(1, &f("transaction_count_1d_sum_1w_sma")),
count_1m_sma: HeightMap::new_bin(1, &f("transaction_count_1m_sma")),
count_1d_sum_1m_sma: DateMap::new_bin(1, &f("transaction_count_1d_sum_1m_sma")),
volume: HeightMap::new_bin(1, &f("transaction_volume")),
volume_1d_sum: DateMap::new_bin(1, &f("transaction_volume_1d_sum")),
volume_1w_sma: HeightMap::new_bin(1, &f("transaction_volume_1w_sma")),
volume_1d_sum_1w_sma: DateMap::new_bin(1, &f("transaction_volume_1d_sum_1w_sma")),
volume_1m_sma: HeightMap::new_bin(1, &f("transaction_volume_1m_sma")),
volume_1d_sum_1m_sma: DateMap::new_bin(1, &f("transaction_volume_1d_sum_1m_sma")),
volume_in_dollars: HeightMap::new_bin(1, &f("transaction_volume_in_dollars")),
// ---
// Inserted
// ---
count: HeightMap::new_bin(1, MapKind::Inserted, &f("transaction_count")),
count_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("transaction_count_1d_sum")),
volume: HeightMap::new_bin(1, MapKind::Inserted, &f("transaction_volume")),
volume_1d_sum: DateMap::new_bin(1, MapKind::Inserted, &f("transaction_volume_1d_sum")),
volume_in_dollars: HeightMap::new_bin(
1,
MapKind::Inserted,
&f("transaction_volume_in_dollars"),
),
volume_in_dollars_1d_sum: DateMap::new_bin(
1,
MapKind::Inserted,
&f("transaction_volume_in_dollars_1d_sum"),
),
// ---
// Inserted
// ---
count_1w_sma: HeightMap::new_bin(1, MapKind::Computed, &f("transaction_count_1w_sma")),
count_1d_sum_1w_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_count_1d_sum_1w_sma"),
),
count_1m_sma: HeightMap::new_bin(1, MapKind::Computed, &f("transaction_count_1m_sma")),
count_1d_sum_1m_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_count_1d_sum_1m_sma"),
),
volume_1w_sma: HeightMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_1w_sma"),
),
volume_1d_sum_1w_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_1d_sum_1w_sma"),
),
volume_1m_sma: HeightMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_1m_sma"),
),
volume_1d_sum_1m_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_1d_sum_1m_sma"),
),
volume_in_dollars_1w_sma: HeightMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_in_dollars_1w_sma"),
),
volume_in_dollars_1d_sum_1w_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_in_dollars_1d_sum_1w_sma"),
),
volume_in_dollars_1m_sma: HeightMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_in_dollars_1m_sma"),
),
volume_in_dollars_1d_sum_1m_sma: DateMap::new_bin(
1,
MapKind::Computed,
&f("transaction_volume_in_dollars_1d_sum_1m_sma"),
),
annualized_volume: DateMap::new_bin(1, &f("annualized_transaction_volume")),
annualized_volume: DateMap::new_bin(
1,
MapKind::Computed,
&f("annualized_transaction_volume"),
),
annualized_volume_in_dollars: DateMap::new_bin(
2,
MapKind::Computed,
&f("annualized_transaction_volume_in_dollars"),
),
velocity: DateMap::new_bin(1, &f("transaction_velocity")),
transactions_per_second: BiMap::new_bin(1, &f("transactions_per_second")),
transactions_per_second_1w_sma: BiMap::new_bin(1, &f("transactions_per_second_1w_sma")),
transactions_per_second_1m_sma: BiMap::new_bin(1, &f("transactions_per_second_1m_sma")),
velocity: DateMap::new_bin(1, MapKind::Computed, &f("transaction_velocity")),
transactions_per_second: BiMap::new_bin(
1,
MapKind::Computed,
&f("transactions_per_second"),
),
transactions_per_second_1w_sma: BiMap::new_bin(
1,
MapKind::Computed,
&f("transactions_per_second_1w_sma"),
),
transactions_per_second_1m_sma: BiMap::new_bin(
1,
MapKind::Computed,
&f("transactions_per_second_1m_sma"),
),
};
s.min_initial_states
@@ -268,98 +323,4 @@ impl AnyDataset for TransactionDataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![&self.count, &self.volume, &self.volume_in_dollars]
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.count,
&mut self.volume,
&mut self.volume_in_dollars,
]
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.count_1d_sum,
&self.volume_1d_sum,
&self.volume_in_dollars_1d_sum,
]
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.count_1d_sum,
&mut self.volume_1d_sum,
&mut self.volume_in_dollars_1d_sum,
]
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
vec![
&self.transactions_per_second,
&self.transactions_per_second_1w_sma,
&self.transactions_per_second_1m_sma,
]
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
vec![
&mut self.transactions_per_second,
&mut self.transactions_per_second_1w_sma,
&mut self.transactions_per_second_1m_sma,
]
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
vec![
&self.count_1w_sma,
&self.count_1m_sma,
&self.volume_1w_sma,
&self.volume_1m_sma,
&self.volume_in_dollars_1w_sma,
&self.volume_in_dollars_1m_sma,
]
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
vec![
&mut self.count_1w_sma,
&mut self.count_1m_sma,
&mut self.volume_1w_sma,
&mut self.volume_1m_sma,
&mut self.volume_in_dollars_1w_sma,
&mut self.volume_in_dollars_1m_sma,
]
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
vec![
&self.count_1d_sum_1w_sma,
&self.count_1d_sum_1m_sma,
&self.volume_1d_sum_1w_sma,
&self.volume_1d_sum_1m_sma,
&self.volume_in_dollars_1d_sum_1w_sma,
&self.volume_in_dollars_1d_sum_1m_sma,
&self.annualized_volume,
&self.annualized_volume_in_dollars,
&self.velocity,
]
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
vec![
&mut self.count_1d_sum_1w_sma,
&mut self.count_1d_sum_1m_sma,
&mut self.volume_1d_sum_1w_sma,
&mut self.volume_1d_sum_1m_sma,
&mut self.volume_in_dollars_1d_sum_1w_sma,
&mut self.volume_in_dollars_1d_sum_1m_sma,
&mut self.annualized_volume,
&mut self.annualized_volume_in_dollars,
&mut self.velocity,
]
}
}

106
parser/src/datasets/utxo/dataset.rs
View File

@@ -1,15 +1,13 @@
use allocative::Allocative;
use itertools::Itertools;
use struct_iterable::Iterable;
use crate::{
datasets::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
states::UTXOCohortId,
structs::{AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Config, Date, Height},
structs::{BiMap, Config, Date, Height},
};
#[derive(Default, Allocative)]
#[derive(Allocative, Iterable)]
pub struct UTXODataset {
id: UTXOCohortId,
@@ -196,100 +194,4 @@ impl AnyDataset for UTXODataset {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
fn to_inserted_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_height_map_vec())
.collect_vec()
}
fn to_inserted_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_date_map_vec())
.collect_vec()
}
fn to_inserted_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_inserted_bi_map_vec())
.collect_vec()
}
fn to_inserted_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_height_map_vec())
.collect_vec()
}
fn to_inserted_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_date_map_vec())
.collect_vec()
}
fn to_inserted_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_inserted_mut_bi_map_vec())
.collect_vec()
}
fn to_computed_height_map_vec(&self) -> Vec<&(dyn AnyHeightMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_height_map_vec())
.collect_vec()
}
fn to_computed_date_map_vec(&self) -> Vec<&(dyn AnyDateMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_date_map_vec())
.collect_vec()
}
fn to_computed_bi_map_vec(&self) -> Vec<&(dyn AnyBiMap + Send + Sync)> {
self.subs
.as_vec()
.into_iter()
.flat_map(|d| d.to_computed_bi_map_vec())
.collect_vec()
}
fn to_computed_mut_height_map_vec(&mut self) -> Vec<&mut dyn AnyHeightMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_height_map_vec())
.collect_vec()
}
fn to_computed_mut_date_map_vec(&mut self) -> Vec<&mut dyn AnyDateMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_date_map_vec())
.collect_vec()
}
fn to_computed_mut_bi_map_vec(&mut self) -> Vec<&mut dyn AnyBiMap> {
self.subs
.as_mut_vec()
.into_iter()
.flat_map(|d| d.to_computed_mut_bi_map_vec())
.collect_vec()
}
}

6
parser/src/datasets/utxo/mod.rs
View File

@@ -23,7 +23,7 @@ pub struct UTXODatasets {
impl UTXODatasets {
pub fn import(parent_path: &str, config: &Config) -> color_eyre::Result<Self> {
let mut cohorts = SplitByUTXOCohort::<UTXODataset>::default();
let mut cohorts = SplitByUTXOCohort::<Option<UTXODataset>>::default();
cohorts
.as_vec()
@@ -32,14 +32,14 @@ impl UTXODatasets {
.collect::<Vec<_>>()
.into_iter()
.try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
*cohorts.get_mut(&id) = dataset?;
cohorts.get_mut(&id).replace(dataset?);
Ok(())
})?;
let mut s = Self {
min_initial_states: MinInitialStates::default(),
cohorts,
cohorts: cohorts.unwrap(),
};
s.min_initial_states

3
parser/src/lib.rs
View File

@@ -9,11 +9,10 @@ mod utils;
pub use crate::{
actions::iter_blocks,
datasets::OHLC,
io::{Binary, Json, Serialization, COMPRESSED_BIN_EXTENSION, JSON_EXTENSION},
structs::{
Config, Date, DateMap, Exit, Height, HeightMap, MapChunkId, MapValue, SerializedBTreeMap,
SerializedVec, HEIGHT_MAP_CHUNK_SIZE,
SerializedVec, HEIGHT_MAP_CHUNK_SIZE, OHLC,
},
utils::{create_rpc, log, reset_logs},
};

3
parser/src/price/binance.rs
View File

@@ -7,9 +7,8 @@ use itertools::Itertools;
use serde_json::Value;
use crate::{
datasets::OHLC,
io::{Json, INPUTS_FOLDER_PATH},
structs::{Date, Timestamp},
structs::{Date, Timestamp, OHLC},
utils::{log, retry},
};

3
parser/src/price/kibo.rs
View File

@@ -6,8 +6,7 @@ use itertools::Itertools;
use serde_json::Value;
use crate::{
datasets::OHLC,
structs::{Date, DateMapChunkId, HeightMapChunkId},
structs::{Date, DateMapChunkId, HeightMapChunkId, OHLC},
utils::{log, retry},
MapChunkId,
};

3
parser/src/price/kraken.rs
View File

@@ -4,8 +4,7 @@ use color_eyre::eyre::ContextCompat;
use serde_json::Value;
use crate::{
datasets::OHLC,
structs::{Date, Timestamp},
structs::{Date, Timestamp, OHLC},
utils::{log, retry},
};

28
parser/src/states/cohorts_states/address/split_by_address_cohort.rs
View File

@@ -243,3 +243,31 @@ where
self.p2tr += rhs.p2tr;
}
}
impl<T> SplitByAddressCohort<Option<T>> {
pub fn unwrap(self) -> SplitByAddressCohort<T> {
SplitByAddressCohort {
all: self.all.unwrap(),
illiquid: self.illiquid.unwrap(),
liquid: self.liquid.unwrap(),
highly_liquid: self.highly_liquid.unwrap(),
plankton: self.plankton.unwrap(),
shrimp: self.shrimp.unwrap(),
crab: self.crab.unwrap(),
fish: self.fish.unwrap(),
shark: self.shark.unwrap(),
whale: self.whale.unwrap(),
humpback: self.humpback.unwrap(),
megalodon: self.megalodon.unwrap(),
p2pk: self.p2pk.unwrap(),
p2pkh: self.p2pkh.unwrap(),
p2sh: self.p2sh.unwrap(),
p2wpkh: self.p2wpkh.unwrap(),
p2wsh: self.p2wsh.unwrap(),
p2tr: self.p2tr.unwrap(),
}
}
}

49
parser/src/states/cohorts_states/utxo/split_by_utxo_cohort.rs
View File

@@ -668,3 +668,52 @@ impl<T> SplitByUTXOCohort<T> {
]
}
}
impl<T> SplitByUTXOCohort<Option<T>> {
pub fn unwrap(self) -> SplitByUTXOCohort<T> {
SplitByUTXOCohort {
sth: self.sth.unwrap(),
lth: self.lth.unwrap(),
up_to_1d: self.up_to_1d.unwrap(),
up_to_1w: self.up_to_1w.unwrap(),
up_to_1m: self.up_to_1m.unwrap(),
up_to_2m: self.up_to_2m.unwrap(),
up_to_3m: self.up_to_3m.unwrap(),
up_to_4m: self.up_to_4m.unwrap(),
up_to_5m: self.up_to_5m.unwrap(),
up_to_6m: self.up_to_6m.unwrap(),
up_to_1y: self.up_to_1y.unwrap(),
up_to_2y: self.up_to_2y.unwrap(),
up_to_3y: self.up_to_3y.unwrap(),
up_to_5y: self.up_to_5y.unwrap(),
up_to_7y: self.up_to_7y.unwrap(),
up_to_10y: self.up_to_10y.unwrap(),
up_to_15y: self.up_to_15y.unwrap(),
from_1d_to_1w: self.from_1d_to_1w.unwrap(),
from_1w_to_1m: self.from_1w_to_1m.unwrap(),
from_1m_to_3m: self.from_1m_to_3m.unwrap(),
from_3m_to_6m: self.from_3m_to_6m.unwrap(),
from_6m_to_1y: self.from_6m_to_1y.unwrap(),
from_1y_to_2y: self.from_1y_to_2y.unwrap(),
from_2y_to_3y: self.from_2y_to_3y.unwrap(),
from_3y_to_5y: self.from_3y_to_5y.unwrap(),
from_5y_to_7y: self.from_5y_to_7y.unwrap(),
from_7y_to_10y: self.from_7y_to_10y.unwrap(),
from_10y_to_15y: self.from_10y_to_15y.unwrap(),
from_1y: self.from_1y.unwrap(),
from_2y: self.from_2y.unwrap(),
from_4y: self.from_4y.unwrap(),
from_10y: self.from_10y.unwrap(),
from_15y: self.from_15y.unwrap(),
epoch_1: self.epoch_1.unwrap(),
epoch_2: self.epoch_2.unwrap(),
epoch_3: self.epoch_3.unwrap(),
epoch_4: self.epoch_4.unwrap(),
epoch_5: self.epoch_5.unwrap(),
}
}
}

4
parser/src/structs/any_map.rs
View File

@@ -2,6 +2,8 @@ use std::path::{Path, PathBuf};
use serde_json::Value;
use super::MapKind;
pub trait AnyMap {
fn path(&self) -> &Path;
fn path_last(&self) -> &Option<PathBuf>;
@@ -25,4 +27,6 @@ pub trait AnyMap {
fn post_export(&mut self);
fn delete_files(&self);
fn kind(&self) -> MapKind;
}

22
parser/src/structs/bi_map.rs
View File

@@ -7,9 +7,11 @@ use allocative::Allocative;
use crate::utils::{LossyFrom, TARGET_BLOCKS_PER_DAY};
use super::{AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapValue};
use super::{
AnyDateMap, AnyHeightMap, AnyMap, Date, DateMap, Height, HeightMap, MapKind, MapValue,
};
#[derive(Default, Allocative)]
#[derive(Allocative)]
pub struct BiMap<Value>
where
Value: MapValue,
@@ -22,17 +24,17 @@ impl<Value> BiMap<Value>
where
Value: MapValue,
{
pub fn new_bin(version: u32, path: &str) -> Self {
pub fn new_bin(version: u32, kind: MapKind, path: &str) -> Self {
Self {
height: HeightMap::_new_bin(version, path, true),
date: DateMap::_new_bin(version, path, false),
height: HeightMap::_new_bin(version, kind, path, true),
date: DateMap::_new_bin(version, kind, path, false),
}
}
pub fn new_json(version: u32, path: &str) -> Self {
pub fn new_json(version: u32, kind: MapKind, path: &str) -> Self {
Self {
height: HeightMap::new_json(version, path, true),
date: DateMap::new_json(version, path, false),
height: HeightMap::new_json(version, kind, path, true),
date: DateMap::new_json(version, kind, path, false),
}
}
@@ -292,6 +294,10 @@ where
self.height.multi_insert_max(heights, &mut source.height);
self.date.multi_insert_max(dates, &mut source.date);
}
pub fn kind(&self) -> MapKind {
self.date.kind()
}
}
pub trait AnyBiMap {

25
parser/src/structs/date_map.rs
View File

@@ -17,7 +17,7 @@ where
last_height: &mut DateMap<Height>,
) {
dates.iter().for_each(|date| {
self.insert(
self.insert_computed(
*date,
source
.get_or_import(&last_height.get_or_import(date).unwrap())
@@ -40,7 +40,7 @@ where
let last_height = last_height.get_or_import(date).unwrap();
let range = (*first_height)..=(*last_height);
self.insert(*date, height_map.sum_range(&range));
self.insert_computed(*date, height_map.sum_range(&range));
})
}
}
@@ -55,27 +55,6 @@ pub trait AnyDateMap: AnyMap {
fn as_any_mut_map(&mut self) -> &mut dyn AnyMap;
}
#[inline(always)]
pub fn date_map_vec_to_any_date_map_vec<T>(
vec: Vec<&DateMap<T>>,
) -> impl Iterator<Item = &(dyn AnyDateMap + Send + Sync)>
where
T: MapValue,
{
vec.into_iter()
.map(|map| map as &(dyn AnyDateMap + Send + Sync))
}
#[inline(always)]
pub fn date_map_vec_to_mut_any_date_map_vec<T>(
vec: Vec<&mut DateMap<T>>,
) -> impl Iterator<Item = &mut (dyn AnyDateMap)>
where
T: MapValue,
{
vec.into_iter().map(|map| map as &mut dyn AnyDateMap)
}
impl<T> AnyDateMap for DateMap<T>
where
T: MapValue,

81
parser/src/structs/generic_map.rs
View File

@@ -22,6 +22,12 @@ use crate::{
use super::{AnyMap, MapValue};
#[derive(Debug, Clone, Copy, Allocative, PartialEq, Eq)]
pub enum MapKind {
Inserted,
Computed,
}
pub trait MapKey<ChunkId>
where
Self: Sized + PartialOrd + Ord + Clone + Copy + Debug,
@@ -68,9 +74,10 @@ where
fn from_usize(id: usize) -> Self;
}
#[derive(Default, Debug, Allocative)]
#[derive(Debug, Allocative)]
pub struct GenericMap<Key, Value, ChunkId, Serialized> {
version: u32,
kind: MapKind,
path_all: PathBuf,
path_last: Option<PathBuf>,
@@ -93,20 +100,28 @@ where
Key: MapKey<ChunkId>,
Serialized: MapSerialized<Key, Value, ChunkId>,
{
pub fn new_bin(version: u32, path: &str) -> Self {
Self::new(version, path, Serialization::Binary, 1, true)
pub fn new_bin(version: u32, kind: MapKind, path: &str) -> Self {
Self::new(version, kind, path, Serialization::Binary, 1, true)
}
pub fn _new_bin(version: u32, path: &str, export_last: bool) -> Self {
Self::new(version, path, Serialization::Binary, 1, export_last)
pub fn _new_bin(version: u32, kind: MapKind, path: &str, export_last: bool) -> Self {
Self::new(version, kind, path, Serialization::Binary, 1, export_last)
}
pub fn new_json(version: u32, path: &str, export_last: bool) -> Self {
Self::new(version, path, Serialization::Json, usize::MAX, export_last)
pub fn new_json(version: u32, kind: MapKind, path: &str, export_last: bool) -> Self {
Self::new(
version,
kind,
path,
Serialization::Json,
usize::MAX,
export_last,
)
}
fn new(
version: u32,
kind: MapKind,
path: &str,
serialization: Serialization,
chunks_in_memory: usize,
@@ -132,6 +147,7 @@ where
let mut s = Self {
version,
kind,
path_all,
path_last,
@@ -202,6 +218,19 @@ where
}
pub fn insert(&mut self, key: Key, value: Value) -> Value {
self.checked_insert(key, value, MapKind::Inserted)
}
pub fn insert_computed(&mut self, key: Key, value: Value) -> Value {
self.checked_insert(key, value, MapKind::Computed)
}
fn checked_insert(&mut self, key: Key, value: Value, kind: MapKind) -> Value {
if self.kind != kind {
dbg!(&self.path());
panic!("Called at the wrong place");
}
if !self.is_key_safe(key) {
self.to_insert
.entry(key.to_chunk_id())
@@ -376,6 +405,10 @@ where
.iter()
.for_each(|(_, path)| fs::remove_file(path).unwrap())
}
fn kind(&self) -> MapKind {
self.kind
}
}
impl<Key, Value, ChunkId, Serialized> GenericMap<Key, Value, ChunkId, Serialized>
@@ -407,13 +440,13 @@ where
F: FnMut(&Key) -> Value,
{
keys.iter().for_each(|key| {
self.insert(*key, callback(key));
self.insert_computed(*key, callback(key));
});
}
pub fn multi_insert_const(&mut self, keys: &[Key], constant: Value) {
keys.iter().for_each(|key| {
self.insert(*key, constant);
self.insert_computed(*key, constant);
});
}
@@ -428,7 +461,7 @@ where
F: FnMut(SourceValue, &Key) -> Value,
{
keys.iter().for_each(|key| {
self.insert(*key, transform(source.get_or_import(key).unwrap(), key));
self.insert_computed(*key, transform(source.get_or_import(key).unwrap(), key));
});
}
@@ -452,7 +485,7 @@ where
keys.iter().for_each(|key| {
let value = transform((source.get_or_import(key).unwrap(), key, source, self));
self.insert(*key, value);
self.insert_computed(*key, value);
});
}
@@ -469,7 +502,7 @@ where
Value: LossyFrom<A> + LossyFrom<B> + Add<Output = Value>,
{
keys.iter().for_each(|key| {
self.insert(
self.insert_computed(
*key,
Value::lossy_from(added.get_or_import(key).unwrap())
+ Value::lossy_from(adder.get_or_import(key).unwrap()),
@@ -490,7 +523,7 @@ where
Value: LossyFrom<A> + LossyFrom<B> + Sub<Output = Value>,
{
keys.iter().for_each(|key| {
self.insert(
self.insert_computed(
*key,
Value::lossy_from(subtracted.get_or_import(key).unwrap())
- Value::lossy_from(subtracter.get_or_import(key).unwrap()),
@@ -511,7 +544,7 @@ where
Value: LossyFrom<A> + LossyFrom<B> + Mul<Output = Value>,
{
keys.iter().for_each(|key| {
self.insert(
self.insert_computed(
*key,
Value::lossy_from(multiplied.get_or_import(key).unwrap())
* Value::lossy_from(multiplier.get_or_import(key).unwrap()),
@@ -565,7 +598,7 @@ where
let multiplier = Value::from(if as_percentage { 100 } else { 1 });
keys.iter().for_each(|key| {
self.insert(
self.insert_computed(
*key,
Value::lossy_from(divided.get_or_import(key).unwrap())
/ Value::lossy_from(divider.get_or_import(key).unwrap())
@@ -634,7 +667,7 @@ where
previous_sum + Value::lossy_from(last_value) - Value::lossy_from(to_subtract),
);
self.insert(*key, sum.unwrap());
self.insert_computed(*key, sum.unwrap());
});
}
@@ -677,7 +710,7 @@ where
average.replace(((previous_average * (len - 1.0) + last_value) / len).into());
self.insert(*key, average.unwrap());
self.insert_computed(*key, average.unwrap());
});
}
@@ -695,7 +728,7 @@ where
let net_change = last_value - previous_value;
self.insert(*key, net_change);
self.insert_computed(*key, net_change);
});
}
@@ -718,7 +751,7 @@ where
let percentage_change = ((last_value / previous_value) - one) * hundred;
self.insert(*key, Value::lossy_from(percentage_change));
self.insert_computed(*key, Value::lossy_from(percentage_change));
});
}
@@ -753,7 +786,7 @@ where
keys.iter().cloned().try_for_each(|key| {
if key < min_percentile_key {
map_and_percentiles.iter_mut().for_each(|(map, _)| {
(*map).insert(key, nan);
(*map).insert_computed(key, nan);
});
return ControlFlow::Continue::<()>(());
}
@@ -818,11 +851,11 @@ where
let float_value = get_percentile::<OrderedFloat<f32>>(vec, *percentile).0;
(*map).insert(key, Value::lossy_from(float_value));
(*map).insert_computed(key, Value::lossy_from(float_value));
});
} else {
map_and_percentiles.iter_mut().for_each(|(map, _)| {
(*map).insert(key, nan);
(*map).insert_computed(key, nan);
});
}
@@ -854,7 +887,7 @@ where
previous_max.replace(last_value);
}
self.insert(*key, previous_max.unwrap());
self.insert_computed(*key, previous_max.unwrap());
});
}
@@ -888,7 +921,7 @@ where
previous_min.replace(last_value);
}
self.insert(*key, previous_min.unwrap_or_default());
self.insert_computed(*key, previous_min.unwrap_or_default());
});
}
}

4
parser/src/structs/map_value.rs
View File

@@ -4,9 +4,7 @@ use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{de::DeserializeOwned, Serialize};
use crate::datasets::OHLC;
use super::{Date, Height, Timestamp};
use super::{Date, Height, Timestamp, OHLC};
pub trait MapValue:
Clone

2
parser/src/structs/mod.rs
View File

@@ -25,6 +25,7 @@ mod height_map;
mod height_map_chunk_id;
mod liquidity;
mod map_value;
mod ohlc;
mod partial_txout_data;
mod price;
mod sent_data;
@@ -61,6 +62,7 @@ pub use height_map::*;
pub use height_map_chunk_id::*;
pub use liquidity::*;
pub use map_value::*;
pub use ohlc::*;
pub use partial_txout_data::*;
pub use price::*;
pub use sent_data::*;

0
parser/src/datasets/price/ohlc.rs → parser/src/structs/ohlc.rs
View File