brk: first commit

_src/parser/datasets/address/all_metadata.rs

@@ -0,0 +1,74 @@
+use allocative::Allocative;
+use struct_iterable::Iterable;
+
+use crate::{
+    parser::datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates},
+    structs::{BiMap, Config, MapKind, MapPath},
+};
+
+#[derive(Allocative, Iterable)]
+pub struct AllAddressesMetadataDataset {
+    min_initial_states: MinInitialStates,
+    created_addreses: BiMap<u32>,
+    empty_addresses: BiMap<u32>,
+    new_addresses: BiMap<u32>,
+}
+
+impl AllAddressesMetadataDataset {
+    pub fn import(path: &MapPath, config: &Config) -> color_eyre::Result<Self> {
+        let f = |s: &str| path.join(s);
+
+        let mut s = Self {
+            min_initial_states: MinInitialStates::default(),
+
+            // Inserted
+            created_addreses: BiMap::new_bin(1, MapKind::Inserted, &f("created_addresses")),
+            empty_addresses: BiMap::new_bin(1, MapKind::Inserted, &f("empty_addresses")),
+
+            // Computed
+            new_addresses: BiMap::new_bin(1, MapKind::Computed, &f("new_addresses")),
+        };
+
+        s.min_initial_states
+            .consume(MinInitialStates::compute_from_dataset(&s, config));
+
+        Ok(s)
+    }
+
+    pub fn insert(&mut self, insert_data: &InsertData) {
+        let &InsertData {
+            databases,
+            height,
+            date,
+            is_date_last_block,
+            ..
+        } = insert_data;
+
+        let created_addresses = self
+            .created_addreses
+            .height
+            .insert(height, *databases.address_to_address_index.metadata.len);
+
+        let empty_addresses = self.empty_addresses.height.insert(
+            height,
+            *databases.address_index_to_empty_address_data.metadata.len,
+        );
+
+        if is_date_last_block {
+            self.created_addreses.date.insert(date, created_addresses);
+
+            self.empty_addresses.date.insert(date, empty_addresses);
+        }
+    }
+
+    pub fn compute(&mut self, &ComputeData { heights, dates, .. }: &ComputeData) {
+        self.new_addresses
+            .multi_insert_net_change(heights, dates, &mut self.created_addreses, 1)
+    }
+}
+
+impl AnyDataset for AllAddressesMetadataDataset {
+    fn get_min_initial_states(&self) -> &MinInitialStates {
+        &self.min_initial_states
+    }
+}

_src/parser/datasets/address/cohort.rs

@@ -0,0 +1,381 @@
+use allocative::Allocative;
+
+use struct_iterable::Iterable;
+
+use crate::{
+    parser::{
+        datasets::{AnyDataset, ComputeData, InsertData, MinInitialStates, SubDataset},
+        states::{AddressCohortId, DurableStates},
+    },
+    structs::{AddressSplit, BiMap, Config, Date, Height, MapPath},
+};
+
+use super::cohort_metadata::AddressCohortMetadataDataset;
+
+#[derive(Allocative, Iterable)]
+pub struct CohortDataset {
+    min_initial_states: MinInitialStates,
+
+    split: AddressSplit,
+
+    metadata: AddressCohortMetadataDataset,
+
+    pub subs: SubDataset,
+}
+
+impl CohortDataset {
+    pub fn import(
+        path: &MapPath,
+        id: AddressCohortId,
+        config: &Config,
+    ) -> color_eyre::Result<Self> {
+        let name = id.as_name().map(|s| s.to_owned());
+        let split = id.as_split();
+
+        let mut s = Self {
+            min_initial_states: MinInitialStates::default(),
+            split,
+            metadata: AddressCohortMetadataDataset::import(path, &name, config)?,
+            subs: SubDataset::import(path, &name, config)?,
+        };
+
+        s.min_initial_states
+            .consume(MinInitialStates::compute_from_dataset(&s, config));
+
+        Ok(s)
+    }
+
+    pub fn sub_datasets_vec(&self) -> Vec<&SubDataset> {
+        vec![&self.subs]
+    }
+
+    pub fn needs_insert_metadata(&self, height: Height, date: Date) -> bool {
+        self.metadata.needs_insert(height, date)
+    }
+
+    pub fn needs_insert_utxo(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.utxo.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_capitalization(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.capitalization.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_supply(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.supply.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_price_paid(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.price_paid.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_realized(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.realized.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_unrealized(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.unrealized.needs_insert(height, date))
+    }
+
+    pub fn needs_insert_input(&self, height: Height, date: Date) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.input.needs_insert(height, date))
+    }
+
+    // fn needs_insert_output(&self, insert_data: &InsertData) -> bool {
+    //     self.sub_datasets_vec()
+    //         .iter()
+    //         .any(|sub| sub.output.needs_insert(height, date))
+    // }
+
+    pub fn insert_realized_data(&mut self, insert_data: &InsertData) {
+        let realized_state = insert_data
+            .address_cohorts_realized_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split)
+            .unwrap();
+
+        self.subs.realized.insert(insert_data, realized_state);
+    }
+
+    fn insert_metadata(&mut self, insert_data: &InsertData) {
+        let address_count = insert_data
+            .states
+            .address_cohorts_durable_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split)
+            .unwrap()
+            .address_count;
+
+        self.metadata.insert(insert_data, address_count);
+    }
+
+    fn insert_supply_data(&mut self, insert_data: &InsertData, durable_states: &DurableStates) {
+        self.subs
+            .supply
+            .insert(insert_data, &durable_states.supply_state);
+    }
+
+    fn insert_utxo_data(&mut self, insert_data: &InsertData, durable_states: &DurableStates) {
+        self.subs
+            .utxo
+            .insert(insert_data, &durable_states.utxo_state);
+    }
+
+    fn insert_capitalization_data(
+        &mut self,
+        insert_data: &InsertData,
+        durable_states: &DurableStates,
+    ) {
+        self.subs
+            .capitalization
+            .insert(insert_data, &durable_states.capitalization_state);
+    }
+
+    fn insert_unrealized_data(&mut self, insert_data: &InsertData) {
+        let states = insert_data
+            .address_cohorts_one_shot_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split)
+            .unwrap();
+
+        self.subs.unrealized.insert(
+            insert_data,
+            &states.unrealized_block_state,
+            &states.unrealized_date_state,
+        );
+    }
+
+    fn insert_price_paid_data(&mut self, insert_data: &InsertData) {
+        let states = insert_data
+            .address_cohorts_one_shot_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split)
+            .unwrap();
+
+        self.subs
+            .price_paid
+            .insert(insert_data, &states.price_paid_state);
+    }
+
+    fn insert_input_data(&mut self, insert_data: &InsertData) {
+        let state = insert_data
+            .address_cohorts_input_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split)
+            .unwrap();
+
+        self.subs.input.insert(insert_data, state);
+    }
+
+    // fn insert_output_data(&mut self, insert_data: &InsertData) {
+    //     let state = insert_data
+    //         .address_cohorts_output_states
+    //         .as_ref()
+    //         .unwrap()
+    //         .get(&self.split)
+    //         .unwrap();
+
+    //     self.output.insert(insert_data, &state.all);
+    //     self.illiquid.output.insert(insert_data, &state.illiquid);
+    //     self.liquid.output.insert(insert_data, &state.liquid);
+    //     self.highly_liquid
+    //         .output
+    //         .insert(insert_data, &state.highly_liquid);
+    // }
+
+    pub fn insert(&mut self, insert_data: &InsertData) {
+        if !insert_data.compute_addresses {
+            return;
+        }
+
+        let address_cohort_durable_states = insert_data
+            .states
+            .address_cohorts_durable_states
+            .as_ref()
+            .unwrap()
+            .get(&self.split);
+
+        if address_cohort_durable_states.is_none() {
+            return; // TODO: Check if should panic instead
+        }
+
+        let address_cohort_durable_states = address_cohort_durable_states.unwrap();
+
+        if self.needs_insert_metadata(insert_data.height, insert_data.date) {
+            self.insert_metadata(insert_data);
+        }
+
+        if self.needs_insert_utxo(insert_data.height, insert_data.date) {
+            self.insert_utxo_data(insert_data, &address_cohort_durable_states.durable_states);
+        }
+
+        if self.needs_insert_capitalization(insert_data.height, insert_data.date) {
+            self.insert_capitalization_data(
+                insert_data,
+                &address_cohort_durable_states.durable_states,
+            );
+        }
+
+        if self.needs_insert_supply(insert_data.height, insert_data.date) {
+            self.insert_supply_data(insert_data, &address_cohort_durable_states.durable_states);
+        }
+
+        if self.needs_insert_realized(insert_data.height, insert_data.date) {
+            self.insert_realized_data(insert_data);
+        }
+
+        if self.needs_insert_unrealized(insert_data.height, insert_data.date) {
+            self.insert_unrealized_data(insert_data);
+        }
+
+        if self.needs_insert_price_paid(insert_data.height, insert_data.date) {
+            self.insert_price_paid_data(insert_data);
+        }
+
+        if self.needs_insert_input(insert_data.height, insert_data.date) {
+            self.insert_input_data(insert_data);
+        }
+
+        // if self.needs_insert_output(insert_data) {
+        //     self.insert_output_data(insert_data);
+        // }
+    }
+
+    // pub fn should_compute_metadata(&self, compute_data: &ComputeData) -> bool {
+    //     self.metadata.should_compute(compute_data)
+    // }
+
+    // pub fn should_compute_utxo(&self, compute_data: &ComputeData) -> bool {
+    //     self.sub_datasets_vec()
+    //         .iter()
+    //         .any(|sub| sub.utxo.should_compute(compute_data))
+    // }
+
+    pub fn should_compute_supply(&self, compute_data: &ComputeData) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.supply.should_compute(compute_data))
+    }
+
+    pub fn should_compute_capitalization(&self, compute_data: &ComputeData) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.capitalization.should_compute(compute_data))
+    }
+
+    fn should_compute_realized(&self, compute_data: &ComputeData) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.realized.should_compute(compute_data))
+    }
+
+    fn should_compute_unrealized(&self, compute_data: &ComputeData) -> bool {
+        self.sub_datasets_vec()
+            .iter()
+            .any(|sub| sub.unrealized.should_compute(compute_data))
+    }
+
+    // fn should_compute_input(&self, compute_data: &ComputeData) -> bool {
+    //     self.sub_datasets_vec()
+    //         .iter()
+    //         .any(|sub| sub.input.should_compute(compute_data))
+    // }
+
+    // fn should_compute_output(&self, compute_data: &ComputeData) -> bool {
+    //     self.sub_datasets_vec()
+    //         .iter()
+    //         .any(|sub| sub.output.should_compute(compute_data))
+    // }
+
+    fn compute_supply_data(
+        &mut self,
+        compute_data: &ComputeData,
+        circulating_supply: &mut BiMap<f64>,
+    ) {
+        self.subs.supply.compute(compute_data, circulating_supply);
+    }
+
+    fn compute_unrealized_data(
+        &mut self,
+        compute_data: &ComputeData,
+        circulating_supply: &mut BiMap<f64>,
+        market_cap: &mut BiMap<f32>,
+    ) {
+        self.subs.unrealized.compute(
+            compute_data,
+            &mut self.subs.supply.supply,
+            circulating_supply,
+            market_cap,
+        );
+    }
+
+    fn compute_realized_data(&mut self, compute_data: &ComputeData, market_cap: &mut BiMap<f32>) {
+        self.subs.realized.compute(compute_data, market_cap);
+    }
+
+    fn compute_capitalization_data(&mut self, compute_data: &ComputeData, closes: &mut BiMap<f32>) {
+        self.subs
+            .capitalization
+            .compute(compute_data, closes, &mut self.subs.supply.supply);
+    }
+
+    // fn compute_output_data(&mut self, compute_data: &ComputeData) {
+    //     self.all
+    //         .output
+    //         .compute(compute_data, &mut self.supply.total);
+    // }
+
+    pub fn compute(
+        &mut self,
+        compute_data: &ComputeData,
+        closes: &mut BiMap<f32>,
+        circulating_supply: &mut BiMap<f64>,
+        market_cap: &mut BiMap<f32>,
+    ) {
+        if self.should_compute_supply(compute_data) {
+            self.compute_supply_data(compute_data, circulating_supply);
+        }
+
+        if self.should_compute_unrealized(compute_data) {
+            self.compute_unrealized_data(compute_data, circulating_supply, market_cap);
+        }
+
+        if self.should_compute_realized(compute_data) {
+            self.compute_realized_data(compute_data, market_cap);
+        }
+
+        // MUST BE after compute_supply
+        if self.should_compute_capitalization(compute_data) {
+            self.compute_capitalization_data(compute_data, closes);
+        }
+
+        // if self.should_compute_output(compute_data) {
+        //     self.compute_output_data(compute_data);
+        // }
+    }
+}
+
+impl AnyDataset for CohortDataset {
+    fn get_min_initial_states(&self) -> &MinInitialStates {
+        &self.min_initial_states
+    }
+}

_src/parser/datasets/address/cohort_metadata.rs

@@ -0,0 +1,70 @@
+use allocative::Allocative;
+use struct_iterable::Iterable;
+
+use crate::{
+    parser::datasets::{AnyDataset, InsertData, MinInitialStates},
+    structs::{BiMap, Config, MapKind, MapPath},
+};
+
+#[derive(Allocative, Iterable)]
+pub struct AddressCohortMetadataDataset {
+    min_initial_states: MinInitialStates,
+
+    address_count: BiMap<f64>,
+    // pub output: OutputSubDataset,
+    // Sending addresses
+    // Receiving addresses
+    // Active addresses (Unique(Sending + Receiving))
+}
+
+impl AddressCohortMetadataDataset {
+    pub fn import(
+        path: &MapPath,
+        name: &Option<String>,
+        config: &Config,
+    ) -> color_eyre::Result<Self> {
+        let f = |s: &str| {
+            if let Some(name) = name {
+                path.join(&format!("{name}/{s}"))
+            } else {
+                path.join(s)
+            }
+        };
+
+        let mut s = Self {
+            min_initial_states: MinInitialStates::default(),
+
+            // Inserted
+            address_count: BiMap::new_bin(1, MapKind::Inserted, &f("address_count")),
+            // output: OutputSubDataset::import(parent_path)?,
+        };
+
+        s.min_initial_states
+            .consume(MinInitialStates::compute_from_dataset(&s, config));
+
+        Ok(s)
+    }
+
+    pub fn insert(
+        &mut self,
+        &InsertData {
+            height,
+            date,
+            is_date_last_block,
+            ..
+        }: &InsertData,
+        address_count: f64,
+    ) {
+        self.address_count.height.insert(height, address_count);
+
+        if is_date_last_block {
+            self.address_count.date.insert(date, address_count);
+        }
+    }
+}
+
+impl AnyDataset for AddressCohortMetadataDataset {
+    fn get_min_initial_states(&self) -> &MinInitialStates {
+        &self.min_initial_states
+    }
+}

_src/parser/datasets/address/mod.rs

@@ -0,0 +1,176 @@
+mod all_metadata;
+mod cohort;
+pub mod cohort_metadata;
+
+use allocative::Allocative;
+use itertools::Itertools;
+use rayon::prelude::*;
+
+use crate::{
+    parser::states::SplitByAddressCohort,
+    structs::{BiMap, Config, Date, Height},
+};
+
+use self::{all_metadata::AllAddressesMetadataDataset, cohort::CohortDataset};
+
+use super::{AnyDataset, AnyDatasets, ComputeData, InsertData, MinInitialStates};
+
+#[derive(Allocative)]
+pub struct AddressDatasets {
+    min_initial_states: MinInitialStates,
+
+    metadata: AllAddressesMetadataDataset,
+
+    pub cohorts: SplitByAddressCohort<CohortDataset>,
+}
+
+impl AddressDatasets {
+    pub fn import(config: &Config) -> color_eyre::Result<Self> {
+        let mut cohorts = SplitByAddressCohort::<Option<CohortDataset>>::default();
+
+        let path_dataset = config.path_datasets();
+
+        cohorts
+            .as_vec()
+            .into_par_iter()
+            .map(|(_, id)| (id, CohortDataset::import(&path_dataset, id, config)))
+            .collect::<Vec<_>>()
+            .into_iter()
+            .try_for_each(|(id, dataset)| -> color_eyre::Result<()> {
+                cohorts.get_mut_from_id(&id).replace(dataset?);
+                Ok(())
+            })?;
+
+        let mut s = Self {
+            min_initial_states: MinInitialStates::default(),
+
+            metadata: AllAddressesMetadataDataset::import(&path_dataset, config)?,
+
+            cohorts: cohorts.unwrap(),
+        };
+
+        s.min_initial_states
+            .consume(MinInitialStates::compute_from_datasets(&s, config));
+
+        Ok(s)
+    }
+
+    pub fn insert(&mut self, insert_data: &InsertData) {
+        self.metadata.insert(insert_data);
+
+        self.cohorts
+            .as_mut_vec()
+            .into_iter()
+            .for_each(|(cohort, _)| cohort.insert(insert_data))
+    }
+
+    pub fn needs_durable_states(&self, height: Height, date: Date) -> bool {
+        let needs_insert_utxo = self.needs_insert_utxo(height, date);
+        let needs_insert_capitalization = self.needs_insert_capitalization(height, date);
+        let needs_insert_supply = self.needs_insert_supply(height, date);
+        let needs_one_shot_states = self.needs_one_shot_states(height, date);
+
+        needs_insert_utxo
+            || needs_insert_capitalization
+            || needs_insert_supply
+            || needs_one_shot_states
+    }
+
+    pub fn needs_one_shot_states(&self, height: Height, date: Date) -> bool {
+        self.needs_insert_price_paid(height, date) || self.needs_insert_unrealized(height, date)
+    }
+
+    // pub fn needs_sent_states(&self, height: Height, date: WNaiveDate) -> bool {
+    //     self.needs_insert_input(height, date) || self.needs_insert_realized(height, date)
+    // }
+
+    pub fn needs_insert_utxo(&self, height: Height, date: Date) -> bool {
+        self.cohorts
+            .as_vec()
+            .iter()
+            .any(|(dataset, _)| dataset.needs_insert_utxo(height, date))
+    }
+
+    pub fn needs_insert_capitalization(&self, height: Height, date: Date) -> bool {
+        self.cohorts
+            .as_vec()
+            .iter()
+            .any(|(dataset, _)| dataset.needs_insert_capitalization(height, date))
+    }
+
+    pub fn needs_insert_supply(&self, height: Height, date: Date) -> bool {
+        self.cohorts
+            .as_vec()
+            .iter()
+            .any(|(dataset, _)| dataset.needs_insert_supply(height, date))
+    }
+
+    pub fn needs_insert_price_paid(&self, height: Height, date: Date) -> bool {
+        self.cohorts
+            .as_vec()
+            .iter()
+            .any(|(dataset, _)| dataset.needs_insert_price_paid(height, date))
+    }
+
+    // pub fn needs_insert_realized(&self, height: Height, date: WNaiveDate) -> bool {
+    //     self.cohorts
+    //         .as_vec()
+    //         .iter()
+    //         .any(|(dataset, _)| dataset.needs_insert_realized(height, date))
+    // }
+
+    pub fn needs_insert_unrealized(&self, height: Height, date: Date) -> bool {
+        self.cohorts
+            .as_vec()
+            .iter()
+            .any(|(dataset, _)| dataset.needs_insert_unrealized(height, date))
+    }
+
+    // pub fn needs_insert_input(&self, height: Height, date: WNaiveDate) -> bool {
+    //     self.cohorts
+    //         .as_vec()
+    //         .iter()
+    //         .any(|(dataset, _)| dataset.needs_insert_input(height, date))
+    // }
+
+    pub fn compute(
+        &mut self,
+        compute_data: &ComputeData,
+        closes: &mut BiMap<f32>,
+        circulating_supply: &mut BiMap<f64>,
+        market_cap: &mut BiMap<f32>,
+    ) {
+        self.metadata.compute(compute_data);
+
+        self.cohorts
+            .as_mut_vec()
+            .into_iter()
+            .for_each(|(cohort, _)| {
+                cohort.compute(compute_data, closes, circulating_supply, market_cap)
+            })
+    }
+}
+
+impl AnyDatasets for AddressDatasets {
+    fn get_min_initial_states(&self) -> &MinInitialStates {
+        &self.min_initial_states
+    }
+
+    fn to_any_dataset_vec(&self) -> Vec<&(dyn AnyDataset + Send + Sync)> {
+        self.cohorts
+            .as_vec()
+            .into_iter()
+            .map(|(d, _)| d as &(dyn AnyDataset + Send + Sync))
+            .chain(vec![&self.metadata as &(dyn AnyDataset + Send + Sync)])
+            .collect_vec()
+    }
+
+    fn to_mut_any_dataset_vec(&mut self) -> Vec<&mut dyn AnyDataset> {
+        self.cohorts
+            .as_mut_vec()
+            .into_iter()
+            .map(|(d, _)| d as &mut dyn AnyDataset)
+            .chain(vec![&mut self.metadata as &mut dyn AnyDataset])
+            .collect_vec()
+    }
+}