brk/_src/parser/datasets/mod.rs

use std::{collections::BTreeMap, ops::RangeInclusive};

use allocative::Allocative;

use itertools::Itertools;

use log::info;
use rayon::prelude::*;

mod _traits;
mod address;
mod block_metadata;
mod coindays;
mod cointime;
mod constant;
mod date_metadata;
mod mining;
mod price;
mod subs;
mod transaction;
mod utxo;

pub use _traits::*;
pub use address::*;
pub use block_metadata::*;
pub use coindays::*;
pub use cointime::*;
pub use constant::*;
pub use date_metadata::*;
pub use mining::*;
pub use price::*;
use serde_json::Value;
pub use subs::*;
pub use transaction::*;
pub use utxo::*;

use crate::{
    io::Json,
    parser::{
        databases::Databases,
        states::{
            AddressCohortsInputStates,
            AddressCohortsOneShotStates,
            AddressCohortsRealizedStates,
            States,
            UTXOCohortsOneShotStates,
            // UTXOCohortsReceivedStates,
            UTXOCohortsSentStates,
        },
    },
    structs::{Amount, Config, Date, Height, Price, Timestamp},
};

pub struct InsertData<'a> {
    pub address_cohorts_input_states: &'a Option<AddressCohortsInputStates>,
    pub address_cohorts_one_shot_states: &'a Option<AddressCohortsOneShotStates>,
    pub address_cohorts_realized_states: &'a Option<AddressCohortsRealizedStates>,
    pub amount_sent: Amount,
    pub block_interval: Timestamp,
    pub block_price: Price,
    pub block_size: usize,
    pub block_vbytes: u64,
    pub block_weight: u64,
    pub coinbase: Amount,
    pub compute_addresses: bool,
    pub databases: &'a Databases,
    pub date: Date,
    pub date_blocks_range: &'a RangeInclusive<u32>,
    pub date_first_height: Height,
    pub difficulty: f64,
    pub fees: &'a Vec<Amount>,
    pub height: Height,
    pub is_date_last_block: bool,
    pub satblocks_destroyed: Amount,
    pub satdays_destroyed: Amount,
    pub states: &'a States,
    pub timestamp: Timestamp,
    pub transaction_count: usize,
    pub utxo_cohorts_one_shot_states: &'a UTXOCohortsOneShotStates,
    // pub utxo_cohorts_received_states: &'a UTXOCohortsReceivedStates,
    pub utxo_cohorts_sent_states: &'a UTXOCohortsSentStates,
}

pub struct ComputeData<'a> {
    pub heights: &'a [Height],
    pub dates: &'a [Date],
}

#[derive(Allocative)]
pub struct Datasets {
    min_initial_states: MinInitialStates,

    pub constant: ConstantDataset,
    pub address: AddressDatasets,
    pub block_metadata: BlockMetadataDataset,
    pub coindays: CoindaysDataset,
    pub cointime: CointimeDataset,
    pub date_metadata: DateMetadataDataset,
    pub mining: MiningDataset,
    pub price: PriceDatasets,
    pub transaction: TransactionDataset,
    pub utxo: UTXODatasets,
}

impl Datasets {
    pub fn import(config: &Config) -> color_eyre::Result<Self> {
        let price = PriceDatasets::import(config)?;

        let constant = ConstantDataset::import(config)?;

        let date_metadata = DateMetadataDataset::import(config)?;

        let cointime = CointimeDataset::import(config)?;

        let coindays = CoindaysDataset::import(config)?;

        let mining = MiningDataset::import(config)?;

        let block_metadata = BlockMetadataDataset::import(config)?;

        let transaction = TransactionDataset::import(config)?;

        let address = AddressDatasets::import(config)?;

        let utxo = UTXODatasets::import(config)?;

        let mut s = Self {
            min_initial_states: MinInitialStates::default(),

            address,
            block_metadata,
            cointime,
            coindays,
            constant,
            date_metadata,
            price,
            mining,
            transaction,
            utxo,
        };

        s.set_initial_states(config);

        info!("Imported datasets");

        Ok(s)
    }

    fn set_initial_states(&mut self, config: &Config) {
        self.min_initial_states
            .consume(MinInitialStates::compute_from_datasets(self, config));
    }

    pub fn insert(&mut self, insert_data: InsertData) {
        if insert_data.compute_addresses {
            self.address.insert(&insert_data);
        }

        self.utxo.insert(&insert_data);

        if self
            .block_metadata
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.block_metadata.insert(&insert_data);
        }

        if self
            .date_metadata
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.date_metadata.insert(&insert_data);
        }

        if self
            .coindays
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.coindays.insert(&insert_data);
        }

        if self
            .mining
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.mining.insert(&insert_data);
        }

        if self
            .transaction
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.transaction.insert(&insert_data);
        }

        if self
            .cointime
            .needs_insert(insert_data.height, insert_data.date)
        {
            self.cointime.insert(&insert_data);
        }
    }

    pub fn compute(&mut self, compute_data: ComputeData) {
        if self.constant.should_compute(&compute_data) {
            self.constant.compute(&compute_data);
        }

        if self.mining.should_compute(&compute_data) {
            self.mining.compute(
                &compute_data,
                &mut self.date_metadata.first_height,
                &mut self.date_metadata.last_height,
            );
        }

        // No compute needed for now
        self.price
            .compute(&compute_data, &mut self.mining.cumulative_subsidy);

        self.address.compute(
            &compute_data,
            &mut self.price.close,
            &mut self.mining.cumulative_subsidy,
            &mut self.price.market_cap,
        );

        self.utxo.compute(
            &compute_data,
            &mut self.price.close,
            &mut self.mining.cumulative_subsidy,
            &mut self.price.market_cap,
        );

        if self.transaction.should_compute(&compute_data) {
            self.transaction.compute(
                &compute_data,
                &mut self.mining.cumulative_subsidy,
                &mut self.mining.block_interval,
            );
        }

        if self.cointime.should_compute(&compute_data) {
            self.cointime.compute(
                &compute_data,
                &mut self.date_metadata.first_height,
                &mut self.date_metadata.last_height,
                &mut self.price.close,
                &mut self.mining.cumulative_subsidy,
                &mut self.address.cohorts.all.subs.capitalization.realized_cap,
                &mut self.address.cohorts.all.subs.capitalization.realized_price,
                &mut self.mining.inflation_rate,
                &mut self.mining.yearly_inflation_rate,
                &mut self.transaction.annualized_volume,
                &mut self.mining.cumulative_subsidy_in_dollars,
            );
        }
    }

    pub fn export(&mut self, config: &Config, height: Height) -> color_eyre::Result<()> {
        let is_new = self
            .min_initial_states
            .min_last_height()
            .map_or(true, |last| last <= height);

        self.to_mut_any_dataset_vec()
            .into_iter()
            .for_each(|dataset| dataset.pre_export());

        self.to_any_dataset_vec()
            .into_par_iter()
            .try_for_each(|dataset| -> color_eyre::Result<()> { dataset.export() })?;

        let mut path_to_last: BTreeMap<String, Value> = BTreeMap::default();

        self.to_mut_any_dataset_vec()
            .into_iter()
            .for_each(|dataset| {
                dataset.post_export();

                if is_new {
                    dataset.to_all_map_vec().iter().for_each(|map| {
                        if map.path_last().is_some() {
                            if let Some(last_value) = map.last_value() {
                                path_to_last.insert(map.id(config), last_value);
                            }
                        }
                    });
                }
            });

        if is_new {
            Json::export(&config.path_datasets_last_values(), &path_to_last)?;
        }

        self.set_initial_states(config);

        Ok(())
    }
}

impl AnyDatasets for Datasets {
    fn get_min_initial_states(&self) -> &MinInitialStates {
        &self.min_initial_states
    }

    fn to_any_dataset_vec(&self) -> Vec<&(dyn AnyDataset + Send + Sync)> {
        vec![
            vec![
                &self.price as &(dyn AnyDataset + Send + Sync),
                &self.constant,
            ],
            self.address.to_any_dataset_vec(),
            self.utxo.to_any_dataset_vec(),
            vec![
                &self.mining,
                &self.transaction,
                &self.block_metadata,
                &self.date_metadata,
                &self.cointime,
                &self.coindays,
            ],
        ]
        .into_iter()
        .flatten()
        .collect_vec()
    }

    fn to_mut_any_dataset_vec(&mut self) -> Vec<&mut dyn AnyDataset> {
        vec![
            vec![&mut self.price as &mut dyn AnyDataset, &mut self.constant],
            self.address.to_mut_any_dataset_vec(),
            self.utxo.to_mut_any_dataset_vec(),
            vec![
                &mut self.mining,
                &mut self.transaction,
                &mut self.block_metadata,
                &mut self.date_metadata,
                &mut self.cointime,
                &mut self.coindays,
            ],
        ]
        .into_iter()
        .flatten()
        .collect_vec()
    }
}