Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-27 16:19:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: removed liquidity split for everything but all addresses

Browse Source
This commit is contained in:
k
2024-10-03 17:38:43 +02:00
parent 669205aa4d
commit 5308796bac
38 changed files with 665 additions and 1074 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
parser/src/actions/parse.rs
View File

@@ -126,7 +126,7 @@ pub fn parse(
(mut txid_to_tx_data, mut txout_index_to_amount_and_address_index),
) = thread::scope(|scope| {
let output_handle = scope.spawn(|| {
let mut txouts_parsing_results = pre_process_outputs(
let mut txouts_parsing_results = prepare_outputs(
&block,
compute_addresses,
&mut states.address_counters.multisig_addresses,
@@ -144,7 +144,7 @@ pub fn parse(
});
let input_handle = scope.spawn(|| {
pre_process_inputs(
prepare_inputs(
&block,
&mut databases.txid_to_tx_data,
&mut databases.txout_index_to_amount,
@@ -780,7 +780,7 @@ pub struct TxoutsParsingResults {
}
#[allow(clippy::too_many_arguments)]
fn pre_process_outputs(
fn prepare_outputs(
block: &Block,
compute_addresses: bool,
multisig_addresses: &mut Counter,
@@ -866,7 +866,7 @@ fn pre_process_outputs(
}
#[allow(clippy::type_complexity)]
fn pre_process_inputs<'a>(
fn prepare_inputs<'a>(
block: &'a Block,
txid_to_tx_data_db: &mut TxidToTxData,
txout_index_to_amount_db: &mut TxoutIndexToAmount,

19
parser/src/databases/address_index_to_address_data.rs
View File

@@ -8,6 +8,7 @@ use allocative::Allocative;
use rayon::prelude::*;
use crate::{
states::AddressCohortsDurableStates,
structs::{AddressData, Date, Height},
utils::time,
};
@@ -82,18 +83,20 @@ impl AddressIndexToAddressData {
})
}
pub fn iter<F>(&mut self, callback: &mut F)
where
F: FnMut((&Key, &Value)),
{
pub fn compute_addres_cohorts_durable_states(&mut self) -> AddressCohortsDurableStates {
time("Iter through address_index_to_address_data", || {
self.open_all();
// MUST CLEAR MAP, otherwise some weird shit in happening later in the export I think
// MUST CLEAR MAP, otherwise some weird things are happening later in the export I think
mem::take(&mut self.map)
.values()
.for_each(|database| database.iter(callback));
});
.par_iter()
.map(|(_, database)| {
let mut s = AddressCohortsDurableStates::default();
database.iter(&mut |(_, a)| s.increment(a).unwrap());
s
})
.sum()
})
}
fn open_all(&mut self) {

1
parser/src/datasets/address/all_metadata.rs
View File

@@ -24,7 +24,6 @@ impl AllAddressesMetadataDataset {
let mut s = Self {
min_initial_states: MinInitialStates::default(),
// TODO: Shouldn't be (like many others)
created_addreses: BiMap::new_bin(1, &f("created_addresses")),
empty_addresses: BiMap::new_bin(1, &f("empty_addresses")),
new_addresses: BiMap::new_bin(1, &f("new_addresses")),

323
parser/src/datasets/address/cohort.rs
View File

@@ -5,7 +5,7 @@ use crate::{
datasets::{
AnyDataset, AnyDatasetGroup, ComputeData, InsertData, MinInitialStates, SubDataset,
},
states::{AddressCohortDurableStates, AddressCohortId},
states::{AddressCohortId, DurableStates},
structs::{AddressSplit, AnyBiMap, AnyDateMap, AnyHeightMap, BiMap, Date, Height},
};
@@ -19,10 +19,7 @@ pub struct CohortDataset {
metadata: MetadataDataset,
pub all: SubDataset,
illiquid: SubDataset,
liquid: SubDataset,
highly_liquid: SubDataset,
pub subs: SubDataset,
}
impl CohortDataset {
@@ -30,24 +27,11 @@ impl CohortDataset {
let name = id.as_name().map(|s| s.to_owned());
let split = id.as_split();
let f = |s: &str| {
if let Some(name) = &name {
Some(format!("{s}/{name}"))
} else {
Some(s.to_owned())
}
};
let mut s = Self {
min_initial_states: MinInitialStates::default(),
split,
metadata: MetadataDataset::import(parent_path, &name)?,
all: SubDataset::import(parent_path, &name)?,
illiquid: SubDataset::import(parent_path, &f("illiquid"))?,
liquid: SubDataset::import(parent_path, &f("liquid"))?,
highly_liquid: SubDataset::import(parent_path, &f("highly_liquid"))?,
subs: SubDataset::import(parent_path, &name)?,
};
s.min_initial_states
@@ -57,7 +41,7 @@ impl CohortDataset {
}
pub fn sub_datasets_vec(&self) -> Vec<&SubDataset> {
vec![&self.all, &self.illiquid, &self.liquid, &self.highly_liquid]
vec![&self.subs]
}
pub fn needs_insert_metadata(&self, height: Height, date: Date) -> bool {
@@ -113,28 +97,14 @@ impl CohortDataset {
// }
pub fn insert_realized_data(&mut self, insert_data: &InsertData) {
let split_realized_state = insert_data
let realized_state = insert_data
.address_cohorts_realized_states
.as_ref()
.unwrap()
.get(&self.split)
.unwrap();
self.all
.realized
.insert(insert_data, &split_realized_state.all);
self.illiquid
.realized
.insert(insert_data, &split_realized_state.illiquid);
self.liquid
.realized
.insert(insert_data, &split_realized_state.liquid);
self.highly_liquid
.realized
.insert(insert_data, &split_realized_state.highly_liquid);
self.subs.realized.insert(insert_data, realized_state);
}
fn insert_metadata(&mut self, insert_data: &InsertData) {
@@ -150,109 +120,26 @@ impl CohortDataset {
self.metadata.insert(insert_data, address_count);
}
fn insert_supply_data(
&mut self,
insert_data: &InsertData,
liquidity_split_state: &AddressCohortDurableStates,
) {
self.all.supply.insert(
insert_data,
&liquidity_split_state.split_durable_states.all.supply_state,
);
self.illiquid.supply.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.illiquid
.supply_state,
);
self.liquid.supply.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.liquid
.supply_state,
);
self.highly_liquid.supply.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.highly_liquid
.supply_state,
);
fn insert_supply_data(&mut self, insert_data: &InsertData, durable_states: &DurableStates) {
self.subs
.supply
.insert(insert_data, &durable_states.supply_state);
}
fn insert_utxo_data(
&mut self,
insert_data: &InsertData,
liquidity_split_state: &AddressCohortDurableStates,
) {
self.all.utxo.insert(
insert_data,
&liquidity_split_state.split_durable_states.all.utxo_state,
);
self.illiquid.utxo.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.illiquid
.utxo_state,
);
self.liquid.utxo.insert(
insert_data,
&liquidity_split_state.split_durable_states.liquid.utxo_state,
);
self.highly_liquid.utxo.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.highly_liquid
.utxo_state,
);
fn insert_utxo_data(&mut self, insert_data: &InsertData, durable_states: &DurableStates) {
self.subs
.utxo
.insert(insert_data, &durable_states.utxo_state);
}
fn insert_capitalization_data(
&mut self,
insert_data: &InsertData,
liquidity_split_state: &AddressCohortDurableStates,
durable_states: &DurableStates,
) {
self.all.capitalization.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.all
.capitalization_state,
);
self.illiquid.capitalization.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.illiquid
.capitalization_state,
);
self.liquid.capitalization.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.liquid
.capitalization_state,
);
self.highly_liquid.capitalization.insert(
insert_data,
&liquidity_split_state
.split_durable_states
.highly_liquid
.capitalization_state,
);
self.subs
.capitalization
.insert(insert_data, &durable_states.capitalization_state);
}
fn insert_unrealized_data(&mut self, insert_data: &InsertData) {
@@ -263,28 +150,10 @@ impl CohortDataset {
.get(&self.split)
.unwrap();
self.all.unrealized.insert(
self.subs.unrealized.insert(
insert_data,
&states.all.unrealized_block_state,
&states.all.unrealized_date_state,
);
self.illiquid.unrealized.insert(
insert_data,
&states.illiquid.unrealized_block_state,
&states.illiquid.unrealized_date_state,
);
self.liquid.unrealized.insert(
insert_data,
&states.liquid.unrealized_block_state,
&states.liquid.unrealized_date_state,
);
self.highly_liquid.unrealized.insert(
insert_data,
&states.highly_liquid.unrealized_block_state,
&states.highly_liquid.unrealized_date_state,
&states.unrealized_block_state,
&states.unrealized_date_state,
);
}
@@ -296,21 +165,9 @@ impl CohortDataset {
.get(&self.split)
.unwrap();
self.all
self.subs
.price_paid
.insert(insert_data, &states.all.price_paid_state);
self.illiquid
.price_paid
.insert(insert_data, &states.illiquid.price_paid_state);
self.liquid
.price_paid
.insert(insert_data, &states.liquid.price_paid_state);
self.highly_liquid
.price_paid
.insert(insert_data, &states.highly_liquid.price_paid_state);
.insert(insert_data, &states.price_paid_state);
}
fn insert_input_data(&mut self, insert_data: &InsertData) {
@@ -321,12 +178,7 @@ impl CohortDataset {
.get(&self.split)
.unwrap();
self.all.input.insert(insert_data, &state.all);
self.illiquid.input.insert(insert_data, &state.illiquid);
self.liquid.input.insert(insert_data, &state.liquid);
self.highly_liquid
.input
.insert(insert_data, &state.highly_liquid);
self.subs.input.insert(insert_data, state);
}
// fn insert_output_data(&mut self, insert_data: &InsertData) {
@@ -337,7 +189,7 @@ impl CohortDataset {
// .get(&self.split)
// .unwrap();
// self.all.output.insert(insert_data, &state.all);
// self.output.insert(insert_data, &state.all);
// self.illiquid.output.insert(insert_data, &state.illiquid);
// self.liquid.output.insert(insert_data, &state.liquid);
// self.highly_liquid
@@ -346,29 +198,17 @@ impl CohortDataset {
// }
fn as_vec(&self) -> Vec<&(dyn AnyDataset + Send + Sync)> {
vec![
self.all.as_vec(),
self.illiquid.as_vec(),
self.liquid.as_vec(),
self.highly_liquid.as_vec(),
vec![&self.metadata],
]
.into_iter()
.flatten()
.collect_vec()
vec![self.subs.as_vec(), vec![&self.metadata]]
.into_iter()
.flatten()
.collect_vec()
}
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyDataset> {
vec![
self.all.as_mut_vec(),
self.illiquid.as_mut_vec(),
self.liquid.as_mut_vec(),
self.highly_liquid.as_mut_vec(),
vec![&mut self.metadata],
]
.into_iter()
.flatten()
.collect_vec()
vec![self.subs.as_mut_vec(), vec![&mut self.metadata]]
.into_iter()
.flatten()
.collect_vec()
}
pub fn insert(&mut self, insert_data: &InsertData) {
@@ -376,34 +216,36 @@ impl CohortDataset {
return;
}
let liquidity_split_processed_address_state = insert_data
let address_cohort_durable_states = insert_data
.states
.address_cohorts_durable_states
.as_ref()
.unwrap()
.get(&self.split);
if liquidity_split_processed_address_state.is_none() {
if address_cohort_durable_states.is_none() {
return; // TODO: Check if should panic instead
}
let liquidity_split_processed_address_state =
liquidity_split_processed_address_state.unwrap();
let address_cohort_durable_states = address_cohort_durable_states.unwrap();
if self.needs_insert_metadata(insert_data.height, insert_data.date) {
self.insert_metadata(insert_data);
}
if self.needs_insert_utxo(insert_data.height, insert_data.date) {
self.insert_utxo_data(insert_data, liquidity_split_processed_address_state);
self.insert_utxo_data(insert_data, &address_cohort_durable_states.durable_states);
}
if self.needs_insert_capitalization(insert_data.height, insert_data.date) {
self.insert_capitalization_data(insert_data, liquidity_split_processed_address_state);
self.insert_capitalization_data(
insert_data,
&address_cohort_durable_states.durable_states,
);
}
if self.needs_insert_supply(insert_data.height, insert_data.date) {
self.insert_supply_data(insert_data, liquidity_split_processed_address_state);
self.insert_supply_data(insert_data, &address_cohort_durable_states.durable_states);
}
if self.needs_insert_realized(insert_data.height, insert_data.date) {
@@ -478,17 +320,7 @@ impl CohortDataset {
compute_data: &ComputeData,
circulating_supply: &mut BiMap<f64>,
) {
self.all.supply.compute(compute_data, circulating_supply);
self.illiquid
.supply
.compute(compute_data, circulating_supply);
self.liquid.supply.compute(compute_data, circulating_supply);
self.highly_liquid
.supply
.compute(compute_data, circulating_supply);
self.subs.supply.compute(compute_data, circulating_supply);
}
fn compute_unrealized_data(
@@ -497,85 +329,28 @@ impl CohortDataset {
circulating_supply: &mut BiMap<f64>,
market_cap: &mut BiMap<f32>,
) {
self.all.unrealized.compute(
self.subs.unrealized.compute(
compute_data,
&mut self.all.supply.supply,
circulating_supply,
market_cap,
);
self.illiquid.unrealized.compute(
compute_data,
&mut self.illiquid.supply.supply,
circulating_supply,
market_cap,
);
self.liquid.unrealized.compute(
compute_data,
&mut self.liquid.supply.supply,
circulating_supply,
market_cap,
);
self.highly_liquid.unrealized.compute(
compute_data,
&mut self.highly_liquid.supply.supply,
&mut self.subs.supply.supply,
circulating_supply,
market_cap,
);
}
fn compute_realized_data(&mut self, compute_data: &ComputeData, market_cap: &mut BiMap<f32>) {
self.all.realized.compute(compute_data, market_cap);
self.illiquid.realized.compute(compute_data, market_cap);
self.liquid.realized.compute(compute_data, market_cap);
self.highly_liquid
.realized
.compute(compute_data, market_cap);
self.subs.realized.compute(compute_data, market_cap);
}
fn compute_capitalization_data(&mut self, compute_data: &ComputeData, closes: &mut BiMap<f32>) {
self.all
self.subs
.capitalization
.compute(compute_data, closes, &mut self.all.supply.supply);
self.illiquid.capitalization.compute(
compute_data,
closes,
&mut self.illiquid.supply.supply,
);
self.liquid
.capitalization
.compute(compute_data, closes, &mut self.liquid.supply.supply);
self.highly_liquid.capitalization.compute(
compute_data,
closes,
&mut self.highly_liquid.supply.supply,
);
.compute(compute_data, closes, &mut self.subs.supply.supply);
}
// fn compute_output_data(&mut self, compute_data: &ComputeData) {
// self.all
// .output
// .compute(compute_data, &mut self.all.supply.total);
// self.illiquid
// .output
// .compute(compute_data, &mut self.illiquid.supply.total);
// self.liquid
// .output
// .compute(compute_data, &mut self.liquid.supply.total);
// self.highly_liquid
// .output
// .compute(compute_data, &mut self.highly_liquid.supply.total);
// .compute(compute_data, &mut self.supply.total);
// }
pub fn compute(

4
parser/src/datasets/address/cohort_metadata.rs
View File

@@ -10,7 +10,7 @@ pub struct MetadataDataset {
min_initial_states: MinInitialStates,
// Inserted
address_count: BiMap<usize>,
address_count: BiMap<f64>,
// pub output: OutputSubDataset,
// Sending addresses
// Receiving addresses
@@ -48,7 +48,7 @@ impl MetadataDataset {
is_date_last_block,
..
}: &InsertData,
address_count: usize,
address_count: f64,
) {
self.address_count.height.insert(height, address_count);

4
parser/src/datasets/mod.rs
View File

@@ -257,8 +257,8 @@ impl AllDatasets {
&mut self.date_metadata.last_height,
&mut self.price.closes,
&mut self.mining.cumulative_subsidy,
&mut self.address.cohorts.all.all.capitalization.realized_cap,
&mut self.address.cohorts.all.all.capitalization.realized_price,
&mut self.address.cohorts.all.subs.capitalization.realized_cap,
&mut self.address.cohorts.all.subs.capitalization.realized_price,
&mut self.mining.yearly_inflation_rate,
&mut self.transaction.annualized_volume,
&mut self.mining.cumulative_subsidy_in_dollars,

1
parser/src/datasets/price/mod.rs
View File

@@ -9,6 +9,7 @@ use color_eyre::eyre::Error;
pub use ohlc::*;
use crate::{
log,
price::{Binance, Kibo, Kraken},
structs::{
AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId,

2
parser/src/datasets/subs/utxo.rs
View File

@@ -11,7 +11,7 @@ pub struct UTXOSubDataset {
min_initial_states: MinInitialStates,
// Inserted
count: BiMap<usize>,
count: BiMap<f64>,
}
impl UTXOSubDataset {

2
parser/src/main.rs
View File

@@ -8,7 +8,7 @@ fn main() -> color_eyre::Result<()> {
reset_logs();
let config = Config::import();
let config = Config::import()?;
let rpc = create_rpc(&config).unwrap();

390
parser/src/states/cohorts_states/address/cohort_durable_states.rs
View File

@@ -1,278 +1,80 @@
use std::ops::AddAssign;
use allocative::Allocative;
use crate::{
states::{DurableStates, OneShotStates, PriceToValue, UnrealizedState},
structs::{Amount, LiquiditySplitResult, Price, SplitByLiquidity},
states::{DurableStates, IsZero, OneShotStates, PriceToValue, UnrealizedState},
structs::{Amount, Price},
};
#[derive(Default, Debug, Allocative)]
pub struct AddressCohortDurableStates {
pub address_count: usize,
pub split_durable_states: SplitByLiquidity<DurableStates>,
pub price_to_split_amount: PriceToValue<SplitByLiquidity<Amount>>,
pub address_count: f64,
pub durable_states: DurableStates,
pub price_to_amount: PriceToValue<Amount>,
}
const ONE_THIRD: f64 = 1.0 / 3.0;
// TODO: Clean that mess, move to a generic liquidity split and somehow support rest for non floats
impl AddressCohortDurableStates {
#[allow(clippy::too_many_arguments)]
pub fn increment(
&mut self,
address_count: f64,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
realized_cap: Price,
mean_price_paid: Price,
split_sat_amount_result: &LiquiditySplitResult,
split_utxo_count_result: &LiquiditySplitResult,
split_realized_cap_result: &LiquiditySplitResult,
) -> color_eyre::Result<()> {
self.address_count += 1;
self.address_count += address_count;
self._crement(
amount,
utxo_count,
realized_cap,
mean_price_paid,
split_sat_amount_result,
split_utxo_count_result,
split_realized_cap_result,
true,
)
self._crement(amount, utxo_count, realized_cap, mean_price_paid, true)
}
#[allow(clippy::too_many_arguments)]
pub fn decrement(
&mut self,
address_count: f64,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
realized_cap: Price,
mean_price_paid: Price,
split_sat_amount_result: &LiquiditySplitResult,
split_utxo_count_result: &LiquiditySplitResult,
split_realized_cap_result: &LiquiditySplitResult,
) -> color_eyre::Result<()> {
self.address_count -= 1;
self.address_count -= address_count;
self._crement(
amount,
utxo_count,
realized_cap,
mean_price_paid,
split_sat_amount_result,
split_utxo_count_result,
split_realized_cap_result,
false,
)
self._crement(amount, utxo_count, realized_cap, mean_price_paid, false)
}
#[allow(clippy::too_many_arguments)]
pub fn _crement(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
realized_cap: Price,
mean_price_paid: Price,
split_sat_amount_result: &LiquiditySplitResult,
split_utxo_count_result: &LiquiditySplitResult,
split_realized_cap_result: &LiquiditySplitResult,
increment: bool,
) -> color_eyre::Result<()> {
if increment {
self.split_durable_states
.all
self.durable_states
.increment(amount, utxo_count, realized_cap)
} else {
self.split_durable_states
.all
self.durable_states
.decrement(amount, utxo_count, realized_cap)
}
.inspect_err(|report| {
dbg!(
report,
"split all failed",
split_sat_amount_result,
split_utxo_count_result
);
dbg!(report);
})?;
let illiquid_amount = split_sat_amount_result.illiquid.trunc();
let illiquid_amount_rest = split_sat_amount_result.illiquid - illiquid_amount;
let mut illiquid_amount = Amount::from_sat(illiquid_amount as u64);
let mut illiquid_utxo_count = split_utxo_count_result.illiquid.trunc() as usize;
let illiquid_utxo_count_rest = split_utxo_count_result.illiquid.fract();
let mut illiquid_realized_cap =
Price::from_cent(split_realized_cap_result.illiquid.trunc() as u64);
let illiquid_realized_cap_rest = split_realized_cap_result.illiquid.fract();
let liquid_amount = split_sat_amount_result.liquid.trunc();
let liquid_amount_rest = split_sat_amount_result.liquid - liquid_amount;
let mut liquid_amount = Amount::from_sat(liquid_amount as u64);
let mut liquid_utxo_count = split_utxo_count_result.liquid.trunc() as usize;
let liquid_utxo_count_rest = split_utxo_count_result.liquid.fract();
let mut liquid_realized_cap =
Price::from_cent(split_realized_cap_result.liquid.trunc() as u64);
let liquid_realized_cap_rest = split_realized_cap_result.liquid.fract();
let mut highly_liquid_amount = amount - illiquid_amount - liquid_amount;
let mut highly_liquid_utxo_count = utxo_count - illiquid_utxo_count - liquid_utxo_count;
let mut highly_liquid_realized_cap =
realized_cap - illiquid_realized_cap - liquid_realized_cap;
let amount_diff = amount - illiquid_amount - liquid_amount - highly_liquid_amount;
if amount_diff > Amount::ZERO {
if illiquid_amount_rest >= ONE_THIRD && illiquid_amount_rest > liquid_amount_rest {
illiquid_amount += amount_diff;
} else if illiquid_amount_rest >= ONE_THIRD {
liquid_amount += amount_diff;
if !amount.is_zero()? {
if increment {
self.price_to_amount.increment(mean_price_paid, amount);
} else {
highly_liquid_amount += amount_diff;
self.price_to_amount
.decrement(mean_price_paid, amount)
.inspect_err(|report| {
dbg!(report, "cents_to_split_amount decrement",);
})?;
}
}
let utxo_count_diff =
utxo_count - illiquid_utxo_count - liquid_utxo_count - highly_liquid_utxo_count;
if utxo_count_diff > 0 {
if illiquid_utxo_count_rest >= ONE_THIRD
&& illiquid_utxo_count_rest > liquid_utxo_count_rest
{
illiquid_utxo_count += utxo_count_diff;
} else if illiquid_utxo_count_rest >= ONE_THIRD {
liquid_utxo_count += utxo_count_diff;
} else {
highly_liquid_utxo_count += utxo_count_diff;
}
}
let realized_cap_diff =
realized_cap - illiquid_realized_cap - liquid_realized_cap - highly_liquid_realized_cap;
if realized_cap_diff > Price::ZERO {
if illiquid_realized_cap_rest >= ONE_THIRD
&& illiquid_realized_cap_rest > liquid_realized_cap_rest
{
illiquid_realized_cap += realized_cap_diff;
} else if illiquid_realized_cap_rest >= ONE_THIRD {
liquid_realized_cap += realized_cap_diff;
} else {
highly_liquid_realized_cap += realized_cap_diff;
}
}
let split_amount = SplitByLiquidity {
all: amount,
illiquid: illiquid_amount,
liquid: liquid_amount,
highly_liquid: highly_liquid_amount,
};
let split_utxo_count = SplitByLiquidity {
all: utxo_count,
illiquid: illiquid_utxo_count,
liquid: liquid_utxo_count,
highly_liquid: highly_liquid_utxo_count,
};
let split_realized_cap = SplitByLiquidity {
all: realized_cap,
illiquid: illiquid_realized_cap,
liquid: liquid_realized_cap,
highly_liquid: highly_liquid_realized_cap,
};
if increment {
self.price_to_split_amount
.increment(mean_price_paid, split_amount);
} else {
self.price_to_split_amount
.decrement(mean_price_paid, split_amount)
.inspect_err(|report| {
dbg!(
report,
"cents_to_split_amount decrement",
split_sat_amount_result,
split_utxo_count_result,
split_amount,
split_utxo_count,
split_realized_cap,
);
})?;
}
if increment {
self.split_durable_states.illiquid.increment(
illiquid_amount,
illiquid_utxo_count,
illiquid_realized_cap,
)
} else {
self.split_durable_states.illiquid.decrement(
illiquid_amount,
illiquid_utxo_count,
illiquid_realized_cap,
)
}
.inspect_err(|report| {
dbg!(
report,
"split illiquid failed",
split_sat_amount_result,
split_utxo_count_result,
split_amount,
split_utxo_count,
split_realized_cap,
);
})?;
if increment {
self.split_durable_states.liquid.increment(
liquid_amount,
liquid_utxo_count,
liquid_realized_cap,
)
} else {
self.split_durable_states.liquid.decrement(
liquid_amount,
liquid_utxo_count,
liquid_realized_cap,
)
}
.inspect_err(|report| {
dbg!(
report,
"split liquid failed",
split_sat_amount_result,
split_utxo_count_result,
split_amount,
split_utxo_count,
split_realized_cap,
);
})?;
if increment {
self.split_durable_states.highly_liquid.increment(
highly_liquid_amount,
highly_liquid_utxo_count,
highly_liquid_realized_cap,
)
} else {
self.split_durable_states.highly_liquid.decrement(
highly_liquid_amount,
highly_liquid_utxo_count,
highly_liquid_realized_cap,
)
}
.inspect_err(|report| {
dbg!(
report,
"split highly liquid failed",
split_sat_amount_result,
split_utxo_count_result,
split_amount,
split_utxo_count,
split_realized_cap,
);
})?;
Ok(())
}
@@ -280,132 +82,40 @@ impl AddressCohortDurableStates {
&self,
block_price: Price,
date_price: Option<Price>,
) -> SplitByLiquidity<OneShotStates> {
let mut one_shot_states: SplitByLiquidity<OneShotStates> = SplitByLiquidity::default();
) -> OneShotStates {
let mut one_shot_states = OneShotStates::default();
if date_price.is_some() {
one_shot_states
.all
.unrealized_date_state
.replace(UnrealizedState::default());
one_shot_states
.illiquid
.unrealized_date_state
.replace(UnrealizedState::default());
one_shot_states
.liquid
.unrealized_date_state
.replace(UnrealizedState::default());
one_shot_states
.highly_liquid
.unrealized_date_state
.replace(UnrealizedState::default());
}
let all_supply = self.split_durable_states.all.supply_state.supply;
let illiquid_supply = self.split_durable_states.illiquid.supply_state.supply;
let liquid_supply = self.split_durable_states.liquid.supply_state.supply;
let highly_liquid_supply = self.split_durable_states.highly_liquid.supply_state.supply;
let one_shot_states_ref = &mut one_shot_states;
self.price_to_split_amount.iterate(
SplitByLiquidity {
all: all_supply,
illiquid: illiquid_supply,
liquid: liquid_supply,
highly_liquid: highly_liquid_supply,
},
|price_paid, split_amount| {
one_shot_states_ref.all.price_paid_state.iterate(
price_paid,
split_amount.all,
all_supply,
);
one_shot_states_ref.all.unrealized_block_state.iterate(
price_paid,
block_price,
split_amount.all,
);
if let Some(unrealized_date_state) =
one_shot_states_ref.all.unrealized_date_state.as_mut()
{
unrealized_date_state.iterate(
price_paid,
date_price.unwrap(),
split_amount.all,
);
}
let supply = self.durable_states.supply_state.supply;
if split_amount.illiquid > Amount::ZERO {
one_shot_states_ref.illiquid.price_paid_state.iterate(
price_paid,
split_amount.illiquid,
illiquid_supply,
);
one_shot_states_ref.illiquid.unrealized_block_state.iterate(
price_paid,
block_price,
split_amount.illiquid,
);
if let Some(unrealized_date_state) =
one_shot_states_ref.illiquid.unrealized_date_state.as_mut()
{
unrealized_date_state.iterate(
price_paid,
date_price.unwrap(),
split_amount.illiquid,
);
}
}
if split_amount.liquid > Amount::ZERO {
one_shot_states_ref.liquid.price_paid_state.iterate(
price_paid,
split_amount.liquid,
liquid_supply,
);
one_shot_states_ref.liquid.unrealized_block_state.iterate(
price_paid,
block_price,
split_amount.liquid,
);
if let Some(unrealized_date_state) =
one_shot_states_ref.liquid.unrealized_date_state.as_mut()
{
unrealized_date_state.iterate(
price_paid,
date_price.unwrap(),
split_amount.liquid,
);
}
}
if split_amount.highly_liquid > Amount::ZERO {
one_shot_states_ref.highly_liquid.price_paid_state.iterate(
price_paid,
split_amount.highly_liquid,
highly_liquid_supply,
);
one_shot_states_ref
.highly_liquid
.unrealized_block_state
.iterate(price_paid, block_price, split_amount.highly_liquid);
if let Some(unrealized_date_state) = one_shot_states_ref
.highly_liquid
.unrealized_date_state
.as_mut()
{
unrealized_date_state.iterate(
price_paid,
date_price.unwrap(),
split_amount.highly_liquid,
);
}
}
},
);
self.price_to_amount.iterate(supply, |price_paid, amount| {
one_shot_states_ref
.price_paid_state
.iterate(price_paid, amount, supply);
one_shot_states_ref
.unrealized_block_state
.iterate(price_paid, block_price, amount);
if let Some(unrealized_date_state) = one_shot_states_ref.unrealized_date_state.as_mut()
{
unrealized_date_state.iterate(price_paid, date_price.unwrap(), amount);
}
});
one_shot_states
}
}
impl AddAssign for AddressCohortDurableStates {
fn add_assign(&mut self, rhs: Self) {
self.address_count += rhs.address_count;
self.durable_states += rhs.durable_states;
self.price_to_amount += rhs.price_to_amount;
}
}

14
parser/src/states/cohorts_states/address/cohort_id.rs
View File

@@ -1,9 +1,13 @@
use crate::structs::{AddressSize, AddressSplit, AddressType};
use crate::structs::{AddressLiquidity, AddressSize, AddressSplit, AddressType};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
pub enum AddressCohortId {
All,
Illiquid,
Liquid,
HighlyLiquid,
Plankton,
Shrimp,
Crab,
@@ -26,6 +30,10 @@ impl AddressCohortId {
match self {
Self::All => None,
Self::Illiquid => Some("illiquid"),
Self::Liquid => Some("liquid"),
Self::HighlyLiquid => Some("highly_liquid"),
Self::Plankton => Some("plankton"),
Self::Shrimp => Some("shrimp"),
Self::Crab => Some("crab"),
@@ -48,6 +56,10 @@ impl AddressCohortId {
match self {
Self::All => AddressSplit::All,
Self::Illiquid => AddressSplit::Liquidity(AddressLiquidity::Illiquid),
Self::Liquid => AddressSplit::Liquidity(AddressLiquidity::Liquid),
Self::HighlyLiquid => AddressSplit::Liquidity(AddressLiquidity::HighlyLiquid),
Self::Plankton => AddressSplit::Size(AddressSize::Plankton),
Self::Shrimp => AddressSplit::Size(AddressSize::Shrimp),
Self::Crab => AddressSplit::Size(AddressSize::Crab),

105
parser/src/states/cohorts_states/address/cohorts_durable_states.rs
View File

@@ -1,11 +1,12 @@
use std::iter::Sum;
use allocative::Allocative;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::{
databases::AddressIndexToAddressData,
structs::{AddressData, AddressRealizedData, Price},
structs::{AddressData, AddressRealizedData, Amount, Price},
};
use super::{AddressCohortDurableStates, AddressCohortsOneShotStates, SplitByAddressCohort};
@@ -15,13 +16,7 @@ pub struct AddressCohortsDurableStates(SplitByAddressCohort<AddressCohortDurable
impl AddressCohortsDurableStates {
pub fn init(address_index_to_address_data: &mut AddressIndexToAddressData) -> Self {
let mut s = Self::default();
// Paralize that, different s could be added together
address_index_to_address_data
.iter(&mut |(_, address_data)| s.increment(address_data).unwrap());
s
address_index_to_address_data.compute_addres_cohorts_durable_states()
}
pub fn iterate(
@@ -46,7 +41,7 @@ impl AddressCohortsDurableStates {
}
/// Should always increment using current address data state
fn increment(&mut self, address_data: &AddressData) -> color_eyre::Result<()> {
pub fn increment(&mut self, address_data: &AddressData) -> color_eyre::Result<()> {
self._crement(address_data, true)
}
@@ -62,57 +57,66 @@ impl AddressCohortsDurableStates {
}
let amount = address_data.amount;
let utxo_count = address_data.outputs_len as usize;
let utxo_count = address_data.outputs_len as f64;
let realized_cap = address_data.realized_cap;
let mean_price_paid = address_data.realized_cap / amount;
let liquidity_classification = address_data.compute_liquidity_classification();
let split_address_count = liquidity_classification.split(1.0);
let split_sat_amount = liquidity_classification.split(amount.to_sat() as f64);
let split_utxo_count = liquidity_classification.split(utxo_count as f64);
let split_realized_cap = liquidity_classification.split(utxo_count as f64);
let split_utxo_count = liquidity_classification.split(utxo_count);
let split_realized_cap = liquidity_classification.split(realized_cap.to_dollar());
self.0
.iterate(address_data, |state: &mut AddressCohortDurableStates| {
self.0.iterate(
address_data,
|state| {
// Unsplit it must be one
let address_count = 1.0;
if increment {
if let Err(report) = state.increment(
state.increment(
address_count,
amount,
utxo_count,
realized_cap,
mean_price_paid,
&split_sat_amount,
&split_utxo_count,
&split_realized_cap,
) {
dbg!(
report.to_string(),
&state,
&address_data,
&liquidity_classification
);
return Err(eyre!("increment error"));
}
} else if let Err(report) = state.decrement(
amount,
utxo_count,
realized_cap,
mean_price_paid,
&split_sat_amount,
&split_utxo_count,
&split_realized_cap,
) {
dbg!(
report.to_string(),
&state,
&address_data,
&liquidity_classification
);
return Err(eyre!("decrement error"));
)
} else {
state.decrement(
address_count,
amount,
utxo_count,
realized_cap,
mean_price_paid,
)
}
},
|liquidity, state| {
let address_count = split_address_count.from(liquidity);
let amount = Amount::from_sat(split_sat_amount.from(liquidity).floor() as u64);
let utxo_count = split_utxo_count.from(liquidity);
let realized_cap = Price::from_dollar(split_realized_cap.from(liquidity));
Ok(())
})?;
if increment {
state.increment(
address_count,
amount,
utxo_count,
realized_cap,
mean_price_paid,
)
} else {
state.decrement(
address_count,
amount,
utxo_count,
realized_cap,
mean_price_paid,
)
}
},
)?;
Ok(())
}
@@ -141,3 +145,12 @@ impl AddressCohortsDurableStates {
one_shot_states
}
}
impl Sum for AddressCohortsDurableStates {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
iter.fold(Self::default(), |mut a, b| {
a.0 += b.0;
a
})
}
}

45
parser/src/states/cohorts_states/address/cohorts_input_states.rs
View File

@@ -2,13 +2,13 @@ use derive_deref::{Deref, DerefMut};
use crate::{
states::InputState,
structs::{AddressRealizedData, Amount, LiquidityClassification, SplitByLiquidity},
structs::{AddressRealizedData, Amount, LiquidityClassification},
};
use super::SplitByAddressCohort;
#[derive(Deref, DerefMut, Default)]
pub struct AddressCohortsInputStates(SplitByAddressCohort<SplitByLiquidity<InputState>>);
pub struct AddressCohortsInputStates(SplitByAddressCohort<InputState>);
impl AddressCohortsInputStates {
pub fn iterate_input(
@@ -19,30 +19,27 @@ impl AddressCohortsInputStates {
let count = realized_data.utxos_destroyed as f64;
let sent = realized_data.sent;
let split_count = liquidity_classification.split(count);
let split_volume = liquidity_classification.split(sent.to_sat() as f64);
let iterate = move |state: &mut SplitByLiquidity<InputState>| -> color_eyre::Result<()> {
state.all.iterate(count, sent);
state.illiquid.iterate(
split_count.illiquid,
Amount::from_sat(split_volume.illiquid.round() as u64),
);
state.liquid.iterate(
split_count.liquid,
Amount::from_sat(split_volume.liquid.round() as u64),
);
state.highly_liquid.iterate(
split_count.highly_liquid,
Amount::from_sat(split_volume.highly_liquid.round() as u64),
);
let normal_iteration = move |state: &mut InputState| -> color_eyre::Result<()> {
state.iterate(count, sent);
Ok(())
};
self.iterate(&realized_data.initial_address_data, iterate)
let split_count = liquidity_classification.split(count);
let split_sent = liquidity_classification.split(sent.to_sat() as f64);
let liquified_iteration =
move |liquidity, state: &mut InputState| -> color_eyre::Result<()> {
state.iterate(
split_count.from(liquidity),
Amount::from_sat(split_sent.from(liquidity).round() as u64),
);
Ok(())
};
self.iterate(
&realized_data.initial_address_data,
normal_iteration,
liquified_iteration,
)
}
}

4
parser/src/states/cohorts_states/address/cohorts_one_shot_states.rs
View File

@@ -1,8 +1,8 @@
use derive_deref::{Deref, DerefMut};
use crate::{states::OneShotStates, structs::SplitByLiquidity};
use crate::states::OneShotStates;
use super::SplitByAddressCohort;
#[derive(Deref, DerefMut, Default)]
pub struct AddressCohortsOneShotStates(pub SplitByAddressCohort<SplitByLiquidity<OneShotStates>>);
pub struct AddressCohortsOneShotStates(pub SplitByAddressCohort<OneShotStates>);

45
parser/src/states/cohorts_states/address/cohorts_output_states.rs
View File

@@ -2,13 +2,13 @@ use derive_deref::{Deref, DerefMut};
use crate::{
states::OutputState,
structs::{AddressRealizedData, Amount, LiquidityClassification, SplitByLiquidity},
structs::{AddressRealizedData, Amount, LiquidityClassification},
};
use super::SplitByAddressCohort;
#[derive(Deref, DerefMut, Default)]
pub struct AddressCohortsOutputStates(SplitByAddressCohort<SplitByLiquidity<OutputState>>);
pub struct AddressCohortsOutputStates(SplitByAddressCohort<OutputState>);
impl AddressCohortsOutputStates {
pub fn iterate_output(
@@ -19,30 +19,27 @@ impl AddressCohortsOutputStates {
let count = realized_data.utxos_created as f64;
let volume = realized_data.received;
let split_count = liquidity_classification.split(count);
let split_volume = liquidity_classification.split(volume.to_sat() as f64);
let iterate = move |state: &mut SplitByLiquidity<OutputState>| -> color_eyre::Result<()> {
state.all.iterate(count, volume);
state.illiquid.iterate(
split_count.illiquid,
Amount::from_sat(split_volume.illiquid.round() as u64),
);
state.liquid.iterate(
split_count.liquid,
Amount::from_sat(split_volume.liquid.round() as u64),
);
state.highly_liquid.iterate(
split_count.highly_liquid,
Amount::from_sat(split_volume.highly_liquid.round() as u64),
);
let normal_iteration = move |state: &mut OutputState| -> color_eyre::Result<()> {
state.iterate(count, volume);
Ok(())
};
self.iterate(&realized_data.initial_address_data, iterate)
let split_count = liquidity_classification.split(count);
let split_volume = liquidity_classification.split(volume.to_sat() as f64);
let liquified_iteration =
move |liquidity, state: &mut OutputState| -> color_eyre::Result<()> {
state.iterate(
split_count.from(liquidity),
Amount::from_sat(split_volume.from(liquidity).round() as u64),
);
Ok(())
};
self.iterate(
&realized_data.initial_address_data,
normal_iteration,
liquified_iteration,
)
}
}

61
parser/src/states/cohorts_states/address/cohorts_realized_states.rs
View File

@@ -2,13 +2,13 @@ use derive_deref::{Deref, DerefMut};
use crate::{
states::RealizedState,
structs::{AddressRealizedData, LiquidityClassification, Price, SplitByLiquidity},
structs::{AddressRealizedData, LiquidityClassification, Price},
};
use super::SplitByAddressCohort;
#[derive(Deref, DerefMut, Default)]
pub struct AddressCohortsRealizedStates(SplitByAddressCohort<SplitByLiquidity<RealizedState>>);
pub struct AddressCohortsRealizedStates(SplitByAddressCohort<RealizedState>);
impl AddressCohortsRealizedStates {
pub fn iterate_realized(
@@ -21,6 +21,16 @@ impl AddressCohortsRealizedStates {
let value_created = realized_data.value_created;
let value_destroyed = realized_data.value_destroyed;
let normal_iteration = move |state: &mut RealizedState| -> color_eyre::Result<()> {
state.iterate(
realized_profit,
realized_loss,
value_created,
value_destroyed,
);
Ok(())
};
let split_realized_profit =
liquidity_classification.split(realized_profit.to_cent() as f64);
let split_realized_loss = liquidity_classification.split(realized_loss.to_cent() as f64);
@@ -28,38 +38,21 @@ impl AddressCohortsRealizedStates {
let split_value_destroyed =
liquidity_classification.split(value_destroyed.to_cent() as f64);
let iterate = move |state: &mut SplitByLiquidity<RealizedState>| -> color_eyre::Result<()> {
state.all.iterate(
realized_profit,
realized_loss,
value_created,
value_destroyed,
);
let liquified_iteration =
move |liquidity, state: &mut RealizedState| -> color_eyre::Result<()> {
state.iterate(
Price::from_cent(split_realized_profit.from(liquidity) as u64),
Price::from_cent(split_realized_loss.from(liquidity) as u64),
Price::from_cent(split_value_created.from(liquidity) as u64),
Price::from_cent(split_value_destroyed.from(liquidity) as u64),
);
Ok(())
};
state.illiquid.iterate(
Price::from_cent(split_realized_profit.illiquid as u64),
Price::from_cent(split_realized_loss.illiquid as u64),
Price::from_cent(split_value_created.illiquid as u64),
Price::from_cent(split_value_destroyed.illiquid as u64),
);
state.liquid.iterate(
Price::from_cent(split_realized_profit.liquid as u64),
Price::from_cent(split_realized_loss.liquid as u64),
Price::from_cent(split_value_created.liquid as u64),
Price::from_cent(split_value_destroyed.liquid as u64),
);
state.highly_liquid.iterate(
Price::from_cent(split_realized_profit.highly_liquid as u64),
Price::from_cent(split_realized_loss.highly_liquid as u64),
Price::from_cent(split_value_created.highly_liquid as u64),
Price::from_cent(split_value_destroyed.highly_liquid as u64),
);
Ok(())
};
self.iterate(&realized_data.initial_address_data, iterate)
self.iterate(
&realized_data.initial_address_data,
normal_iteration,
liquified_iteration,
)
}
}

82
parser/src/states/cohorts_states/address/split_by_address_cohort.rs
View File

@@ -1,6 +1,8 @@
use std::ops::AddAssign;
use allocative::Allocative;
use crate::structs::{AddressData, AddressSize, AddressSplit, AddressType};
use crate::structs::{AddressData, AddressLiquidity, AddressSize, AddressSplit, AddressType};
use super::AddressCohortId;
@@ -8,6 +10,10 @@ use super::AddressCohortId;
pub struct SplitByAddressCohort<T> {
pub all: T,
pub illiquid: T,
pub liquid: T,
pub highly_liquid: T,
pub plankton: T,
pub shrimp: T,
pub crab: T,
@@ -30,6 +36,12 @@ impl<T> SplitByAddressCohort<T> {
match &split {
AddressSplit::All => Some(&self.all),
AddressSplit::Liquidity(address_liquidity) => match address_liquidity {
AddressLiquidity::Illiquid => Some(&self.illiquid),
AddressLiquidity::Liquid => Some(&self.liquid),
AddressLiquidity::HighlyLiquid => Some(&self.highly_liquid),
},
AddressSplit::Type(address_type) => match address_type {
AddressType::P2PK => Some(&self.p2pk),
AddressType::P2PKH => Some(&self.p2pkh),
@@ -61,21 +73,32 @@ impl<T> SplitByAddressCohort<T> {
pub fn iterate(
&mut self,
address_data: &AddressData,
iterate: impl Fn(&mut T) -> color_eyre::Result<()>,
normal_iteration: impl Fn(&mut T) -> color_eyre::Result<()>,
liquified_iteration: impl Fn(AddressLiquidity, &mut T) -> color_eyre::Result<()>,
) -> color_eyre::Result<()> {
if let Some(state) = self.get_mut_from_split(&AddressSplit::All) {
iterate(state)?;
}
normal_iteration(self.get_mut_from_split(&AddressSplit::All).unwrap())?;
let mut _liquified_iteration = |address_liquidity| {
liquified_iteration(
address_liquidity,
self.get_mut_from_split(&AddressSplit::Liquidity(address_liquidity))
.unwrap(),
)
};
_liquified_iteration(AddressLiquidity::Illiquid)?;
_liquified_iteration(AddressLiquidity::Liquid)?;
_liquified_iteration(AddressLiquidity::HighlyLiquid)?;
if let Some(state) = self.get_mut_from_split(&AddressSplit::Type(address_data.address_type))
{
iterate(state)?;
normal_iteration(state)?;
}
if let Some(state) = self.get_mut_from_split(&AddressSplit::Size(AddressSize::from_amount(
address_data.amount,
))) {
iterate(state)?;
normal_iteration(state)?;
}
Ok(())
@@ -85,6 +108,12 @@ impl<T> SplitByAddressCohort<T> {
match &split {
AddressSplit::All => Some(&mut self.all),
AddressSplit::Liquidity(address_liquidity) => match address_liquidity {
AddressLiquidity::Illiquid => Some(&mut self.illiquid),
AddressLiquidity::Liquid => Some(&mut self.liquid),
AddressLiquidity::HighlyLiquid => Some(&mut self.highly_liquid),
},
AddressSplit::Type(address_type) => match address_type {
AddressType::P2PK => Some(&mut self.p2pk),
AddressType::P2PKH => Some(&mut self.p2pkh),
@@ -117,6 +146,10 @@ impl<T> SplitByAddressCohort<T> {
match id {
AddressCohortId::All => &mut self.all,
AddressCohortId::Illiquid => &mut self.illiquid,
AddressCohortId::Liquid => &mut self.liquid,
AddressCohortId::HighlyLiquid => &mut self.highly_liquid,
AddressCohortId::Plankton => &mut self.plankton,
AddressCohortId::Shrimp => &mut self.shrimp,
AddressCohortId::Crab => &mut self.crab,
@@ -138,6 +171,9 @@ impl<T> SplitByAddressCohort<T> {
pub fn as_vec(&self) -> Vec<(&T, AddressCohortId)> {
vec![
(&self.all, AddressCohortId::All),
(&self.illiquid, AddressCohortId::Illiquid),
(&self.liquid, AddressCohortId::Liquid),
(&self.highly_liquid, AddressCohortId::HighlyLiquid),
(&self.plankton, AddressCohortId::Plankton),
(&self.shrimp, AddressCohortId::Shrimp),
(&self.crab, AddressCohortId::Crab),
@@ -158,6 +194,9 @@ impl<T> SplitByAddressCohort<T> {
pub fn as_mut_vec(&mut self) -> Vec<(&mut T, AddressCohortId)> {
vec![
(&mut self.all, AddressCohortId::All),
(&mut self.illiquid, AddressCohortId::Illiquid),
(&mut self.liquid, AddressCohortId::Liquid),
(&mut self.highly_liquid, AddressCohortId::HighlyLiquid),
(&mut self.plankton, AddressCohortId::Plankton),
(&mut self.shrimp, AddressCohortId::Shrimp),
(&mut self.crab, AddressCohortId::Crab),
@@ -175,3 +214,32 @@ impl<T> SplitByAddressCohort<T> {
]
}
}
impl<T> AddAssign for SplitByAddressCohort<T>
where
T: AddAssign,
{
fn add_assign(&mut self, rhs: Self) {
self.all += rhs.all;
self.illiquid += rhs.illiquid;
self.liquid += rhs.liquid;
self.highly_liquid += rhs.highly_liquid;
self.plankton += rhs.plankton;
self.shrimp += rhs.shrimp;
self.crab += rhs.crab;
self.fish += rhs.fish;
self.shark += rhs.shark;
self.whale += rhs.whale;
self.humpback += rhs.humpback;
self.megalodon += rhs.megalodon;
self.p2pk += rhs.p2pk;
self.p2pkh += rhs.p2pkh;
self.p2sh += rhs.p2sh;
self.p2wpkh += rhs.p2wpkh;
self.p2wsh += rhs.p2wsh;
self.p2tr += rhs.p2tr;
}
}

8
parser/src/states/cohorts_states/any/capitalization_state.rs
View File

@@ -1,3 +1,5 @@
use std::ops::AddAssign;
use allocative::Allocative;
use crate::structs::Price;
@@ -16,3 +18,9 @@ impl CapitalizationState {
self.realized_cap -= realized_cap;
}
}
impl AddAssign for CapitalizationState {
fn add_assign(&mut self, rhs: Self) {
self.realized_cap += rhs.realized_cap;
}
}

41
parser/src/states/cohorts_states/any/durable_states.rs
View File

@@ -1,5 +1,6 @@
use std::ops::AddAssign;
use allocative::Allocative;
use color_eyre::eyre::eyre;
use crate::structs::{Amount, Price};
@@ -16,19 +17,12 @@ impl DurableStates {
pub fn increment(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
realized_cap: Price,
) -> color_eyre::Result<()> {
if amount == Amount::ZERO {
if utxo_count != 0 {
dbg!(amount, utxo_count);
return Err(eyre!("Shouldn't be possible"));
}
} else {
self.capitalization_state.increment(realized_cap);
self.supply_state.increment(amount);
self.utxo_state.increment(utxo_count);
}
self.utxo_state.increment(utxo_count);
self.capitalization_state.increment(realized_cap);
self.supply_state.increment(amount);
Ok(())
}
@@ -36,20 +30,21 @@ impl DurableStates {
pub fn decrement(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
realized_cap: Price,
) -> color_eyre::Result<()> {
if amount == Amount::ZERO {
if utxo_count != 0 {
dbg!(amount, utxo_count);
unreachable!("Shouldn't be possible")
}
} else {
self.capitalization_state.decrement(realized_cap);
self.supply_state.decrement(amount)?;
self.utxo_state.decrement(utxo_count)?;
}
self.utxo_state.decrement(utxo_count)?;
self.capitalization_state.decrement(realized_cap);
self.supply_state.decrement(amount)?;
Ok(())
}
}
impl AddAssign for DurableStates {
fn add_assign(&mut self, rhs: Self) {
self.capitalization_state += rhs.capitalization_state;
self.supply_state += rhs.supply_state;
self.utxo_state += rhs.utxo_state;
}
}

10
parser/src/states/cohorts_states/any/price_paid_state.rs
View File

@@ -26,9 +26,9 @@ pub struct PricePaidState {
}
impl PricePaidState {
pub fn iterate(&mut self, price: Price, amount: Amount, total_supply: Amount) {
pub fn iterate(&mut self, price: Price, amount: Amount, supply: Amount) {
let PricePaidState {
processed_amount,
processed_amount: processed_supply,
pp_05p,
pp_10p,
pp_15p,
@@ -50,14 +50,14 @@ impl PricePaidState {
pp_95p,
} = self;
*processed_amount += amount;
*processed_supply += amount;
if pp_95p.is_some() {
return;
}
let processed_sat_amount = processed_amount.to_sat();
let total_sat_supply = total_supply.to_sat();
let processed_sat_amount = processed_supply.to_sat();
let total_sat_supply = supply.to_sat();
if processed_sat_amount >= total_sat_supply * 95 / 100 {
pp_95p.replace(price);

40
parser/src/states/cohorts_states/any/price_to_value.rs
View File

@@ -8,7 +8,7 @@ use allocative::Allocative;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use crate::structs::{Amount, Price, SplitByLiquidity};
use crate::structs::{Amount, Price};
#[derive(Deref, DerefMut, Default, Debug, Allocative)]
pub struct PriceToValue<T>(BTreeMap<u32, T>);
@@ -78,6 +78,20 @@ where
}
}
impl<T> AddAssign for PriceToValue<T>
where
T: AddAssign + Copy,
{
fn add_assign(&mut self, rhs: Self) {
rhs.0.into_iter().for_each(|(key, value)| {
self.0
.entry(key)
.and_modify(|previous| *previous += value)
.or_insert(value);
});
}
}
pub trait CanSubtract {
fn can_subtract(&self, other: &Self) -> bool;
}
@@ -88,15 +102,6 @@ impl CanSubtract for Amount {
}
}
impl CanSubtract for SplitByLiquidity<Amount> {
fn can_subtract(&self, other: &Self) -> bool {
self.all >= other.all
&& self.illiquid >= other.illiquid
&& self.liquid >= other.liquid
&& self.highly_liquid >= other.highly_liquid
}
}
pub trait IsZero {
fn is_zero(&self) -> color_eyre::Result<bool>;
}
@@ -106,18 +111,3 @@ impl IsZero for Amount {
Ok(*self == Amount::ZERO)
}
}
impl IsZero for SplitByLiquidity<Amount> {
fn is_zero(&self) -> color_eyre::Result<bool> {
if self.all == Amount::ZERO
&& (self.illiquid != Amount::ZERO
|| self.liquid != Amount::ZERO
|| self.highly_liquid != Amount::ZERO)
{
dbg!(&self);
Err(eyre!("Bad split"))
} else {
Ok(self.all == Amount::ZERO)
}
}
}

8
parser/src/states/cohorts_states/any/supply_state.rs
View File

@@ -1,3 +1,5 @@
use std::ops::AddAssign;
use allocative::Allocative;
use color_eyre::eyre::eyre;
@@ -25,3 +27,9 @@ impl SupplyState {
Ok(())
}
}
impl AddAssign for SupplyState {
fn add_assign(&mut self, rhs: Self) {
self.supply += rhs.supply;
}
}

14
parser/src/states/cohorts_states/any/utxo_state.rs
View File

@@ -1,17 +1,19 @@
use std::ops::AddAssign;
use allocative::Allocative;
use color_eyre::eyre::eyre;
#[derive(Debug, Default, Allocative)]
pub struct UTXOState {
pub count: usize,
pub count: f64,
}
impl UTXOState {
pub fn increment(&mut self, utxo_count: usize) {
pub fn increment(&mut self, utxo_count: f64) {
self.count += utxo_count;
}
pub fn decrement(&mut self, utxo_count: usize) -> color_eyre::Result<()> {
pub fn decrement(&mut self, utxo_count: f64) -> color_eyre::Result<()> {
if self.count < utxo_count {
dbg!(self.count, utxo_count);
@@ -23,3 +25,9 @@ impl UTXOState {
Ok(())
}
}
impl AddAssign for UTXOState {
fn add_assign(&mut self, rhs: Self) {
self.count += rhs.count;
}
}

6
parser/src/states/cohorts_states/utxo/cohort_durable_states.rs
View File

@@ -15,7 +15,7 @@ impl UTXOCohortDurableStates {
pub fn increment(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
price: Price,
) -> color_eyre::Result<()> {
self._crement(amount, utxo_count, price, true)
@@ -24,7 +24,7 @@ impl UTXOCohortDurableStates {
pub fn decrement(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
price: Price,
) -> color_eyre::Result<()> {
self._crement(amount, utxo_count, price, false)
@@ -33,7 +33,7 @@ impl UTXOCohortDurableStates {
pub fn _crement(
&mut self,
amount: Amount,
utxo_count: usize,
utxo_count: f64,
price: Price,
increment: bool,
) -> color_eyre::Result<()> {

6
parser/src/states/cohorts_states/utxo/cohorts_durable_states.rs
View File

@@ -25,7 +25,7 @@ impl UTXOCohortsDurableStates {
date_data.blocks.iter().for_each(|block_data| {
let amount = block_data.amount;
let utxo_count = block_data.utxos as usize;
let utxo_count = block_data.utxos as f64;
// No need to either insert or remove if 0
if amount == Amount::ZERO {
@@ -56,7 +56,7 @@ impl UTXOCohortsDurableStates {
previous_last_block_data: Option<&BlockData>,
) {
let amount = block_data.amount;
let utxo_count = block_data.utxos as usize;
let utxo_count = block_data.utxos as f64;
let price = block_data.price;
// No need to either insert or remove if 0
@@ -110,7 +110,7 @@ impl UTXOCohortsDurableStates {
previous_last_block_data: &BlockData,
) {
let amount = sent_data.volume;
let utxo_count = sent_data.count as usize;
let utxo_count = sent_data.count as f64;
// No need to either insert or remove if 0
if amount == Amount::ZERO {

1
parser/src/states/mod.rs
View File

@@ -45,7 +45,6 @@ impl States {
self.utxo_cohorts_durable_states = None;
// TODO: Check that they are ONLY computed in an `if include_addresses`
if include_addresses {
let _ = self.address_counters.reset();

9
parser/src/structs/address_liquidity.rs Normal file
View File

@@ -0,0 +1,9 @@
use allocative::Allocative;
#[derive(Default, Allocative, Clone, Copy)]
pub enum AddressLiquidity {
#[default]
Illiquid,
Liquid,
HighlyLiquid,
}

3
parser/src/structs/address_split.rs
View File

@@ -1,6 +1,6 @@
use allocative::Allocative;
use super::{AddressSize, AddressType};
use super::{AddressLiquidity, AddressSize, AddressType};
#[derive(Default, Allocative)]
pub enum AddressSplit {
@@ -8,4 +8,5 @@ pub enum AddressSplit {
All,
Type(AddressType),
Size(AddressSize),
Liquidity(AddressLiquidity),
}

26
parser/src/structs/config.rs
View File

@@ -3,6 +3,8 @@ use std::fs::{self};
use clap::Parser;
use serde::{Deserialize, Serialize};
use crate::log;
#[derive(Parser, Debug, Clone, Default, Serialize, Deserialize)]
#[command(version, about, long_about = None)]
pub struct Config {
@@ -40,9 +42,9 @@ pub struct Config {
}
impl Config {
const PATH: &'static str = "config.toml";
const PATH: &'static str = "./config.toml";
pub fn import() -> Self {
pub fn import() -> color_eyre::Result<Self> {
let mut config_saved = fs::read_to_string(Self::PATH)
.map_or(Config::default(), |contents| {
toml::from_str(&contents).unwrap_or_default()
@@ -56,14 +58,10 @@ impl Config {
if let Some(rpcconnect) = config_args.rpcconnect {
config_saved.rpcconnect = Some(rpcconnect);
} else {
config_saved.rpcconnect = Some("localhost".to_string())
}
if let Some(rpcport) = config_args.rpcport {
config_saved.rpcport = Some(rpcport);
} else {
config_saved.rpcport = Some(8332);
}
if let Some(rpcuser) = config_args.rpcuser {
@@ -84,9 +82,21 @@ impl Config {
config.check();
config.write().unwrap();
config.write()?;
config
log("---");
log("Configuration:");
log(&format!("datadir: {:?}", config.datadir));
log(&format!("rpcconnect: {:?}", config.rpcconnect));
log(&format!("rpcport: {:?}", config.rpcport));
log(&format!("rpcuser: {:?}", config.rpcuser));
log(&format!("rpcpassword: {:?}", config.rpcpassword));
log(&format!("delay: {:?}", config.delay));
log(&format!("dry_run: {}", config.dry_run));
log(&format!("record_ram_usage: {}", config.record_ram_usage));
log("---");
Ok(config)
}
fn check(&self) {

48
parser/src/structs/liquidity.rs
View File

@@ -1,11 +1,6 @@
use std::{
f64::consts::E,
ops::{AddAssign, SubAssign},
};
use std::f64::consts::E;
use allocative::Allocative;
use super::Amount;
use super::{AddressLiquidity, Amount};
#[derive(Debug)]
pub struct LiquidityClassification {
@@ -97,37 +92,12 @@ pub struct LiquiditySplitResult {
pub highly_liquid: f64,
}
#[derive(Debug, Default, PartialEq, PartialOrd, Clone, Copy, Allocative)]
pub struct SplitByLiquidity<T>
where
T: Default,
{
pub all: T,
pub illiquid: T,
pub liquid: T,
pub highly_liquid: T,
}
impl<T> AddAssign for SplitByLiquidity<T>
where
T: AddAssign + Default,
{
fn add_assign(&mut self, rhs: Self) {
self.all += rhs.all;
self.illiquid += rhs.illiquid;
self.liquid += rhs.liquid;
self.highly_liquid += rhs.highly_liquid;
}
}
impl<T> SubAssign for SplitByLiquidity<T>
where
T: SubAssign + Default,
{
fn sub_assign(&mut self, rhs: Self) {
self.all -= rhs.all;
self.illiquid -= rhs.illiquid;
self.liquid -= rhs.liquid;
self.highly_liquid -= rhs.highly_liquid;
impl LiquiditySplitResult {
pub fn from(&self, address_liquidity: AddressLiquidity) -> f64 {
match address_liquidity {
AddressLiquidity::Illiquid => self.illiquid,
AddressLiquidity::Liquid => self.liquid,
AddressLiquidity::HighlyLiquid => self.highly_liquid,
}
}
}

2
parser/src/structs/mod.rs
View File

@@ -1,5 +1,6 @@
mod address;
mod address_data;
mod address_liquidity;
mod address_realized_data;
mod address_size;
mod address_split;
@@ -33,6 +34,7 @@ mod txout_index;
pub use address::*;
pub use address_data::*;
pub use address_liquidity::*;
pub use address_realized_data::*;
pub use address_size::*;
pub use address_split::*;

7
parser/src/utils/rpc.rs
View File

@@ -6,8 +6,11 @@ pub fn create_rpc(config: &Config) -> color_eyre::Result<Client> {
Ok(Client::new(
&format!(
"http://{}:{}",
config.rpcconnect.as_ref().unwrap(),
config.rpcport.unwrap()
config
.rpcconnect
.as_ref()
.unwrap_or(&"localhost".to_owned()),
config.rpcport.unwrap_or(8332)
),
Auth::UserPass(
config.rpcuser.clone().unwrap(),