diff --git a/.gitignore b/.gitignore index 86d71e5f0..749a5981f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ paths.d.ts # Outputs _outputs + +# Python +.ropeproject diff --git a/exit/src/lib.rs b/exit/src/lib.rs index 87ea02c9f..9daabe4af 100644 --- a/exit/src/lib.rs +++ b/exit/src/lib.rs @@ -56,7 +56,7 @@ impl Exit { self.blocked.store(false, Ordering::SeqCst); } - pub fn active(&self) -> bool { + pub fn blocked(&self) -> bool { self.active.load(Ordering::SeqCst) } } diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 157f61381..7d3af88cc 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -647,7 +647,7 @@ impl Indexer { txinindex_global += Txinindex::from(inputs_len); txoutindex_global += Txoutindex::from(outputs_len); - let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active(); + let should_snapshot = _height != 0 && _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.blocked(); if should_snapshot { export(trees, vecs, height)?; } diff --git a/pricer/src/price.rs b/pricer/src/price.rs new file mode 100644 index 000000000..ca79bd945 --- /dev/null +++ b/pricer/src/price.rs @@ -0,0 +1,733 @@ +use std::collections::BTreeMap; + +use allocative::Allocative; +use chrono::Days; +use color_eyre::eyre::Error; + +use struct_iterable::Iterable; + +use crate::{ + parser::price::{Binance, Kibo, Kraken}, + structs::{ + Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, + MapKind, Timestamp, OHLC, + }, + utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, +}; + +use super::{AnyDataset, ComputeData, MinInitialStates, RatioDataset}; + +#[derive(Allocative, Iterable)] +pub struct PriceDatasets { + min_initial_states: MinInitialStates, + + kraken_daily: Option>, + kraken_1mn: Option>, + binance_1mn: Option>, + binance_daily: Option>, + binance_har: Option>, + kibo_by_height: BTreeMap>, + kibo_by_date: BTreeMap>, + + pub ohlc: BiMap, + pub open: BiMap, + pub high: BiMap, + pub low: BiMap, + pub close: BiMap, + pub market_cap: BiMap, + pub price_1w_sma: BiMap, + pub price_1w_sma_ratio: RatioDataset, + pub price_1m_sma: BiMap, + pub price_1m_sma_ratio: RatioDataset, + pub price_1y_sma: BiMap, + pub price_1y_sma_ratio: RatioDataset, + pub price_2y_sma: BiMap, + pub price_2y_sma_ratio: RatioDataset, + pub price_4y_sma: BiMap, + pub price_4y_sma_ratio: RatioDataset, + pub price_8d_sma: BiMap, + pub price_8d_sma_ratio: RatioDataset, + pub price_13d_sma: BiMap, + pub price_13d_sma_ratio: RatioDataset, + pub price_21d_sma: BiMap, + pub price_21d_sma_ratio: RatioDataset, + pub price_34d_sma: BiMap, + pub price_34d_sma_ratio: RatioDataset, + pub price_55d_sma: BiMap, + pub price_55d_sma_ratio: RatioDataset, + pub price_89d_sma: BiMap, + pub price_89d_sma_ratio: RatioDataset, + pub price_144d_sma: BiMap, + pub price_144d_sma_ratio: RatioDataset, + pub price_200w_sma: BiMap, + pub price_200w_sma_ratio: RatioDataset, + pub price_1d_total_return: DateMap, + pub price_1m_total_return: DateMap, + pub price_6m_total_return: DateMap, + pub price_1y_total_return: DateMap, + pub price_2y_total_return: DateMap, + pub price_3y_total_return: DateMap, + pub price_4y_total_return: DateMap, + pub price_6y_total_return: DateMap, + pub price_8y_total_return: DateMap, + pub price_10y_total_return: DateMap, + pub price_4y_compound_return: DateMap, + // projection via lowest 4y compound value + pub all_time_high: BiMap, + pub all_time_high_date: DateMap, + pub days_since_all_time_high: DateMap, + pub max_days_between_all_time_highs: DateMap, + pub max_years_between_all_time_highs: DateMap, + pub market_price_to_all_time_high_ratio: BiMap, + pub drawdown: BiMap, + pub sats_per_dollar: BiMap, + // volatility +} + +impl PriceDatasets { + pub fn import(config: &Config) -> color_eyre::Result { + let path_dataset = config.path_datasets(); + let f = |s: &str| path_dataset.join(s); + + let mut s = Self { + min_initial_states: MinInitialStates::default(), + + binance_1mn: None, + binance_daily: None, + binance_har: None, + kraken_1mn: None, + kraken_daily: None, + kibo_by_height: BTreeMap::default(), + kibo_by_date: BTreeMap::default(), + + // --- + // Inserted + // --- + ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()), + + // --- + // Computed + // --- + open: BiMap::new_bin(1, MapKind::Computed, &f("open")), + high: BiMap::new_bin(1, MapKind::Computed, &f("high")), + low: BiMap::new_bin(1, MapKind::Computed, &f("low")), + close: BiMap::new_bin(1, MapKind::Computed, &f("close")), + market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")), + price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")), + price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?, + price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")), + price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?, + price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")), + price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?, + price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")), + price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?, + price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")), + price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?, + price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")), + price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?, + price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")), + price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?, + price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")), + price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?, + price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")), + price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?, + price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")), + price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?, + price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")), + price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?, + price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")), + price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?, + price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")), + price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?, + price_1d_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_1d_total_return"), + ), + price_1m_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_1m_total_return"), + ), + price_6m_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_6m_total_return"), + ), + price_1y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_1y_total_return"), + ), + price_2y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_2y_total_return"), + ), + price_3y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_3y_total_return"), + ), + price_4y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_4y_total_return"), + ), + price_6y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_6y_total_return"), + ), + price_8y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_8y_total_return"), + ), + price_10y_total_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_10y_total_return"), + ), + price_4y_compound_return: DateMap::new_bin( + 1, + MapKind::Computed, + &f("price_4y_compound_return"), + ), + all_time_high: BiMap::new_bin(1, MapKind::Computed, &f("all_time_high")), + all_time_high_date: DateMap::new_bin(1, MapKind::Computed, &f("all_time_high_date")), + days_since_all_time_high: DateMap::new_bin( + 1, + MapKind::Computed, + &f("days_since_all_time_high"), + ), + max_days_between_all_time_highs: DateMap::new_bin( + 1, + MapKind::Computed, + &f("max_days_between_all_time_highs"), + ), + max_years_between_all_time_highs: DateMap::new_bin( + 2, + MapKind::Computed, + &f("max_years_between_all_time_highs"), + ), + market_price_to_all_time_high_ratio: BiMap::new_bin( + 1, + MapKind::Computed, + &f("market_price_to_all_time_high_ratio"), + ), + drawdown: BiMap::new_bin(1, MapKind::Computed, &f("drawdown")), + sats_per_dollar: BiMap::new_bin(1, MapKind::Computed, &f("sats_per_dollar")), + }; + + s.min_initial_states + .consume(MinInitialStates::compute_from_dataset(&s, config)); + + Ok(s) + } + + pub fn compute(&mut self, compute_data: &ComputeData, circulating_supply: &mut BiMap) { + let &ComputeData { dates, heights, .. } = compute_data; + + self.open + .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open); + + self.high + .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high); + + self.low + .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low); + + self.close + .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); + + self.market_cap + .multi_insert_multiply(heights, dates, &mut self.close, circulating_supply); + + self.price_1w_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + ONE_WEEK_IN_DAYS, + ); + + self.price_1m_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + ONE_MONTH_IN_DAYS, + ); + + self.price_1y_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + ONE_YEAR_IN_DAYS, + ); + + self.price_2y_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + 2 * ONE_YEAR_IN_DAYS, + ); + + self.price_4y_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + 4 * ONE_YEAR_IN_DAYS, + ); + + self.price_8d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 8); + + self.price_13d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 13); + + self.price_21d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 21); + + self.price_34d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 34); + + self.price_55d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 55); + + self.price_89d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 89); + + self.price_144d_sma + .multi_insert_simple_average(heights, dates, &mut self.close, 144); + + self.price_200w_sma.multi_insert_simple_average( + heights, + dates, + &mut self.close, + 200 * ONE_WEEK_IN_DAYS, + ); + + self.price_1d_total_return + .multi_insert_percentage_change(dates, &mut self.close.date, 1); + self.price_1m_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + ONE_MONTH_IN_DAYS, + ); + self.price_6m_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 6 * ONE_MONTH_IN_DAYS, + ); + self.price_1y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + ONE_YEAR_IN_DAYS, + ); + self.price_2y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 2 * ONE_YEAR_IN_DAYS, + ); + self.price_3y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 3 * ONE_YEAR_IN_DAYS, + ); + self.price_4y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 4 * ONE_YEAR_IN_DAYS, + ); + self.price_6y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 6 * ONE_YEAR_IN_DAYS, + ); + self.price_8y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 8 * ONE_YEAR_IN_DAYS, + ); + self.price_10y_total_return.multi_insert_percentage_change( + dates, + &mut self.close.date, + 10 * ONE_YEAR_IN_DAYS, + ); + + self.price_4y_compound_return + .multi_insert_complex_transform( + dates, + &mut self.close.date, + |(last_value, date, closes, _)| { + let previous_value = date + .checked_sub_days(Days::new(4 * ONE_YEAR_IN_DAYS as u64)) + .and_then(|date| closes.get_or_import(&Date::wrap(date))) + .unwrap_or_default(); + + (((last_value / previous_value).powf(1.0 / 4.0)) - 1.0) * 100.0 + }, + ); + + self.price_1w_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_1w_sma); + self.price_1m_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_1m_sma); + self.price_1y_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_1y_sma); + self.price_2y_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_2y_sma); + self.price_4y_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_4y_sma); + self.price_8d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_8d_sma); + self.price_13d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_13d_sma); + self.price_21d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_21d_sma); + self.price_34d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_34d_sma); + self.price_55d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_55d_sma); + self.price_89d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_89d_sma); + self.price_144d_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_144d_sma); + self.price_200w_sma_ratio + .compute(compute_data, &mut self.close, &mut self.price_200w_sma); + + self.all_time_high + .multi_insert_max(heights, dates, &mut self.high); + + self.market_price_to_all_time_high_ratio + .multi_insert_percentage(heights, dates, &mut self.close, &mut self.all_time_high); + + self.all_time_high_date.multi_insert_complex_transform( + dates, + &mut self.all_time_high.date, + |(value, date, _, map)| { + let high = self.high.date.get_or_import(date).unwrap(); + let is_ath = high == value; + + if is_ath { + *date + } else { + let previous_date = date.checked_sub(1).unwrap(); + *map.get_or_import(&previous_date).as_ref().unwrap_or(date) + } + }, + ); + + self.days_since_all_time_high.multi_insert_simple_transform( + dates, + &mut self.all_time_high_date, + |value, key| key.difference_in_days_between(value), + ); + + self.max_days_between_all_time_highs + .multi_insert_max(dates, &mut self.days_since_all_time_high); + + self.max_years_between_all_time_highs + .multi_insert_simple_transform( + dates, + &mut self.max_days_between_all_time_highs, + |days, _| (days as f64 / ONE_YEAR_IN_DAYS as f64) as f32, + ); + + self.drawdown.multi_insert_simple_transform( + heights, + dates, + &mut self.market_price_to_all_time_high_ratio, + &|v| -(100.0 - v), + ); + + self.sats_per_dollar.multi_insert_simple_transform( + heights, + dates, + &mut self.close, + &|price| Amount::ONE_BTC_F32 / price, + ); + } + + pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { + if self.ohlc.date.is_key_safe(date) { + Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned()) + } else { + let ohlc = self + .get_from_daily_kraken(&date) + .or_else(|_| self.get_from_daily_binance(&date)) + .or_else(|_| self.get_from_date_kibo(&date))?; + + self.ohlc.date.insert(date, ohlc); + + Ok(ohlc) + } + } + + fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result { + let chunk_id = date.to_chunk_id(); + + #[allow(clippy::map_entry)] + if !self.kibo_by_date.contains_key(&chunk_id) + || self + .kibo_by_date + .get(&chunk_id) + .unwrap() + .last_key_value() + .unwrap() + .0 + < date + { + self.kibo_by_date + .insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?); + } + + self.kibo_by_date + .get(&chunk_id) + .unwrap() + .get(date) + .cloned() + .ok_or(Error::msg("Couldn't find date in satonomics")) + } + + fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result { + if self.kraken_daily.is_none() + || self + .kraken_daily + .as_ref() + .unwrap() + .last_key_value() + .unwrap() + .0 + < date + { + self.kraken_daily.replace(Kraken::fetch_daily_prices()?); + } + + self.kraken_daily + .as_ref() + .unwrap() + .get(date) + .cloned() + .ok_or(Error::msg("Couldn't find date")) + } + + fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result { + if self.binance_daily.is_none() + || self + .binance_daily + .as_ref() + .unwrap() + .last_key_value() + .unwrap() + .0 + < date + { + self.binance_daily.replace(Binance::fetch_daily_prices()?); + } + + self.binance_daily + .as_ref() + .unwrap() + .get(date) + .cloned() + .ok_or(Error::msg("Couldn't find date")) + } + + pub fn get_height_ohlc( + &mut self, + height: Height, + timestamp: Timestamp, + previous_timestamp: Option, + config: &Config, + ) -> color_eyre::Result { + if let Some(ohlc) = self.ohlc.height.get_or_import(&height) { + return Ok(ohlc); + } + + let timestamp = timestamp.to_floored_seconds(); + + if previous_timestamp.is_none() && !height.is_first() { + panic!("Shouldn't be possible"); + } + + let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds()); + + let ohlc = self + .get_from_1mn_kraken(timestamp, previous_timestamp) + .unwrap_or_else(|_| { + self.get_from_1mn_binance(timestamp, previous_timestamp) + .unwrap_or_else(|_| { + self.get_from_har_binance(timestamp, previous_timestamp, config) + .unwrap_or_else(|_| { + self.get_from_height_kibo(&height).unwrap_or_else(|_| { + let date = timestamp.to_date(); + + panic!( + "Can't find the price for: height: {height} - date: {date} +1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's +How to fix this: +1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot +2. Select 1mn interval +3. Open the inspector/dev tools +4. Go to the Network Tab +5. Filter URLs by 'uiKlines' +6. Go back to the chart and scroll until you pass the date mentioned few lines ago +7. Go back to the dev tools +8. Export to a har file (if there is no explicit button, click on the cog button) +9. Move the file to 'parser/imports/binance.har' +" + ) + }) + }) + }) + }); + + self.ohlc.height.insert(height, ohlc); + + Ok(ohlc) + } + + fn get_from_height_kibo(&mut self, height: &Height) -> color_eyre::Result { + let chunk_id = height.to_chunk_id(); + + #[allow(clippy::map_entry)] + if !self.kibo_by_height.contains_key(&chunk_id) + || ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) + <= height.to_usize()) + { + self.kibo_by_height + .insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?); + } + + self.kibo_by_height + .get(&chunk_id) + .unwrap() + .get(height.to_serialized_key().to_usize()) + .cloned() + .ok_or(Error::msg("Couldn't find height in kibo")) + } + + fn get_from_1mn_kraken( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if self.kraken_1mn.is_none() + || self + .kraken_1mn + .as_ref() + .unwrap() + .last_key_value() + .unwrap() + .0 + <= ×tamp + { + self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?); + } + + Self::find_height_ohlc(&self.kraken_1mn, timestamp, previous_timestamp, "kraken 1m") + } + + fn get_from_1mn_binance( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if self.binance_1mn.is_none() + || self + .binance_1mn + .as_ref() + .unwrap() + .last_key_value() + .unwrap() + .0 + <= ×tamp + { + self.binance_1mn.replace(Binance::fetch_1mn_prices()?); + } + + Self::find_height_ohlc( + &self.binance_1mn, + timestamp, + previous_timestamp, + "binance 1m", + ) + } + + fn get_from_har_binance( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + config: &Config, + ) -> color_eyre::Result { + if self.binance_har.is_none() { + self.binance_har + .replace(Binance::read_har_file(config).unwrap_or_default()); + } + + Self::find_height_ohlc( + &self.binance_har, + timestamp, + previous_timestamp, + "binance har", + ) + } + + fn find_height_ohlc( + tree: &Option>, + timestamp: Timestamp, + previous_timestamp: Option, + name: &str, + ) -> color_eyre::Result { + let tree = tree.as_ref().unwrap(); + + let err = Error::msg(format!("Couldn't find timestamp in {name}")); + + let previous_ohlc = previous_timestamp + .map_or(Some(OHLC::default()), |previous_timestamp| { + tree.get(&previous_timestamp).cloned() + }); + + let last_ohlc = tree.get(×tamp); + + if previous_ohlc.is_none() || last_ohlc.is_none() { + return Err(err); + } + + let previous_ohlc = previous_ohlc.unwrap(); + + let mut final_ohlc = OHLC { + open: previous_ohlc.close, + high: previous_ohlc.close, + low: previous_ohlc.close, + close: previous_ohlc.close, + }; + + let start = previous_timestamp.unwrap_or_default(); + let end = timestamp; + + // Otherwise it's a re-org + if start < end { + tree.range(&*start..=&*end).skip(1).for_each(|(_, ohlc)| { + if ohlc.high > final_ohlc.high { + final_ohlc.high = ohlc.high + } + + if ohlc.low < final_ohlc.low { + final_ohlc.low = ohlc.low + } + + final_ohlc.close = ohlc.close; + }); + } + + Ok(final_ohlc) + } +} + +impl AnyDataset for PriceDatasets { + fn get_min_initial_states(&self) -> &MinInitialStates { + &self.min_initial_states + } +} diff --git a/pricer/src/price/binance.rs b/pricer/src/price/binance.rs new file mode 100644 index 000000000..0967ad1e0 --- /dev/null +++ b/pricer/src/price/binance.rs @@ -0,0 +1,213 @@ +#![allow(dead_code)] + +use std::{collections::BTreeMap, fs}; + +use color_eyre::eyre::ContextCompat; +use itertools::Itertools; +use log::info; +use serde_json::Value; + +use crate::{ + io::Json, + structs::{Config, Date, Timestamp, OHLC}, + utils::retry, +}; + +pub struct Binance; + +impl Binance { + pub fn read_har_file(config: &Config) -> color_eyre::Result> { + info!("binance: read har file"); + + let path = config.path_inputs(); + + fs::create_dir_all(&path)?; + + let path_binance_har = path.join("binance.har"); + + let json: BTreeMap = Json::import(&path_binance_har).unwrap_or_default(); + + Ok(json + .get("log") + .context("Expect object to have log attribute")? + .as_object() + .context("Expect to be an object")? + .get("entries") + .context("Expect object to have entries")? + .as_array() + .context("Expect to be an array")? + .iter() + .filter(|entry| { + entry + .as_object() + .unwrap() + .get("request") + .unwrap() + .as_object() + .unwrap() + .get("url") + .unwrap() + .as_str() + .unwrap() + .contains("/uiKlines") + }) + .flat_map(|entry| { + let response = entry + .as_object() + .unwrap() + .get("response") + .unwrap() + .as_object() + .unwrap(); + + let content = response.get("content").unwrap().as_object().unwrap(); + + let text = content.get("text"); + + if text.is_none() { + return vec![]; + } + + let text = text.unwrap().as_str().unwrap(); + + let arrays: Value = serde_json::from_str(text).unwrap(); + + arrays + .as_array() + .unwrap() + .iter() + .map(|array| { + let array = array.as_array().unwrap(); + + let timestamp = (array.first().unwrap().as_u64().unwrap() / 1000) as u32; + + let get_f32 = |index: usize| { + array + .get(index) + .unwrap() + .as_str() + .unwrap() + .parse::() + .unwrap() + }; + + ( + timestamp, + OHLC { + open: get_f32(1), + high: get_f32(2), + low: get_f32(3), + close: get_f32(4), + }, + ) + }) + .collect_vec() + }) + .collect::>()) + } + + pub fn fetch_1mn_prices() -> color_eyre::Result> { + info!("binance: fetch 1mn"); + + retry( + |_| { + let body: Value = reqwest::blocking::get( + "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000", + )? + .json()?; + + Ok(body + .as_array() + .context("Expect to be an array")? + .iter() + .map(|value| -> color_eyre::Result<_> { + // [timestamp, open, high, low, close, volume, ...] + let array = value.as_array().context("Expect to be array")?; + + let timestamp = (array + .first() + .context("Expect to have first")? + .as_u64() + .context("Expect to be convertible to u64")? + / 1_000) as u32; + + let get_f32 = |index: usize| -> color_eyre::Result { + Ok(array + .get(index) + .context("Expect to have index")? + .as_str() + .context("Expect to have &str")? + .parse::()?) + }; + + Ok(( + timestamp, + OHLC { + open: get_f32(1)?, + high: get_f32(2)?, + low: get_f32(3)?, + close: get_f32(4)?, + }, + )) + }) + .collect::, _>>()?) + }, + 30, + 10, + ) + } + + pub fn fetch_daily_prices() -> color_eyre::Result> { + info!("binance: fetch 1d"); + + retry( + |_| { + let body: Value = reqwest::blocking::get( + "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d", + )? + .json()?; + + Ok(body + .as_array() + .context("Expect to be an array")? + .iter() + .map(|value| -> color_eyre::Result<_> { + // [timestamp, open, high, low, close, volume, ...] + let array = value.as_array().context("Expect to be array")?; + + let date = Timestamp::from( + (array + .first() + .context("Expect to have first")? + .as_u64() + .context("Expect to be convertible to u64")? + / 1_000) as u32, + ) + .to_date(); + + let get_f32 = |index: usize| -> color_eyre::Result { + Ok(array + .get(index) + .context("Expect to have index")? + .as_str() + .context("Expect to have &str")? + .parse::()?) + }; + + Ok(( + date, + OHLC { + open: get_f32(1)?, + high: get_f32(2)?, + low: get_f32(3)?, + close: get_f32(4)?, + }, + )) + }) + .collect::, _>>()?) + }, + 30, + 10, + ) + } +} diff --git a/pricer/src/price/kibo.rs b/pricer/src/price/kibo.rs new file mode 100644 index 000000000..858b7d7ac --- /dev/null +++ b/pricer/src/price/kibo.rs @@ -0,0 +1,118 @@ +use std::{collections::BTreeMap, str::FromStr}; + +use chrono::NaiveDate; +use color_eyre::eyre::ContextCompat; +use log::info; +use serde_json::Value; + +use crate::{ + structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC}, + utils::retry, +}; + +pub struct Kibo; + +const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api"; +const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api"; + +const RETRIES: usize = 10; + +impl Kibo { + fn get_base_url(try_index: usize) -> &'static str { + if try_index < RETRIES / 2 { + KIBO_OFFICIAL_URL + } else { + KIBO_OFFICIAL_BACKUP_URL + } + } + + pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result> { + info!("kibo: fetch height prices"); + + retry( + |try_index| { + let base_url = Self::get_base_url(try_index); + + let body: Value = reqwest::blocking::get(format!( + "{base_url}/height-to-price?chunk={}", + chunk_id.to_usize() + ))? + .json()?; + + let vec = body + .as_object() + .context("Expect to be an object")? + .get("dataset") + .context("Expect object to have dataset")? + .as_object() + .context("Expect to be an object")? + .get("map") + .context("Expect to have map")? + .as_array() + .context("Expect to be an array")? + .iter() + .map(Self::value_to_ohlc) + .collect::, _>>()?; + + Ok(vec) + }, + 30, + RETRIES, + ) + } + + pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result> { + info!("kibo: fetch date prices"); + + retry( + |try_index| { + let base_url = Self::get_base_url(try_index); + + let body: Value = reqwest::blocking::get(format!( + "{base_url}/date-to-price?chunk={}", + chunk_id.to_usize() + ))? + .json()?; + + Ok(body + .as_object() + .context("Expect to be an object")? + .get("dataset") + .context("Expect object to have dataset")? + .as_object() + .context("Expect to be an object")? + .get("map") + .context("Expect to have map")? + .as_object() + .context("Expect to be an object")? + .iter() + .map(|(serialized_date, value)| -> color_eyre::Result<_> { + let date = Date::wrap(NaiveDate::from_str(serialized_date)?); + Ok((date, Self::value_to_ohlc(value)?)) + }) + .collect::, _>>()?) + }, + 30, + RETRIES, + ) + } + + fn value_to_ohlc(value: &Value) -> color_eyre::Result { + let ohlc = value.as_object().context("Expect as_object to work")?; + + let get_value = |key: &str| -> color_eyre::Result { + Ok(ohlc + .get(key) + .context("Expect get key to work")? + .as_f64() + .context("Expect as_f64 to work")? as f32) + }; + + Ok(OHLC { + open: get_value("open")?, + high: get_value("high")?, + low: get_value("low")?, + close: get_value("close")?, + }) + } +} diff --git a/pricer/src/price/kraken.rs b/pricer/src/price/kraken.rs new file mode 100644 index 000000000..99566fee0 --- /dev/null +++ b/pricer/src/price/kraken.rs @@ -0,0 +1,133 @@ +use std::collections::BTreeMap; + +use color_eyre::eyre::ContextCompat; +use log::info; +use serde_json::Value; + +use crate::{ + structs::{Date, Timestamp, OHLC}, + utils::retry, +}; + +pub struct Kraken; + +impl Kraken { + pub fn fetch_1mn_prices() -> color_eyre::Result> { + info!("kraken: fetch 1mn"); + + retry( + |_| { + let body: Value = reqwest::blocking::get( + "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1", + )? + .json()?; + + Ok(body + .as_object() + .context("Expect to be an object")? + .get("result") + .context("Expect object to have result")? + .as_object() + .context("Expect to be an object")? + .get("XXBTZUSD") + .context("Expect to have XXBTZUSD")? + .as_array() + .context("Expect to be an array")? + .iter() + .map(|value| -> color_eyre::Result<_> { + let array = value.as_array().context("Expect as_array to work")?; + + let timestamp = array + .first() + .context("Expect first to work")? + .as_u64() + .expect("Expect as_u64 to work") + as u32; + + let get_f32 = |index: usize| -> color_eyre::Result { + Ok(array + .get(index) + .context("Expect get index to work")? + .as_str() + .context("Expect as_str to work")? + .parse::()?) + }; + + Ok(( + timestamp, + OHLC { + open: get_f32(1)?, + high: get_f32(2)?, + low: get_f32(3)?, + close: get_f32(4)?, + }, + )) + }) + .collect::, _>>()?) + }, + 30, + 10, + ) + } + + pub fn fetch_daily_prices() -> color_eyre::Result> { + info!("fetch kraken daily"); + + retry( + |_| { + let body: Value = reqwest::blocking::get( + "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440", + )? + .json()?; + + Ok(body + .as_object() + .context("Expect to be an object")? + .get("result") + .context("Expect object to have result")? + .as_object() + .context("Expect to be an object")? + .get("XXBTZUSD") + .context("Expect to have XXBTZUSD")? + .as_array() + .context("Expect to be an array")? + .iter() + .map(|value| -> color_eyre::Result<_> { + let array = value.as_array().context("Expect as_array to work")?; + + let date = Timestamp::from( + array + .first() + .context("Expect first to work")? + .as_u64() + .context("Expect as_u64 to work")? + as u32, + ) + .to_date(); + + let get_f32 = |index: usize| -> color_eyre::Result { + Ok(array + .get(index) + .context("Expect get index to work")? + .as_str() + .context("Expect as_str to work")? + .parse::()?) + }; + + Ok(( + date, + OHLC { + open: get_f32(1)?, + high: get_f32(2)?, + low: get_f32(3)?, + close: get_f32(4)?, + }, + )) + }) + .collect::, _>>()?) + }, + 30, + 10, + ) + } +} diff --git a/pricer/src/price/mod.rs b/pricer/src/price/mod.rs new file mode 100644 index 000000000..41cea4435 --- /dev/null +++ b/pricer/src/price/mod.rs @@ -0,0 +1,7 @@ +mod binance; +mod kibo; +mod kraken; + +pub use binance::*; +pub use kibo::*; +pub use kraken::*; diff --git a/python/example.py b/python/example.py new file mode 100644 index 000000000..a07c64a32 --- /dev/null +++ b/python/example.py @@ -0,0 +1,12 @@ +# Here's an example on how to parse the output from the indexer +# We're aiming to read the first 21 values from the height_to_timestamp vec + +import sys +import datetime + +with open("../_outputs/indexes/vecs/height_to_timestamp/vec", "rb") as file: + for x in range(0, 21): + bytes = file.read(4) # Need to check the rust side to find the size, at least for now + number = int.from_bytes(bytes, sys.byteorder) + date = datetime.date.fromtimestamp(number) + print(date)