diff --git a/.gitignore b/.gitignore index 7b02fdf25..4f590c2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ TODO.md .stfolder /charts +/price diff --git a/CHANGELOG.md b/CHANGELOG.md index dc60dcab1..c0083f7ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Improved self-hosting by: - Fixing an incredibly annoying bug that made the program panic because of a wrong utxo/address durable state after a or many new datasets were added/changed after a first successful parse of the chain - Fixing a bug that would crash the program if launched for the first time ever + - Auto fetch prices from the main Satonomics instance if missing instead of only trying Kraken's and Binance's API which are limited to the last 16 hours - Merged the core of `HeightMap` and `DateMap` structs into `GenericMap` - Added `Height` struct and many others - CLI @@ -70,6 +71,7 @@ - Run file - Only run with a watcher if `cargo watch` is available +- Added trigger folder to automatically restart when a new dataset has been added in the parser ## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08 diff --git a/parser/src/bitcoin/daemon.rs b/parser/src/bitcoin/daemon.rs index 45f58e52b..9ba837a0e 100644 --- a/parser/src/bitcoin/daemon.rs +++ b/parser/src/bitcoin/daemon.rs @@ -96,7 +96,7 @@ impl BitcoinDaemon { fn get_blockchain_info(&self) -> BlockchainInfo { retry( - || { + |_| { // bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo let output = Command::new("bitcoin-cli") .arg(self.datadir_arg()) @@ -122,7 +122,7 @@ impl BitcoinDaemon { Ok(BlockchainInfo { headers, blocks }) }, 1, - u64::MAX, + usize::MAX, ) .unwrap() } diff --git a/parser/src/datasets/price/mod.rs b/parser/src/datasets/price/mod.rs index b9f2e0bed..d8fbf1249 100644 --- a/parser/src/datasets/price/mod.rs +++ b/parser/src/datasets/price/mod.rs @@ -9,8 +9,11 @@ use color_eyre::eyre::Error; pub use ohlc::*; use crate::{ - price::{Binance, Kraken}, - structs::{AnyBiMap, AnyDateMap, BiMap, Date, DateMap, Height, MapKey}, + price::{Binance, Kraken, Satonomics}, + structs::{ + AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, + MapKey, + }, utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, }; @@ -24,7 +27,8 @@ pub struct PriceDatasets { kraken_1mn: Option>, binance_1mn: Option>, binance_har: Option>, - satonomics_by_height: BTreeMap>>, + satonomics_by_height: BTreeMap>, + satonomics_by_date: BTreeMap>, // Inserted pub ohlcs: BiMap, @@ -89,6 +93,7 @@ impl PriceDatasets { kraken_1mn: None, kraken_daily: None, satonomics_by_height: BTreeMap::default(), + satonomics_by_date: BTreeMap::default(), ohlcs: BiMap::new_json(1, price_path), closes: BiMap::new_bin(1, &f("close")), @@ -304,7 +309,9 @@ impl PriceDatasets { if self.ohlcs.date.is_key_safe(date) { Ok(self.ohlcs.date.get(&date).unwrap().to_owned()) } else { - let ohlc = self.get_from_daily_kraken(&date)?; + let ohlc = self + .get_from_date_satonomics(&date) + .or_else(|_| self.get_from_daily_kraken(&date))?; self.ohlcs.date.insert(date, ohlc); @@ -312,12 +319,27 @@ impl PriceDatasets { } } + fn get_from_date_satonomics(&mut self, date: &Date) -> color_eyre::Result { + let chunk_id = date.to_chunk_id(); + + #[allow(clippy::map_entry)] + if !self.satonomics_by_date.contains_key(&chunk_id) { + self.satonomics_by_date + .insert(chunk_id, Satonomics::fetch_date_prices(chunk_id)?); + } + + self.satonomics_by_date + .get(&chunk_id) + .unwrap() + .get(date) + .cloned() + .ok_or(Error::msg("Couldn't find date in satonomics")) + } + fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result { if self.kraken_daily.is_none() { - self.kraken_daily.replace( - Kraken::fetch_daily_prices() - .unwrap_or_else(|_| Binance::fetch_daily_prices().unwrap()), - ); + self.kraken_daily + .replace(Kraken::fetch_daily_prices().or_else(|_| Binance::fetch_daily_prices())?); } self.kraken_daily @@ -325,7 +347,7 @@ impl PriceDatasets { .unwrap() .get(date) .cloned() - .ok_or(Error::msg("Couldn't find date in daily kraken")) + .ok_or(Error::msg("Couldn't find date")) } pub fn get_height_ohlc( @@ -358,15 +380,17 @@ impl PriceDatasets { let previous_timestamp = previous_timestamp.map(clean_timestamp); let ohlc = self - .get_from_1mn_kraken(timestamp, previous_timestamp) + .get_from_height_satonomics(&height) .unwrap_or_else(|_| { - self.get_from_1mn_binance(timestamp, previous_timestamp) + self.get_from_1mn_kraken(timestamp, previous_timestamp) .unwrap_or_else(|_| { - self.get_from_har_binance(timestamp, previous_timestamp) + self.get_from_1mn_binance(timestamp, previous_timestamp) .unwrap_or_else(|_| { - let date = Date::from_timestamp(timestamp); + self.get_from_har_binance(timestamp, previous_timestamp) + .unwrap_or_else(|_| { + let date = Date::from_timestamp(timestamp); - panic!( + panic!( "Can't find the price for: height: {height} - date: {date} 1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's How to fix this: @@ -381,6 +405,7 @@ How to fix this: 9. Move the file to 'parser/imports/binance.har' " ) + }) }) }) }); @@ -390,6 +415,23 @@ How to fix this: Ok(ohlc) } + fn get_from_height_satonomics(&mut self, height: &Height) -> color_eyre::Result { + let chunk_id = height.to_chunk_id(); + + #[allow(clippy::map_entry)] + if !self.satonomics_by_height.contains_key(&chunk_id) { + self.satonomics_by_height + .insert(chunk_id, Satonomics::fetch_height_prices(chunk_id)?); + } + + self.satonomics_by_height + .get(&chunk_id) + .unwrap() + .get(height.to_serialized_key().to_usize()) + .cloned() + .ok_or(Error::msg("Couldn't find height in satonomics")) + } + fn get_from_1mn_kraken( &mut self, timestamp: u32, diff --git a/parser/src/price/binance.rs b/parser/src/price/binance.rs index 2520d9690..6c8934f2b 100644 --- a/parser/src/price/binance.rs +++ b/parser/src/price/binance.rs @@ -107,7 +107,7 @@ impl Binance { log("binance: fetch 1mn"); retry( - || { + |_| { let body: Value = reqwest::blocking::get( "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000", )? @@ -154,7 +154,7 @@ impl Binance { log("binance: fetch 1d"); retry( - || { + |_| { let body: Value = reqwest::blocking::get( "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d", )? diff --git a/parser/src/price/kraken.rs b/parser/src/price/kraken.rs index 09c3f896a..cd29a8e5b 100644 --- a/parser/src/price/kraken.rs +++ b/parser/src/price/kraken.rs @@ -16,7 +16,7 @@ impl Kraken { log("kraken: fetch 1mn"); retry( - || { + |_| { let body: Value = reqwest::blocking::get( "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1", )? @@ -70,7 +70,7 @@ impl Kraken { log("fetch kraken daily"); retry( - || { + |_| { let body: Value = reqwest::blocking::get( "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440", )? diff --git a/parser/src/price/mod.rs b/parser/src/price/mod.rs index 6b8c8c847..dc0269af7 100644 --- a/parser/src/price/mod.rs +++ b/parser/src/price/mod.rs @@ -1,5 +1,7 @@ mod binance; mod kraken; +mod satonomics; pub use binance::*; pub use kraken::*; +pub use satonomics::*; diff --git a/parser/src/price/satonomics.rs b/parser/src/price/satonomics.rs new file mode 100644 index 000000000..2453b9481 --- /dev/null +++ b/parser/src/price/satonomics.rs @@ -0,0 +1,113 @@ +use std::{collections::BTreeMap, str::FromStr}; + +use chrono::NaiveDate; +use color_eyre::eyre::ContextCompat; +use itertools::Itertools; +use serde_json::Value; + +use crate::{ + datasets::OHLC, + structs::{Date, DateMapChunkId, HeightMapChunkId}, + utils::{log, retry}, + MapChunkId, +}; + +pub struct Satonomics; + +const SATONOMICS_OFFICIAL_URL: &str = "https://api.satonomics.xyz"; +const SATONOMICS_OFFICIAL_BACKUP_URL: &str = "https://api-bkp.satonomics.xyz"; + +const RETRIES: usize = 10; + +impl Satonomics { + fn get_base_url(try_index: usize) -> &'static str { + if try_index < RETRIES / 2 { + SATONOMICS_OFFICIAL_URL + } else { + SATONOMICS_OFFICIAL_BACKUP_URL + } + } + + pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result> { + log("satonomics: fetch height prices"); + + retry( + |try_index| { + let base_url = Self::get_base_url(try_index); + + let body: Value = reqwest::blocking::get(format!( + "{base_url}/height-to-price?chunk={}", + chunk_id.to_usize() + ))? + .json()?; + + Ok(body + .as_object() + .context("Expect to be an object")? + .get("dataset") + .context("Expect object to have dataset")? + .as_object() + .context("Expect to be an object")? + .get("map") + .context("Expect to have map")? + .as_array() + .context("Expect to be an array")? + .iter() + .map(Self::value_to_ohlc) + .collect_vec()) + }, + 10, + RETRIES, + ) + } + + pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result> { + log("satonomics: date height prices"); + + retry( + |try_index| { + let base_url = Self::get_base_url(try_index); + + let body: Value = reqwest::blocking::get(format!( + "{base_url}/date-to-price?chunk={}", + chunk_id.to_usize() + ))? + .json()?; + + Ok(body + .as_object() + .context("Expect to be an object")? + .get("dataset") + .context("Expect object to have dataset")? + .as_object() + .context("Expect to be an object")? + .get("map") + .context("Expect to have map")? + .as_object() + .context("Expect to be an object")? + .iter() + .map(|(serialized_date, value)| { + let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap()); + + (date, Self::value_to_ohlc(value)) + }) + .collect::>()) + }, + 10, + RETRIES, + ) + } + + fn value_to_ohlc(value: &Value) -> OHLC { + let ohlc = value.as_object().unwrap(); + + let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32; + + OHLC { + open: get_value("open"), + high: get_value("high"), + low: get_value("low"), + close: get_value("close"), + } + } +} diff --git a/parser/src/utils/retry.rs b/parser/src/utils/retry.rs index f5dee64b8..6f22e968c 100644 --- a/parser/src/utils/retry.rs +++ b/parser/src/utils/retry.rs @@ -1,9 +1,9 @@ use std::{thread::sleep, time::Duration}; pub fn retry( - function: impl Fn() -> color_eyre::Result, + function: impl Fn(usize) -> color_eyre::Result, sleep_in_s: u64, - retries: u64, + retries: usize, ) -> color_eyre::Result { if retries < 1 { unreachable!() @@ -12,7 +12,7 @@ pub fn retry( let mut i = 0; loop { - let res = function(); + let res = function(i); if i == retries || res.is_ok() { return res;