parser: add auto fetch price from main instance

.gitignore

@@ -8,3 +8,4 @@ TODO.md
 
 .stfolder
 /charts
+/price

CHANGELOG.md

@@ -8,6 +8,7 @@
   - Improved self-hosting by:
     - Fixing an incredibly annoying bug that made the program panic because of a wrong utxo/address durable state after a or many new datasets were added/changed after a first successful parse of the chain
     - Fixing a bug that would crash the program if launched for the first time ever
+    - Auto fetch prices from the main Satonomics instance if missing instead of only trying Kraken's and Binance's API which are limited to the last 16 hours
   - Merged the core of `HeightMap` and `DateMap` structs into `GenericMap`
   - Added `Height` struct and many others
 - CLI
@@ -70,6 +71,7 @@
 
 - Run file
   - Only run with a watcher if `cargo watch` is available
+- Added trigger folder to automatically restart when a new dataset has been added in the parser
 
 ## v. 0.2.0 | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
 

parser/src/bitcoin/daemon.rs

@@ -96,7 +96,7 @@ impl BitcoinDaemon {
 
     fn get_blockchain_info(&self) -> BlockchainInfo {
         retry(
-            || {
+            |_| {
                 // bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo
                 let output = Command::new("bitcoin-cli")
                     .arg(self.datadir_arg())
@@ -122,7 +122,7 @@ impl BitcoinDaemon {
                 Ok(BlockchainInfo { headers, blocks })
             },
             1,
-            u64::MAX,
+            usize::MAX,
         )
         .unwrap()
     }

parser/src/datasets/price/mod.rs

@@ -9,8 +9,11 @@ use color_eyre::eyre::Error;
 pub use ohlc::*;
 
 use crate::{
-    price::{Binance, Kraken},
+    price::{Binance, Kraken, Satonomics},
-    structs::{AnyBiMap, AnyDateMap, BiMap, Date, DateMap, Height, MapKey},
+    structs::{
+        AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId,
+        MapKey,
+    },
     utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
 };
 
@@ -24,7 +27,8 @@ pub struct PriceDatasets {
     kraken_1mn: Option<BTreeMap<u32, OHLC>>,
     binance_1mn: Option<BTreeMap<u32, OHLC>>,
     binance_har: Option<BTreeMap<u32, OHLC>>,
-    satonomics_by_height: BTreeMap<usize, Option<BTreeMap<usize, OHLC>>>,
+    satonomics_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
+    satonomics_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
 
     // Inserted
     pub ohlcs: BiMap<OHLC>,
@@ -89,6 +93,7 @@ impl PriceDatasets {
             kraken_1mn: None,
             kraken_daily: None,
             satonomics_by_height: BTreeMap::default(),
+            satonomics_by_date: BTreeMap::default(),
 
             ohlcs: BiMap::new_json(1, price_path),
             closes: BiMap::new_bin(1, &f("close")),
@@ -304,7 +309,9 @@ impl PriceDatasets {
         if self.ohlcs.date.is_key_safe(date) {
             Ok(self.ohlcs.date.get(&date).unwrap().to_owned())
         } else {
-            let ohlc = self.get_from_daily_kraken(&date)?;
+            let ohlc = self
+                .get_from_date_satonomics(&date)
+                .or_else(|_| self.get_from_daily_kraken(&date))?;
 
             self.ohlcs.date.insert(date, ohlc);
 
@@ -312,12 +319,27 @@ impl PriceDatasets {
         }
     }
 
+    fn get_from_date_satonomics(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
+        let chunk_id = date.to_chunk_id();
+
+        #[allow(clippy::map_entry)]
+        if !self.satonomics_by_date.contains_key(&chunk_id) {
+            self.satonomics_by_date
+                .insert(chunk_id, Satonomics::fetch_date_prices(chunk_id)?);
+        }
+
+        self.satonomics_by_date
+            .get(&chunk_id)
+            .unwrap()
+            .get(date)
+            .cloned()
+            .ok_or(Error::msg("Couldn't find date in satonomics"))
+    }
+
     fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
         if self.kraken_daily.is_none() {
-            self.kraken_daily.replace(
+            self.kraken_daily
-                Kraken::fetch_daily_prices()
+                .replace(Kraken::fetch_daily_prices().or_else(|_| Binance::fetch_daily_prices())?);
-                    .unwrap_or_else(|_| Binance::fetch_daily_prices().unwrap()),
-            );
         }
 
         self.kraken_daily
@@ -325,7 +347,7 @@ impl PriceDatasets {
             .unwrap()
             .get(date)
             .cloned()
-            .ok_or(Error::msg("Couldn't find date in daily kraken"))
+            .ok_or(Error::msg("Couldn't find date"))
     }
 
     pub fn get_height_ohlc(
@@ -358,15 +380,17 @@ impl PriceDatasets {
         let previous_timestamp = previous_timestamp.map(clean_timestamp);
 
         let ohlc = self
-            .get_from_1mn_kraken(timestamp, previous_timestamp)
+            .get_from_height_satonomics(&height)
             .unwrap_or_else(|_| {
-                self.get_from_1mn_binance(timestamp, previous_timestamp)
+                self.get_from_1mn_kraken(timestamp, previous_timestamp)
                     .unwrap_or_else(|_| {
-                        self.get_from_har_binance(timestamp, previous_timestamp)
+                        self.get_from_1mn_binance(timestamp, previous_timestamp)
                             .unwrap_or_else(|_| {
-                                let date = Date::from_timestamp(timestamp);
+                                self.get_from_har_binance(timestamp, previous_timestamp)
+                                    .unwrap_or_else(|_| {
+                                        let date = Date::from_timestamp(timestamp);
 
-                                panic!(
+                                        panic!(
                                     "Can't find the price for: height: {height} - date: {date}
 1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
 How to fix this:
@@ -381,6 +405,7 @@ How to fix this:
 9. Move the file to 'parser/imports/binance.har'
 "
                                 )
+                                    })
                             })
                     })
             });
@@ -390,6 +415,23 @@ How to fix this:
         Ok(ohlc)
     }
 
+    fn get_from_height_satonomics(&mut self, height: &Height) -> color_eyre::Result<OHLC> {
+        let chunk_id = height.to_chunk_id();
+
+        #[allow(clippy::map_entry)]
+        if !self.satonomics_by_height.contains_key(&chunk_id) {
+            self.satonomics_by_height
+                .insert(chunk_id, Satonomics::fetch_height_prices(chunk_id)?);
+        }
+
+        self.satonomics_by_height
+            .get(&chunk_id)
+            .unwrap()
+            .get(height.to_serialized_key().to_usize())
+            .cloned()
+            .ok_or(Error::msg("Couldn't find height in satonomics"))
+    }
+
     fn get_from_1mn_kraken(
         &mut self,
         timestamp: u32,

parser/src/price/binance.rs

@@ -107,7 +107,7 @@ impl Binance {
         log("binance: fetch 1mn");
 
         retry(
-            || {
+            |_| {
                 let body: Value = reqwest::blocking::get(
                     "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000",
                 )?
@@ -154,7 +154,7 @@ impl Binance {
         log("binance: fetch 1d");
 
         retry(
-            || {
+            |_| {
                 let body: Value = reqwest::blocking::get(
                     "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d",
                 )?

parser/src/price/kraken.rs

@@ -16,7 +16,7 @@ impl Kraken {
         log("kraken: fetch 1mn");
 
         retry(
-            || {
+            |_| {
                 let body: Value = reqwest::blocking::get(
                     "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1",
                 )?
@@ -70,7 +70,7 @@ impl Kraken {
         log("fetch kraken daily");
 
         retry(
-            || {
+            |_| {
                 let body: Value = reqwest::blocking::get(
                     "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440",
                 )?

parser/src/price/mod.rs

@@ -1,5 +1,7 @@
 mod binance;
 mod kraken;
+mod satonomics;
 
 pub use binance::*;
 pub use kraken::*;
+pub use satonomics::*;

parser/src/price/satonomics.rs

@@ -0,0 +1,113 @@
+use std::{collections::BTreeMap, str::FromStr};
+
+use chrono::NaiveDate;
+use color_eyre::eyre::ContextCompat;
+use itertools::Itertools;
+use serde_json::Value;
+
+use crate::{
+    datasets::OHLC,
+    structs::{Date, DateMapChunkId, HeightMapChunkId},
+    utils::{log, retry},
+    MapChunkId,
+};
+
+pub struct Satonomics;
+
+const SATONOMICS_OFFICIAL_URL: &str = "https://api.satonomics.xyz";
+const SATONOMICS_OFFICIAL_BACKUP_URL: &str = "https://api-bkp.satonomics.xyz";
+
+const RETRIES: usize = 10;
+
+impl Satonomics {
+    fn get_base_url(try_index: usize) -> &'static str {
+        if try_index < RETRIES / 2 {
+            SATONOMICS_OFFICIAL_URL
+        } else {
+            SATONOMICS_OFFICIAL_BACKUP_URL
+        }
+    }
+
+    pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
+        log("satonomics: fetch height prices");
+
+        retry(
+            |try_index| {
+                let base_url = Self::get_base_url(try_index);
+
+                let body: Value = reqwest::blocking::get(format!(
+                    "{base_url}/height-to-price?chunk={}",
+                    chunk_id.to_usize()
+                ))?
+                .json()?;
+
+                Ok(body
+                    .as_object()
+                    .context("Expect to be an object")?
+                    .get("dataset")
+                    .context("Expect object to have dataset")?
+                    .as_object()
+                    .context("Expect to be an object")?
+                    .get("map")
+                    .context("Expect to have map")?
+                    .as_array()
+                    .context("Expect to be an array")?
+                    .iter()
+                    .map(Self::value_to_ohlc)
+                    .collect_vec())
+            },
+            10,
+            RETRIES,
+        )
+    }
+
+    pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
+        log("satonomics: date height prices");
+
+        retry(
+            |try_index| {
+                let base_url = Self::get_base_url(try_index);
+
+                let body: Value = reqwest::blocking::get(format!(
+                    "{base_url}/date-to-price?chunk={}",
+                    chunk_id.to_usize()
+                ))?
+                .json()?;
+
+                Ok(body
+                    .as_object()
+                    .context("Expect to be an object")?
+                    .get("dataset")
+                    .context("Expect object to have dataset")?
+                    .as_object()
+                    .context("Expect to be an object")?
+                    .get("map")
+                    .context("Expect to have map")?
+                    .as_object()
+                    .context("Expect to be an object")?
+                    .iter()
+                    .map(|(serialized_date, value)| {
+                        let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap());
+
+                        (date, Self::value_to_ohlc(value))
+                    })
+                    .collect::<BTreeMap<_, _>>())
+            },
+            10,
+            RETRIES,
+        )
+    }
+
+    fn value_to_ohlc(value: &Value) -> OHLC {
+        let ohlc = value.as_object().unwrap();
+
+        let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32;
+
+        OHLC {
+            open: get_value("open"),
+            high: get_value("high"),
+            low: get_value("low"),
+            close: get_value("close"),
+        }
+    }
+}

parser/src/utils/retry.rs

@@ -1,9 +1,9 @@
 use std::{thread::sleep, time::Duration};
 
 pub fn retry<T>(
-    function: impl Fn() -> color_eyre::Result<T>,
+    function: impl Fn(usize) -> color_eyre::Result<T>,
     sleep_in_s: u64,
-    retries: u64,
+    retries: usize,
 ) -> color_eyre::Result<T> {
     if retries < 1 {
         unreachable!()
@@ -12,7 +12,7 @@ pub fn retry<T>(
     let mut i = 0;
 
     loop {
-        let res = function();
+        let res = function(i);
 
         if i == retries || res.is_ok() {
             return res;