diff --git a/crates/brk_core/src/structs/dateindex.rs b/crates/brk_core/src/structs/dateindex.rs index 47d927374..38796e88e 100644 --- a/crates/brk_core/src/structs/dateindex.rs +++ b/crates/brk_core/src/structs/dateindex.rs @@ -1,4 +1,7 @@ -use std::ops::Add; +use std::{ + fmt, + ops::{Add, Rem}, +}; use serde::Serialize; // use color_eyre::eyre::eyre; @@ -77,3 +80,16 @@ impl CheckedSub for DateIndex { self.0.checked_sub(rhs.0).map(Self) } } + +impl Rem for DateIndex { + type Output = Self; + fn rem(self, rhs: usize) -> Self::Output { + Self(self.0 % rhs as u16) + } +} + +impl fmt::Display for DateIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index 27fa33cf6..fc9bb620d 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -1,11 +1,15 @@ -use brk_core::Date; -use brk_fetcher::Fetcher; +use brk_core::{Date, Height}; +use brk_fetcher::{BRK, Fetcher}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; brk_logger::init(None); + let mut brk = BRK::default(); + dbg!(brk.get_from_height(Height::new(900_000))?); + dbg!(brk.get_from_date(Date::new(2025, 6, 7))?); + let mut fetcher = Fetcher::import(None)?; dbg!(fetcher.get_date(Date::new(2025, 6, 5))?); diff --git a/crates/brk_fetcher/src/fetchers/brk.rs b/crates/brk_fetcher/src/fetchers/brk.rs new file mode 100644 index 000000000..88333b87e --- /dev/null +++ b/crates/brk_fetcher/src/fetchers/brk.rs @@ -0,0 +1,143 @@ +use std::collections::BTreeMap; + +use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents}; +use color_eyre::eyre::{ContextCompat, eyre}; +use log::info; +use serde_json::Value; + +use crate::{Close, Dollars, High, Low, Open, fetchers::retry}; + +#[derive(Default, Clone)] +pub struct BRK { + height_to_ohlc: BTreeMap>, + dateindex_to_ohlc: BTreeMap>, +} + +const API_URL: &str = "https://bitcoinresearchkit.org/api"; + +const RETRIES: usize = 10; + +const CHUNK_SIZE: usize = 10_000; + +impl BRK { + pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result { + let key = height.checked_sub(height % CHUNK_SIZE).unwrap(); + + #[allow(clippy::map_entry)] + if !self.height_to_ohlc.contains_key(&key) + || ((key + self.height_to_ohlc.get(&key).unwrap().len()) <= height) + { + self.height_to_ohlc.insert( + key, + Self::fetch_height_prices(key).inspect_err(|e| { + dbg!(e); + })?, + ); + } + + self.height_to_ohlc + .get(&key) + .unwrap() + .get(usize::from(height.checked_sub(key).unwrap())) + .cloned() + .ok_or(eyre!("Couldn't find height in kibo")) + } + + fn fetch_height_prices(height: Height) -> color_eyre::Result> { + info!("Fetching Kibo height {height} prices..."); + + retry( + |_| { + let url = format!( + "{API_URL}/query?index=height&values=ohlc&from={}&to={}", + height, + height + CHUNK_SIZE + ); + + let body: Value = minreq::get(url).send()?.json()?; + + body.as_array() + .context("Expect to be an array")? + .iter() + .map(Self::value_to_ohlc) + .collect::, _>>() + }, + 30, + RETRIES, + ) + } + + pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result { + let dateindex = DateIndex::try_from(date)?; + + let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap(); + + #[allow(clippy::map_entry)] + if !self.dateindex_to_ohlc.contains_key(&key) + || ((key + self.dateindex_to_ohlc.get(&key).unwrap().len()) <= dateindex) + { + self.dateindex_to_ohlc.insert( + key, + Self::fetch_date_prices(key).inspect_err(|e| { + dbg!(e); + })?, + ); + } + + self.dateindex_to_ohlc + .get(&key) + .unwrap() + .get(usize::from(dateindex.checked_sub(key).unwrap())) + .cloned() + .ok_or(eyre!("Couldn't find date in kibo")) + } + + fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result> { + info!("Fetching Kibo dateindex {dateindex} prices..."); + + retry( + |_| { + let url = format!( + "{API_URL}/query?index=dateindex&values=ohlc&from={}&to={}", + dateindex, + dateindex + CHUNK_SIZE + ); + + let body: Value = minreq::get(url).send()?.json()?; + + body.as_array() + .context("Expect to be an array")? + .iter() + .map(Self::value_to_ohlc) + .collect::, _>>() + }, + 30, + RETRIES, + ) + } + + fn value_to_ohlc(value: &Value) -> color_eyre::Result { + let ohlc = value.as_array().context("Expect as_array to work")?; + + let get_value = |index: usize| -> color_eyre::Result<_> { + Ok(Cents::from(Dollars::from( + ohlc.get(index) + .context("Expect index key to work")? + .as_f64() + .context("Expect as_f64 to work")?, + ))) + }; + + Ok(OHLCCents::from(( + Open::new(get_value(0)?), + High::new(get_value(1)?), + Low::new(get_value(2)?), + Close::new(get_value(3)?), + ))) + } + + pub fn clear(&mut self) { + self.height_to_ohlc.clear(); + self.dateindex_to_ohlc.clear(); + } +} diff --git a/crates/brk_fetcher/src/fetchers/kibo.rs b/crates/brk_fetcher/src/fetchers/kibo.rs deleted file mode 100644 index aa3eaafc3..000000000 --- a/crates/brk_fetcher/src/fetchers/kibo.rs +++ /dev/null @@ -1,155 +0,0 @@ -use std::{collections::BTreeMap, str::FromStr}; - -use brk_core::{CheckedSub, Date, Height, OHLCCents}; -use color_eyre::eyre::{ContextCompat, eyre}; -use log::info; -use serde_json::Value; - -use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry}; - -#[derive(Default, Clone)] -pub struct Kibo { - height_to_ohlc_vec: BTreeMap>, - year_to_date_to_ohlc: BTreeMap>, -} - -const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api"; - -const RETRIES: usize = 10; - -impl Kibo { - pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result { - let key = height.checked_sub(height % 10_000).unwrap_or_default(); - - #[allow(clippy::map_entry)] - if !self.height_to_ohlc_vec.contains_key(&key) - || ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height) - { - self.height_to_ohlc_vec.insert( - key, - Self::fetch_height_prices(key).inspect_err(|e| { - dbg!(e); - })?, - ); - } - - self.height_to_ohlc_vec - .get(&key) - .unwrap() - .get(usize::from(height.checked_sub(key).unwrap())) - .cloned() - .ok_or(eyre!("Couldn't find height in kibo")) - } - - fn fetch_height_prices(height: Height) -> color_eyre::Result> { - info!("Fetching Kibo height {height} prices..."); - - retry( - |_| { - let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height); - - let body: Value = minreq::get(url).send()?.json()?; - - body.as_object() - .context("Expect to be an object")? - .get("dataset") - .context("Expect object to have dataset")? - .as_object() - .context("Expect to be an object")? - .get("map") - .context("Expect to have map")? - .as_array() - .context("Expect to be an array")? - .iter() - .map(Self::value_to_ohlc) - .collect::, _>>() - }, - 30, - RETRIES, - ) - } - - pub fn get_from_date(&mut self, date: &Date) -> color_eyre::Result { - let year = date.year(); - - #[allow(clippy::map_entry)] - if !self.year_to_date_to_ohlc.contains_key(&year) - || self - .year_to_date_to_ohlc - .get(&year) - .unwrap() - .last_key_value() - .unwrap() - .0 - <= date - { - self.year_to_date_to_ohlc - .insert(year, Self::fetch_date_prices(year)?); - } - - self.year_to_date_to_ohlc - .get(&year) - .unwrap() - .get(date) - .cloned() - .ok_or(eyre!("Couldn't find date in kibo")) - } - - fn fetch_date_prices(year: u16) -> color_eyre::Result> { - info!("Fetching Kibo date {year} prices..."); - - retry( - |_| { - let body: Value = - minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year)) - .send()? - .json()?; - - body.as_object() - .context("Expect to be an object")? - .get("dataset") - .context("Expect object to have dataset")? - .as_object() - .context("Expect to be an object")? - .get("map") - .context("Expect to have map")? - .as_object() - .context("Expect to be an object")? - .iter() - .map(|(serialized_date, value)| -> color_eyre::Result<_> { - let date = - Date::from(jiff::civil::Date::from_str(serialized_date).unwrap()); - Ok((date, Self::value_to_ohlc(value)?)) - }) - .collect::, _>>() - }, - 30, - RETRIES, - ) - } - - fn value_to_ohlc(value: &Value) -> color_eyre::Result { - let ohlc = value.as_object().context("Expect as_object to work")?; - - let get_value = |key: &str| -> color_eyre::Result<_> { - Ok(Cents::from(Dollars::from( - ohlc.get(key) - .context("Expect get key to work")? - .as_f64() - .context("Expect as_f64 to work")?, - ))) - }; - - Ok(OHLCCents::from(( - Open::new(get_value("open")?), - High::new(get_value("high")?), - Low::new(get_value("low")?), - Close::new(get_value("close")?), - ))) - } - - pub fn clear(&mut self) { - self.height_to_ohlc_vec.clear(); - self.year_to_date_to_ohlc.clear(); - } -} diff --git a/crates/brk_fetcher/src/fetchers/mod.rs b/crates/brk_fetcher/src/fetchers/mod.rs index 27770f09d..42822ce4e 100644 --- a/crates/brk_fetcher/src/fetchers/mod.rs +++ b/crates/brk_fetcher/src/fetchers/mod.rs @@ -1,9 +1,9 @@ mod binance; -// mod kibo; +mod brk; mod kraken; mod retry; pub use binance::*; -// pub use kibo::*; +pub use brk::*; pub use kraken::*; use retry::*; diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index 375ecb133..88a284ed5 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -10,7 +10,7 @@ use color_eyre::eyre::Error; mod fetchers; -use fetchers::*; +pub use fetchers::*; use log::info; const TRIES: usize = 12 * 60; @@ -19,7 +19,7 @@ const TRIES: usize = 12 * 60; pub struct Fetcher { binance: Binance, kraken: Kraken, - // kibo: Kibo, + brk: BRK, } impl Fetcher { @@ -31,7 +31,7 @@ impl Fetcher { Ok(Self { binance: Binance::init(hars_path), kraken: Kraken::default(), - // kibo: Kibo::default(), + brk: BRK::default(), }) } @@ -46,6 +46,10 @@ impl Fetcher { // eprintln!("{e}"); self.kraken.get_from_1d(&date) }) + .or_else(|_| { + // eprintln!("{e}"); + self.brk.get_from_date(date) + }) .or_else(|e| { sleep(Duration::from_secs(60)); @@ -94,28 +98,28 @@ impl Fetcher { .get_from_1mn(timestamp, previous_timestamp) .unwrap_or_else(|_report| { // // eprintln!("{_report}"); - // self.kibo.get_from_height(height).unwrap_or_else(|_report| { - // eprintln!("{_report}"); + self.brk.get_from_height(height).unwrap_or_else(|_report| { + // eprintln!("{_report}"); - sleep(Duration::from_secs(60)); + sleep(Duration::from_secs(60)); - if tries < TRIES { - self.clear(); + if tries < TRIES { + self.clear(); - info!("Retrying to fetch height prices..."); - // dbg!((height, timestamp, previous_timestamp)); + info!("Retrying to fetch height prices..."); + // dbg!((height, timestamp, previous_timestamp)); - return self - .get_height_(height, timestamp, previous_timestamp, tries + 1) - .unwrap(); - } + return self + .get_height_(height, timestamp, previous_timestamp, tries + 1) + .unwrap(); + } - info!("Failed to fetch height prices"); + info!("Failed to fetch height prices"); - let date = Date::from(timestamp); - // eprintln!("{e}"); - panic!( - " + let date = Date::from(timestamp); + // eprintln!("{e}"); + panic!( + " Can't find the price for: height: {height} - date: {date} 1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's How to fix this: @@ -130,8 +134,8 @@ How to fix this: 8. Export to a har file (if there is no explicit button, click on the cog button) 9. Move the file to 'parser/imports/binance.har' " - ) - // }) + ) + }) }) }); @@ -182,7 +186,7 @@ How to fix this: pub fn clear(&mut self) { self.binance.clear(); - // self.kibo.clear(); + self.brk.clear(); self.kraken.clear(); } }