diff --git a/.gitignore b/.gitignore index 749a5981f..70eb0fa00 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ docker/kibo # Types paths.d.ts +vecid-to-indexes.d.ts # Outputs _outputs diff --git a/Cargo.lock b/Cargo.lock index 739206fa0..c57d7887d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2118,6 +2118,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "storable_vec", "zerocopy 0.8.17", ] diff --git a/computer/src/structs/mod.rs b/computer/src/structs/mod.rs index 7e26c175a..7edf1230f 100644 --- a/computer/src/structs/mod.rs +++ b/computer/src/structs/mod.rs @@ -1,11 +1,11 @@ mod addressindextxoutindex; mod bitcoin; mod feerate; -mod ohlc; +// mod ohlc; mod unit; pub use addressindextxoutindex::*; pub use bitcoin::*; pub use feerate::*; -pub use ohlc::*; +// pub use ohlc::*; pub use unit::*; diff --git a/pricer/Cargo.toml b/pricer/Cargo.toml index c9db534d0..82473bf62 100644 --- a/pricer/Cargo.toml +++ b/pricer/Cargo.toml @@ -8,8 +8,16 @@ color-eyre = { workspace = true } derive_deref = { workspace = true } indexer = { workspace = true } logger = { workspace = true } -reqwest = { version = "0.12.12", features = ["blocking", "brotli", "deflate", "gzip", "json", "zstd"] } +reqwest = { version = "0.12.12", features = [ + "blocking", + "brotli", + "deflate", + "gzip", + "json", + "zstd", +] } jiff = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +storable_vec = { workspace = true } zerocopy = { workspace = true } diff --git a/pricer/src/fetchers/binance.rs b/pricer/src/fetchers/binance.rs index f8ecbb0f3..944e25688 100644 --- a/pricer/src/fetchers/binance.rs +++ b/pricer/src/fetchers/binance.rs @@ -2,7 +2,7 @@ use std::{ collections::BTreeMap, fs::{self, File}, io::BufReader, - path::Path, + path::{Path, PathBuf}, str::FromStr, }; @@ -10,17 +10,48 @@ use color_eyre::eyre::{eyre, ContextCompat}; use indexer::Timestamp; use logger::info; use serde_json::Value; +use storable_vec::STATELESS; use crate::{ fetchers::retry, structs::{Cents, OHLC}, - Close, Date, Dollars, High, Low, Open, + Close, Date, Dollars, High, Low, Open, Pricer, }; -pub struct Binance; +pub struct Binance { + path: PathBuf, + _1mn: Option>, + _1d: Option>, + har: Option>, +} impl Binance { - pub fn fetch_1mn_prices() -> color_eyre::Result> { + pub fn init(path: &Path) -> Self { + Self { + path: path.to_owned(), + _1mn: None, + _1d: None, + har: None, + } + } + + fn get_from_1mn( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { + self._1mn.replace(Self::fetch_1mn()?); + } + Pricer::::find_height_ohlc( + &self._1mn.as_ref().unwrap(), + timestamp, + previous_timestamp, + "binance 1mn", + ) + } + + pub fn fetch_1mn() -> color_eyre::Result> { info!("Fetching 1mn prices from Binance..."); retry( @@ -30,7 +61,20 @@ impl Binance { ) } - pub fn fetch_daily_prices() -> color_eyre::Result> { + pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date { + self._1d.replace(Self::fetch_1d()?); + } + + self._1d + .as_ref() + .unwrap() + .get(date) + .cloned() + .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) + } + + fn fetch_1d() -> color_eyre::Result> { info!("Fetching daily prices from Kraken..."); retry( @@ -40,10 +84,28 @@ impl Binance { ) } - pub fn read_har_file(path: &Path) -> color_eyre::Result> { + pub fn get_from_har_binance( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if self.har.is_none() { + self.har.replace(self.read_har().unwrap_or_default()); + } + Pricer::::find_height_ohlc( + &self.har.as_ref().unwrap(), + timestamp, + previous_timestamp, + "binance har", + ) + } + + fn read_har(&self) -> color_eyre::Result> { info!("Reading Binance har file..."); - fs::create_dir_all(&path)?; + let path = &self.path; + + fs::create_dir_all(path)?; let path_binance_har = path.join("binance.har"); diff --git a/pricer/src/fetchers/kibo.rs b/pricer/src/fetchers/kibo.rs index 8b137090b..8cefa2299 100644 --- a/pricer/src/fetchers/kibo.rs +++ b/pricer/src/fetchers/kibo.rs @@ -1,12 +1,21 @@ use std::{collections::BTreeMap, str::FromStr}; use color_eyre::eyre::ContextCompat; +use indexer::Height; use logger::info; use serde_json::Value; -use crate::structs::{Date, OHLC}; +use crate::{ + fetchers::retry, + structs::{Date, OHLC}, + Cents, Close, Dollars, High, Low, Open, +}; -pub struct Kibo; +#[derive(Default)] +pub struct Kibo { + height_to_ohlc_vec: BTreeMap>, + year_to_date_to_ohlc: BTreeMap>, +} const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api"; const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api"; @@ -22,16 +31,32 @@ impl Kibo { } } - pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result> { - info!("kibo: fetch height prices"); + pub fn get_from_height_kibo(&mut self, height: Height) -> color_eyre::Result { + #[allow(clippy::map_entry)] + if !self.height_to_ohlc_vec.contains_key(&height) + || ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len()) <= usize::from(height)) + { + self.height_to_ohlc_vec + .insert(height, Self::fetch_height_prices(height)?); + } + + self.height_to_ohlc_vec + .get(&height) + .unwrap() + .get(usize::from(height)) + .cloned() + .ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo")) + } + + fn fetch_height_prices(height: Height) -> color_eyre::Result> { + info!("Fetching Kibo height prices..."); retry( |try_index| { let base_url = Self::get_base_url(try_index); let body: Value = - reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))? - .json()?; + reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", height))?.json()?; let vec = body .as_object() @@ -55,19 +80,41 @@ impl Kibo { ) } - pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result> { - info!("kibo: fetch date prices"); + pub fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result { + let year = date.year(); + + #[allow(clippy::map_entry)] + if !self.year_to_date_to_ohlc.contains_key(&year) + || self + .year_to_date_to_ohlc + .get(&year) + .unwrap() + .last_key_value() + .unwrap() + .0 + < date + { + self.year_to_date_to_ohlc.insert(year, Self::fetch_date_prices(year)?); + } + + self.year_to_date_to_ohlc + .get(&year) + .unwrap() + .get(date) + .cloned() + .ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo")) + } + + fn fetch_date_prices(year: u16) -> color_eyre::Result> { + info!("Fetching Kibo date prices..."); retry( |try_index| { let base_url = Self::get_base_url(try_index); - let body: Value = - reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))? - .json()?; + let body: Value = reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", year))?.json()?; - Ok(body - .as_object() + body.as_object() .context("Expect to be an object")? .get("dataset") .context("Expect object to have dataset")? @@ -79,10 +126,10 @@ impl Kibo { .context("Expect to be an object")? .iter() .map(|(serialized_date, value)| -> color_eyre::Result<_> { - let date = Date::wrap(NaiveDate::from_str(serialized_date)?); + let date = Date::from(jiff::civil::Date::from_str(serialized_date).unwrap()); Ok((date, Self::value_to_ohlc(value)?)) }) - .collect::, _>>()?) + .collect::, _>>() }, 30, RETRIES, @@ -92,19 +139,20 @@ impl Kibo { fn value_to_ohlc(value: &Value) -> color_eyre::Result { let ohlc = value.as_object().context("Expect as_object to work")?; - let get_value = |key: &str| -> color_eyre::Result { - Ok(ohlc - .get(key) - .context("Expect get key to work")? - .as_f64() - .context("Expect as_f64 to work")? as f32) + let get_value = |key: &str| -> color_eyre::Result<_> { + Ok(Cents::from(Dollars::from( + ohlc.get(key) + .context("Expect get key to work")? + .as_f64() + .context("Expect as_f64 to work")?, + ))) }; - Ok(OHLC { - open: get_value("open")?, - high: get_value("high")?, - low: get_value("low")?, - close: get_value("close")?, - }) + Ok(( + Open::from(get_value("open")?), + High::from(get_value("high")?), + Low::from(get_value("low")?), + Close::from(get_value("close")?), + )) } } diff --git a/pricer/src/fetchers/kraken.rs b/pricer/src/fetchers/kraken.rs index 2f38725fa..4e8689756 100644 --- a/pricer/src/fetchers/kraken.rs +++ b/pricer/src/fetchers/kraken.rs @@ -4,13 +4,29 @@ use color_eyre::eyre::ContextCompat; use indexer::Timestamp; use logger::info; use serde_json::Value; +use storable_vec::STATELESS; -use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC}; +use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, Pricer, OHLC}; -pub struct Kraken; +#[derive(Default)] +pub struct Kraken { + _1mn: Option>, + _1d: Option>, +} impl Kraken { - pub fn fetch_1mn_prices() -> color_eyre::Result> { + pub fn get_from_1mn( + &mut self, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { + self._1mn.replace(Self::fetch_1mn()?); + } + Pricer::::find_height_ohlc(&self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m") + } + + fn fetch_1mn() -> color_eyre::Result> { info!("Fetching 1mn prices from Kraken..."); retry( @@ -20,7 +36,19 @@ impl Kraken { ) } - pub fn fetch_daily_prices() -> color_eyre::Result> { + pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result { + if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date { + self._1d.replace(Kraken::fetch_1d()?); + } + self._1d + .as_ref() + .unwrap() + .get(date) + .cloned() + .ok_or(color_eyre::eyre::Error::msg("Couldn't find date")) + } + + fn fetch_1d() -> color_eyre::Result> { info!("Fetching daily prices from Kraken..."); retry( diff --git a/pricer/src/lib.rs b/pricer/src/lib.rs index 01562e543..6d5f0f617 100644 --- a/pricer/src/lib.rs +++ b/pricer/src/lib.rs @@ -1,5 +1,277 @@ +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use color_eyre::eyre::Error; + mod fetchers; mod structs; pub use fetchers::*; +use indexer::{Height, Indexer, Timestamp}; +use storable_vec::{AnyJsonStorableVec, AnyStorableVec, StorableVec, Version, SINGLE_THREAD}; pub use structs::*; + +pub struct Pricer { + path: PathBuf, + binance: Binance, + kraken: Kraken, + kibo: Kibo, + + pub dateindex_to_close_in_cents: StorableVec, MODE>, + pub dateindex_to_close_in_dollars: StorableVec, MODE>, + pub dateindex_to_high_in_cents: StorableVec, MODE>, + pub dateindex_to_high_in_dollars: StorableVec, MODE>, + pub dateindex_to_low_in_cents: StorableVec, MODE>, + pub dateindex_to_low_in_dollars: StorableVec, MODE>, + pub dateindex_to_open_in_cents: StorableVec, MODE>, + pub dateindex_to_open_in_dollars: StorableVec, MODE>, + pub height_to_close_in_cents: StorableVec, MODE>, + pub height_to_close_in_dollars: StorableVec, MODE>, + pub height_to_high_in_cents: StorableVec, MODE>, + pub height_to_high_in_dollars: StorableVec, MODE>, + pub height_to_low_in_cents: StorableVec, MODE>, + pub height_to_low_in_dollars: StorableVec, MODE>, + pub height_to_open_in_cents: StorableVec, MODE>, + pub height_to_open_in_dollars: StorableVec, MODE>, +} + +impl Pricer { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + path: path.to_owned(), + binance: Binance::init(path), + kraken: Kraken::default(), + kibo: Kibo::default(), + + // binance_1mn: None, + // binance_daily: None, + // binance_har: None, + // kraken_1mn: None, + // kraken_daily: None, + // kibo_by_height: BTreeMap::default(), + // kibo_by_date: BTreeMap::default(), + dateindex_to_close_in_cents: StorableVec::import( + &path.join("dateindex_to_close_in_cents"), + Version::from(1), + )?, + dateindex_to_close_in_dollars: StorableVec::import( + &path.join("dateindex_to_close_in_dollars"), + Version::from(1), + )?, + dateindex_to_high_in_cents: StorableVec::import( + &path.join("dateindex_to_high_in_cents"), + Version::from(1), + )?, + dateindex_to_high_in_dollars: StorableVec::import( + &path.join("dateindex_to_high_in_dollars"), + Version::from(1), + )?, + dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?, + dateindex_to_low_in_dollars: StorableVec::import( + &path.join("dateindex_to_low_in_dollars"), + Version::from(1), + )?, + dateindex_to_open_in_cents: StorableVec::import( + &path.join("dateindex_to_open_in_cents"), + Version::from(1), + )?, + dateindex_to_open_in_dollars: StorableVec::import( + &path.join("dateindex_to_open_in_dollars"), + Version::from(1), + )?, + height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?, + height_to_close_in_dollars: StorableVec::import( + &path.join("height_to_close_in_dollars"), + Version::from(1), + )?, + height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?, + height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?, + height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?, + height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?, + height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?, + height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?, + }) + } + + pub fn compute_if_needed(&mut self, indexer: &mut Indexer) { + // TODO: Remove all outdated + + indexer + .vecs + .height_to_timestamp + .iter_from(Height::default(), |v| Ok(())); + + // self.open + // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open); + + // self.high + // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high); + + // self.low + // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low); + + // self.close + // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); + } + + fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { + if self.ohlc.date.is_key_safe(date) { + Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned()) + } else { + let ohlc = self + .get_from_daily_kraken(&date) + .or_else(|_| self.get_from_daily_binance(&date)) + .or_else(|_| self.get_from_date_kibo(&date))?; + + self.ohlc.date.insert(date, ohlc); + + Ok(ohlc) + } + } + + fn get_height_ohlc( + &mut self, + height: Height, + timestamp: Timestamp, + previous_timestamp: Option, + ) -> color_eyre::Result { + if let Some(ohlc) = self.ohlc.height.get_or_import(&height) { + return Ok(ohlc); + } + + let timestamp = timestamp.to_floored_seconds(); + + if previous_timestamp.is_none() && !height.is_first() { + panic!("Shouldn't be possible"); + } + + let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds()); + + let ohlc = self + .get_from_1mn_kraken(timestamp, previous_timestamp) + .unwrap_or_else(|_| { + self.get_from_1mn_binance(timestamp, previous_timestamp) + .unwrap_or_else(|_| { + self.get_from_har_binance(timestamp, previous_timestamp, config) + .unwrap_or_else(|_| { + self.get_from_height_kibo(&height).unwrap_or_else(|_| { + let date = timestamp.to_date(); + + panic!( + "Can't find the price for: height: {height} - date: {date} +1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's +How to fix this: +1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot +2. Select 1mn interval +3. Open the inspector/dev tools +4. Go to the Network Tab +5. Filter URLs by 'uiKlines' +6. Go back to the chart and scroll until you pass the date mentioned few lines ago +7. Go back to the dev tools +8. Export to a har file (if there is no explicit button, click on the cog button) +9. Move the file to 'parser/imports/binance.har' +" + ) + }) + }) + }) + }); + + // self.ohlc.height.insert(height, ohlc); + + Ok(ohlc) + } + + fn find_height_ohlc( + tree: &BTreeMap, + timestamp: Timestamp, + previous_timestamp: Option, + name: &str, + ) -> color_eyre::Result { + let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| { + tree.get(&previous_timestamp).cloned() + }); + + let last_ohlc = tree.get(×tamp); + + if previous_ohlc.is_none() || last_ohlc.is_none() { + return Err(Error::msg(format!("Couldn't find timestamp in {name}"))); + } + + let previous_ohlc = previous_ohlc.unwrap(); + + let mut final_ohlc = ( + Open::from(previous_ohlc.3), + High::from(previous_ohlc.3), + Low::from(previous_ohlc.3), + previous_ohlc.3, + ); + + let start = previous_timestamp.unwrap_or(Timestamp::from(0)); + let end = timestamp; + + // Otherwise it's a re-org + if start < end { + tree.range(start..=end).skip(1).for_each(|(_, ohlc)| { + if ohlc.1 > final_ohlc.1 { + final_ohlc.1 = ohlc.1 + } + + if ohlc.2 < final_ohlc.2 { + final_ohlc.2 = ohlc.2 + } + + final_ohlc.3 = ohlc.3; + }); + } + + Ok(final_ohlc) + } + + pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 16] { + [ + &self.dateindex_to_close_in_cents as &dyn AnyJsonStorableVec, + &self.dateindex_to_close_in_dollars, + &self.dateindex_to_high_in_cents, + &self.dateindex_to_high_in_dollars, + &self.dateindex_to_low_in_cents, + &self.dateindex_to_low_in_dollars, + &self.dateindex_to_open_in_cents, + &self.dateindex_to_open_in_dollars, + &self.height_to_close_in_cents, + &self.height_to_close_in_dollars, + &self.height_to_high_in_cents, + &self.height_to_high_in_dollars, + &self.height_to_low_in_cents, + &self.height_to_low_in_dollars, + &self.height_to_open_in_cents, + &self.height_to_open_in_dollars, + ] + } + + pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 16] { + [ + &mut self.dateindex_to_close_in_cents as &mut dyn AnyStorableVec, + &mut self.dateindex_to_close_in_dollars, + &mut self.dateindex_to_high_in_cents, + &mut self.dateindex_to_high_in_dollars, + &mut self.dateindex_to_low_in_cents, + &mut self.dateindex_to_low_in_dollars, + &mut self.dateindex_to_open_in_cents, + &mut self.dateindex_to_open_in_dollars, + &mut self.height_to_close_in_cents, + &mut self.height_to_close_in_dollars, + &mut self.height_to_high_in_cents, + &mut self.height_to_high_in_dollars, + &mut self.height_to_low_in_cents, + &mut self.height_to_low_in_dollars, + &mut self.height_to_open_in_cents, + &mut self.height_to_open_in_dollars, + ] + } +} diff --git a/pricer/src/main.rs b/pricer/src/main.rs index c73ec56b7..dcfafe61a 100644 --- a/pricer/src/main.rs +++ b/pricer/src/main.rs @@ -1,16 +1,19 @@ // fn main() {} -use pricer::{Binance, Kraken}; +use indexer::Height; +use pricer::{Binance, Kibo, Kraken}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; logger::init_log(None); - // dbg!(Binance::fetch_daily_prices()); + dbg!(Binance::fetch_1d_prices()?); // dbg!(Binance::fetch_1mn_prices()); - // dbg!(Kraken::fetch_daily_prices()); - dbg!(Kraken::fetch_1mn_prices()); + dbg!(Kraken::fetch_1d()?); + // dbg!(Kraken::fetch_1mn_prices()?); + dbg!(Kibo::fetch_date_prices(2025)?); + dbg!(Kibo::fetch_height_prices(Height::from(880_000_u32))?); Ok(()) } diff --git a/pricer/src/price.rs b/pricer/src/price.rs deleted file mode 100644 index 5c98da0a1..000000000 --- a/pricer/src/price.rs +++ /dev/null @@ -1,316 +0,0 @@ -use std::collections::BTreeMap; - -use allocative::Allocative; -use chrono::Days; -use color_eyre::eyre::Error; - -use struct_iterable::Iterable; - -use crate::{ - parser::price::{Binance, Kibo, Kraken}, - structs::{ - Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, MapKind, Timestamp, - OHLC, - }, - utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, -}; - -use super::{AnyDataset, ComputeData, MinInitialStates, RatioDataset}; - -#[derive(Allocative, Iterable)] -pub struct PriceDatasets { - min_initial_states: MinInitialStates, - - kraken_daily: Option>, - kraken_1mn: Option>, - binance_1mn: Option>, - binance_daily: Option>, - binance_har: Option>, - kibo_by_height: BTreeMap>, - kibo_by_date: BTreeMap>, - - pub ohlc: BiMap, - pub open: BiMap, - pub high: BiMap, - pub low: BiMap, - pub close: BiMap, -} - -impl PriceDatasets { - pub fn import(config: &Config) -> color_eyre::Result { - let path_dataset = config.path_datasets(); - let f = |s: &str| path_dataset.join(s); - - let mut s = Self { - min_initial_states: MinInitialStates::default(), - - binance_1mn: None, - binance_daily: None, - binance_har: None, - kraken_1mn: None, - kraken_daily: None, - kibo_by_height: BTreeMap::default(), - kibo_by_date: BTreeMap::default(), - - // --- - // Inserted - // --- - ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()), - - // --- - // Computed - // --- - open_cents: BiMap::new_bin(1, MapKind::Computed, &f("open")), - high_cents: BiMap::new_bin(1, MapKind::Computed, &f("high")), - low_cents: BiMap::new_bin(1, MapKind::Computed, &f("low")), - close: BiMap::new_bin(1, MapKind::Computed, &f("close")), - }; - - s.min_initial_states - .consume(MinInitialStates::compute_from_dataset(&s, config)); - - Ok(s) - } - - pub fn compute(&mut self, compute_data: &ComputeData, circulating_supply: &mut BiMap) { - let &ComputeData { dates, heights, .. } = compute_data; - - self.open - .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open); - - self.high - .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high); - - self.low - .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low); - - self.close - .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); - } - - pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { - if self.ohlc.date.is_key_safe(date) { - Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned()) - } else { - let ohlc = self - .get_from_daily_kraken(&date) - .or_else(|_| self.get_from_daily_binance(&date)) - .or_else(|_| self.get_from_date_kibo(&date))?; - - self.ohlc.date.insert(date, ohlc); - - Ok(ohlc) - } - } - - fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result { - let chunk_id = date.to_chunk_id(); - - #[allow(clippy::map_entry)] - if !self.kibo_by_date.contains_key(&chunk_id) - || self.kibo_by_date.get(&chunk_id).unwrap().last_key_value().unwrap().0 < date - { - self.kibo_by_date.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?); - } - - self.kibo_by_date - .get(&chunk_id) - .unwrap() - .get(date) - .cloned() - .ok_or(Error::msg("Couldn't find date in satonomics")) - } - - fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result { - if self.kraken_daily.is_none() || self.kraken_daily.as_ref().unwrap().last_key_value().unwrap().0 < date { - self.kraken_daily.replace(Kraken::fetch_daily_prices()?); - } - - self.kraken_daily - .as_ref() - .unwrap() - .get(date) - .cloned() - .ok_or(Error::msg("Couldn't find date")) - } - - fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result { - if self.binance_daily.is_none() || self.binance_daily.as_ref().unwrap().last_key_value().unwrap().0 < date { - self.binance_daily.replace(Binance::fetch_daily_prices()?); - } - - self.binance_daily - .as_ref() - .unwrap() - .get(date) - .cloned() - .ok_or(Error::msg("Couldn't find date")) - } - - pub fn get_height_ohlc( - &mut self, - height: Height, - timestamp: Timestamp, - previous_timestamp: Option, - config: &Config, - ) -> color_eyre::Result { - if let Some(ohlc) = self.ohlc.height.get_or_import(&height) { - return Ok(ohlc); - } - - let timestamp = timestamp.to_floored_seconds(); - - if previous_timestamp.is_none() && !height.is_first() { - panic!("Shouldn't be possible"); - } - - let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds()); - - let ohlc = self - .get_from_1mn_kraken(timestamp, previous_timestamp) - .unwrap_or_else(|_| { - self.get_from_1mn_binance(timestamp, previous_timestamp) - .unwrap_or_else(|_| { - self.get_from_har_binance(timestamp, previous_timestamp, config) - .unwrap_or_else(|_| { - self.get_from_height_kibo(&height).unwrap_or_else(|_| { - let date = timestamp.to_date(); - - panic!( - "Can't find the price for: height: {height} - date: {date} -1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's -How to fix this: -1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot -2. Select 1mn interval -3. Open the inspector/dev tools -4. Go to the Network Tab -5. Filter URLs by 'uiKlines' -6. Go back to the chart and scroll until you pass the date mentioned few lines ago -7. Go back to the dev tools -8. Export to a har file (if there is no explicit button, click on the cog button) -9. Move the file to 'parser/imports/binance.har' -" - ) - }) - }) - }) - }); - - self.ohlc.height.insert(height, ohlc); - - Ok(ohlc) - } - - fn get_from_height_kibo(&mut self, height: &Height) -> color_eyre::Result { - let chunk_id = height.to_chunk_id(); - - #[allow(clippy::map_entry)] - if !self.kibo_by_height.contains_key(&chunk_id) - || ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) <= height.to_usize()) - { - self.kibo_by_height - .insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?); - } - - self.kibo_by_height - .get(&chunk_id) - .unwrap() - .get(height.to_serialized_key().to_usize()) - .cloned() - .ok_or(Error::msg("Couldn't find height in kibo")) - } - - fn get_from_1mn_kraken( - &mut self, - timestamp: Timestamp, - previous_timestamp: Option, - ) -> color_eyre::Result { - if self.kraken_1mn.is_none() || self.kraken_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { - self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?); - } - - Self::find_height_ohlc(&self.kraken_1mn, timestamp, previous_timestamp, "kraken 1m") - } - - fn get_from_1mn_binance( - &mut self, - timestamp: Timestamp, - previous_timestamp: Option, - ) -> color_eyre::Result { - if self.binance_1mn.is_none() || self.binance_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { - self.binance_1mn.replace(Binance::fetch_1mn_prices()?); - } - - Self::find_height_ohlc(&self.binance_1mn, timestamp, previous_timestamp, "binance 1m") - } - - fn get_from_har_binance( - &mut self, - timestamp: Timestamp, - previous_timestamp: Option, - config: &Config, - ) -> color_eyre::Result { - if self.binance_har.is_none() { - self.binance_har - .replace(Binance::read_har_file(config).unwrap_or_default()); - } - - Self::find_height_ohlc(&self.binance_har, timestamp, previous_timestamp, "binance har") - } - - fn find_height_ohlc( - tree: &Option>, - timestamp: Timestamp, - previous_timestamp: Option, - name: &str, - ) -> color_eyre::Result { - let tree = tree.as_ref().unwrap(); - - let err = Error::msg(format!("Couldn't find timestamp in {name}")); - - let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| { - tree.get(&previous_timestamp).cloned() - }); - - let last_ohlc = tree.get(×tamp); - - if previous_ohlc.is_none() || last_ohlc.is_none() { - return Err(err); - } - - let previous_ohlc = previous_ohlc.unwrap(); - - let mut final_ohlc = OHLC { - open: previous_ohlc.close, - high: previous_ohlc.close, - low: previous_ohlc.close, - close: previous_ohlc.close, - }; - - let start = previous_timestamp.unwrap_or_default(); - let end = timestamp; - - // Otherwise it's a re-org - if start < end { - tree.range(&*start..=&*end).skip(1).for_each(|(_, ohlc)| { - if ohlc.high > final_ohlc.high { - final_ohlc.high = ohlc.high - } - - if ohlc.low < final_ohlc.low { - final_ohlc.low = ohlc.low - } - - final_ohlc.close = ohlc.close; - }); - } - - Ok(final_ohlc) - } -} - -impl AnyDataset for PriceDatasets { - fn get_min_initial_states(&self) -> &MinInitialStates { - &self.min_initial_states - } -} diff --git a/pricer/src/structs/cents.rs b/pricer/src/structs/cents.rs index 839c5fcea..6130b14f8 100644 --- a/pricer/src/structs/cents.rs +++ b/pricer/src/structs/cents.rs @@ -4,7 +4,22 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Dollars; -#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Deref, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Serialize, +)] pub struct Cents(u64); impl From for Cents { diff --git a/pricer/src/structs/close.rs b/pricer/src/structs/close.rs index c3a15b543..a87318ea2 100644 --- a/pricer/src/structs/close.rs +++ b/pricer/src/structs/close.rs @@ -2,7 +2,22 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] #[repr(C)] pub struct Close(T); impl From for Close { diff --git a/pricer/src/structs/date.rs b/pricer/src/structs/date.rs index e4c422888..766b661c1 100644 --- a/pricer/src/structs/date.rs +++ b/pricer/src/structs/date.rs @@ -1,18 +1,17 @@ -use std::ops::Add; - -use color_eyre::eyre::eyre; use indexer::Timestamp; use jiff::{civil::Date as Date_, tz::TimeZone, Span}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use super::Dateindex; + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct Date(u32); impl Date { - const INDEX_ZERO: Self = Self(20090103); - const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3); - const INDEX_ONE: Self = Self(20090109); - const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9); + pub const INDEX_ZERO: Self = Self(20090103); + pub const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3); + pub const INDEX_ONE: Self = Self(20090109); + pub const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9); pub fn year(&self) -> u16 { (self.0 / 1_00_00) as u16 @@ -51,33 +50,12 @@ impl From for Date { } } -// impl TryFrom for usize { -// type Error = color_eyre::Report; -// fn try_from(value: Date) -> Result { -// let value_ = Date_::from(value); -// if value_ < Date::INDEX_ZERO_ { -// Err(eyre!("Date is too early")) -// } else if value == Date::INDEX_ZERO { -// Ok(0) -// } else if value_ < Date::INDEX_ONE_ { -// Err(eyre!("Date is between first and second")) -// } else if value == Date::INDEX_ONE { -// Ok(1) -// } else { -// Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1) -// } -// } -// } - -// impl From for Date { -// fn from(value: usize) -> Self { -// Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap()) -// } -// } - -// impl Add for Date { -// type Output = Self; -// fn add(self, rhs: usize) -> Self::Output { -// Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap()) -// } -// } +impl From for Date { + fn from(value: Dateindex) -> Self { + Self::from( + Self::INDEX_ZERO_ + .checked_add(Span::new().days(i64::from(value))) + .unwrap(), + ) + } +} diff --git a/pricer/src/structs/dateindex.rs b/pricer/src/structs/dateindex.rs index 64338bed9..fc314e42e 100644 --- a/pricer/src/structs/dateindex.rs +++ b/pricer/src/structs/dateindex.rs @@ -1,7 +1,10 @@ use std::ops::Add; +use color_eyre::eyre::eyre; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use super::Date; + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct Dateindex(u16); @@ -17,9 +20,33 @@ impl From for Dateindex { } } +impl From for i64 { + fn from(value: Dateindex) -> Self { + value.0 as i64 + } +} + impl Add for Dateindex { type Output = Self; fn add(self, rhs: usize) -> Self::Output { Self(self.0 + rhs as u16) } } + +impl TryFrom for Dateindex { + type Error = color_eyre::Report; + fn try_from(value: Date) -> Result { + let value_ = jiff::civil::Date::from(value); + if value_ < Date::INDEX_ZERO_ { + Err(eyre!("Date is too early")) + } else if value == Date::INDEX_ZERO { + Ok(Self(0)) + } else if value_ < Date::INDEX_ONE_ { + Err(eyre!("Date is between first and second")) + } else if value == Date::INDEX_ONE { + Ok(Self(1)) + } else { + Ok(Self(Date::INDEX_ONE_.until(value_)?.get_days() as u16 + 1)) + } + } +} diff --git a/pricer/src/structs/dollars.rs b/pricer/src/structs/dollars.rs index 321106bba..4ea1892b3 100644 --- a/pricer/src/structs/dollars.rs +++ b/pricer/src/structs/dollars.rs @@ -1,8 +1,10 @@ use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Cents; -#[derive(Debug, Default, Clone, Copy, Deref)] +#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] pub struct Dollars(f64); impl From for Dollars { diff --git a/pricer/src/structs/high.rs b/pricer/src/structs/high.rs index 51947090a..53fa506cd 100644 --- a/pricer/src/structs/high.rs +++ b/pricer/src/structs/high.rs @@ -2,7 +2,24 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +use super::Close; + +#[derive( + Debug, + Default, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] #[repr(C)] pub struct High(T); impl From for High { @@ -10,3 +27,12 @@ impl From for High { Self(value) } } + +impl From> for High +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} diff --git a/pricer/src/structs/low.rs b/pricer/src/structs/low.rs index 3c4988fea..39f8c52ea 100644 --- a/pricer/src/structs/low.rs +++ b/pricer/src/structs/low.rs @@ -2,7 +2,24 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +use super::Close; + +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] #[repr(C)] pub struct Low(T); impl From for Low { @@ -10,3 +27,12 @@ impl From for Low { Self(value) } } + +impl From> for Low +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} diff --git a/pricer/src/structs/open.rs b/pricer/src/structs/open.rs index b6632c90d..fe2843268 100644 --- a/pricer/src/structs/open.rs +++ b/pricer/src/structs/open.rs @@ -2,7 +2,24 @@ use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +use super::Close; + +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, + Deref, + Serialize, +)] #[repr(C)] pub struct Open(T); impl From for Open { @@ -10,3 +27,12 @@ impl From for Open { Self(value) } } + +impl From> for Open +where + T: Copy, +{ + fn from(value: Close) -> Self { + Self(*value) + } +} diff --git a/server/src/api/vecs/index.rs b/server/src/api/vecs/index.rs index 69d0c7ac0..ab439a075 100644 --- a/server/src/api/vecs/index.rs +++ b/server/src/api/vecs/index.rs @@ -17,6 +17,26 @@ pub enum Index { Txoutindex, } +impl Index { + pub fn all() -> [Self; 13] { + [ + Self::Addressindex, + Self::Dateindex, + Self::Height, + Self::P2PK33index, + Self::P2PK65index, + Self::P2PKHindex, + Self::P2SHindex, + Self::P2TRindex, + Self::P2WPKHindex, + Self::P2WSHindex, + Self::Txindex, + Self::Txinindex, + Self::Txoutindex, + ] + } +} + impl TryFrom<&str> for Index { type Error = (); fn try_from(value: &str) -> Result { diff --git a/server/src/api/vecs/tree.rs b/server/src/api/vecs/tree.rs index b58be8866..d3bc4e7e2 100644 --- a/server/src/api/vecs/tree.rs +++ b/server/src/api/vecs/tree.rs @@ -1,8 +1,10 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, fs, io}; use derive_deref::{Deref, DerefMut}; use storable_vec::AnyJsonStorableVec; +use crate::WEBSITE_DEV_PATH; + use super::index::Index; #[derive(Default, Deref, DerefMut)] @@ -32,6 +34,44 @@ impl VecIdToIndexToVec { panic!() } } + + pub fn generate_dts_file(&self) -> io::Result<()> { + if !fs::exists(WEBSITE_DEV_PATH)? { + return Ok(()); + } + + let path = format!("{WEBSITE_DEV_PATH}/scripts/types/vecid-to-indexes.d.ts"); + + let mut contents = Index::all() + .into_iter() + .enumerate() + .map(|(i_of_i, i)| format!("type {} = {};", i, i_of_i)) + .collect::>() + .join("\n"); + + contents += "\n\ninterface VecIdToIndexes {\n"; + + self.iter().for_each(|(id, index_to_vec)| { + let indexes = index_to_vec + .keys() + .map(|i| i.to_string()) + .collect::>() + .join(", "); + + contents += &format!( + " {}: [{indexes}]\n", + if id.contains("-") { + format!("\"{id}\"") + } else { + id.to_owned() + } + ); + }); + + contents.push('}'); + + fs::write(path, contents) + } } #[derive(Default, Deref, DerefMut)] diff --git a/server/src/files/file.rs b/server/src/files/file.rs index fec5e86b3..3cf3a543a 100644 --- a/server/src/files/file.rs +++ b/server/src/files/file.rs @@ -16,12 +16,11 @@ use reqwest::StatusCode; use crate::{ log_result, traits::{HeaderMapExtended, ModifiedState, ResponseExtended}, + WEBSITE_DEV_PATH, }; use super::minify::minify_js; -const WEBSITE_DEV_PATH: &str = "../website/"; - pub async fn file_handler(headers: HeaderMap, path: extract::Path) -> Response { any_handler(headers, Some(path)) } diff --git a/server/src/lib.rs b/server/src/lib.rs index 3f7655f6e..d7a8a6f00 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -23,10 +23,9 @@ pub struct AppState { computer: &'static Computer, } -pub async fn main(indexer: Indexer, computer: Computer) -> color_eyre::Result<()> { - // pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> { - // routes.generate_dts_file(); +pub const WEBSITE_DEV_PATH: &str = "../website/"; +pub async fn main(indexer: Indexer, computer: Computer) -> color_eyre::Result<()> { let indexer = Box::leak(Box::new(indexer)); let computer = Box::leak(Box::new(computer)); let vecs = Box::leak(Box::new(VecIdToIndexToVec::default())); @@ -37,6 +36,8 @@ pub async fn main(indexer: Indexer, computer: Computer) -> .into_iter() .for_each(|vec| vecs.insert(vec)); + vecs.generate_dts_file()?; + let state = AppState { vecs, indexer,