diff --git a/Cargo.lock b/Cargo.lock index 06bdecac6..739206fa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,14 +262,12 @@ dependencies = [ "jiff", "logger", "oxc", - "regex", "reqwest", "serde", "serde_json", "storable_vec", "tokio", "tower-http", - "zstd", ] [[package]] @@ -400,7 +398,7 @@ dependencies = [ "derive_deref", "exit", "fjall", - "jiff", + "pricer", "storable_vec", "zerocopy 0.8.17", ] @@ -2111,6 +2109,17 @@ dependencies = [ [[package]] name = "pricer" version = "0.1.0" +dependencies = [ + "bindex", + "color-eyre", + "derive_deref", + "jiff", + "logger", + "reqwest", + "serde", + "serde_json", + "zerocopy 0.8.17", +] [[package]] name = "proc-macro2" @@ -2240,6 +2249,7 @@ version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", @@ -2270,6 +2280,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower", "tower-service", "url", diff --git a/Cargo.toml b/Cargo.toml index d7aee1aa7..73e061e28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ iterator = { path = "iterator", package = "biter" } jiff = "0.2.0" logger = { path = "logger" } rayon = "1.10.0" +pricer = { path = "pricer" } rlimit = { version = "0.10.2" } serde = { version = "1.0.217", features = ["derive"] } serde_json = { version = "1.0.138", features = ["float_roundtrip"] } diff --git a/computer/Cargo.toml b/computer/Cargo.toml index 0d0f44e26..35d646a2b 100644 --- a/computer/Cargo.toml +++ b/computer/Cargo.toml @@ -10,6 +10,6 @@ derive_deref = { workspace = true } exit = { workspace = true } fjall = { workspace = true } indexer = { workspace = true } -jiff = { workspace = true } +pricer = { workspace = true } storable_vec = { workspace = true } zerocopy = { workspace = true } diff --git a/computer/src/lib.rs b/computer/src/lib.rs index 775ed732b..5faa376c4 100644 --- a/computer/src/lib.rs +++ b/computer/src/lib.rs @@ -7,6 +7,7 @@ pub use iterator::rpc; mod storage; mod structs; +use pricer::Date; use storable_vec::SINGLE_THREAD; use storage::{Fjalls, StorableVecs}; pub use structs::*; diff --git a/computer/src/storage/storable_vecs.rs b/computer/src/storage/storable_vecs.rs index 39fe0207b..73a1b10c4 100644 --- a/computer/src/storage/storable_vecs.rs +++ b/computer/src/storage/storable_vecs.rs @@ -1,12 +1,10 @@ use std::{fs, path::Path}; -use indexer::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex}; +use indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; +use pricer::{Date, Dateindex}; use storable_vec::{StorableVec, Version}; -use crate::{ - structs::{Date, Feerate}, - Dateindex, -}; +use crate::structs::Feerate; // mod base; @@ -30,16 +28,16 @@ pub struct StorableVecs { // pub height_to_subsidy: StorableVec, // pub height_to_totalfees: StorableVec, // pub height_to_txcount: StorableVec, - pub txindex_to_fee: StorableVec, + pub txindex_to_fee: StorableVec, pub txindex_to_height: StorableVec, pub txindex_to_is_coinbase: StorableVec, // pub txindex_to_feerate: StorableVec, pub txindex_to_inputs_count: StorableVec, - pub txindex_to_inputs_sum: StorableVec, + pub txindex_to_inputs_sum: StorableVec, pub txindex_to_last_txinindex: StorableVec, pub txindex_to_last_txoutindex: StorableVec, pub txindex_to_outputs_count: StorableVec, - pub txindex_to_outputs_sum: StorableVec, + pub txindex_to_outputs_sum: StorableVec, } impl StorableVecs { diff --git a/computer/src/structs/bitcoin.rs b/computer/src/structs/bitcoin.rs new file mode 100644 index 000000000..28f4621d5 --- /dev/null +++ b/computer/src/structs/bitcoin.rs @@ -0,0 +1,14 @@ +use indexer::Sats; + +#[derive(Debug, Default, Clone, Copy)] +pub struct Bitcoin(f64); + +impl Bitcoin { + const ONE: Self = Self(100_000_000.0); +} + +impl From for Bitcoin { + fn from(value: Sats) -> Self { + Self((*value as f64) / Self::ONE.0) + } +} diff --git a/computer/src/structs/mod.rs b/computer/src/structs/mod.rs index 60235dc16..7e26c175a 100644 --- a/computer/src/structs/mod.rs +++ b/computer/src/structs/mod.rs @@ -1,11 +1,11 @@ mod addressindextxoutindex; -mod date; -mod dateindex; +mod bitcoin; mod feerate; +mod ohlc; mod unit; pub use addressindextxoutindex::*; -pub use date::*; -pub use dateindex::*; +pub use bitcoin::*; pub use feerate::*; +pub use ohlc::*; pub use unit::*; diff --git a/computer/src/structs/ohlc.rs b/computer/src/structs/ohlc.rs new file mode 100644 index 000000000..a37e5ba2d --- /dev/null +++ b/computer/src/structs/ohlc.rs @@ -0,0 +1,36 @@ +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use super::{Cents, Close, High, Low, Open}; + +// #[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +// #[repr(C)] +// pub struct OHLCCents(OHLC); + +#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +#[repr(C)] +pub struct OHLCCents(Open, High, Low, Close); + +impl OHLCCents { + pub fn open(&self) -> Open { + self.0 + } + + pub fn high(&self) -> High { + self.1 + } + + pub fn low(&self) -> Low { + self.2 + } + + pub fn close(&self) -> Close { + self.3 + } +} + +impl From<(Open, High, Low, Close)> for OHLCCents { + fn from(value: (Open, High, Low, Close)) -> Self { + Self(value.0, value.1, value.2, value.3) + } +} diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 7d3af88cc..98fa77fe0 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -92,7 +92,7 @@ impl Indexer { iterator::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc) .iter() .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { - info!("Processing block {_height}..."); + info!("Indexing block {_height}..."); let blockhash = BlockHash::from(blockhash); height = Height::from(_height); @@ -440,13 +440,13 @@ impl Indexer { (txout, txindex, vout, addresstype, addressbytes_res, addressindex_opt, _tx), )| -> color_eyre::Result<()> { - let amount = Amount::from(txout.value); + let sats = Sats::from(txout.value); if vout.is_zero() { vecs.txindex_to_first_txoutindex.push_if_needed(txindex, txoutindex)?; } - vecs.txoutindex_to_amount.push_if_needed(txoutindex, amount)?; + vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?; let mut addressindex = addressindex_global; diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index 9abab5799..3e4c2c39a 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -5,11 +5,10 @@ use rayon::prelude::*; use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS}; use crate::structs::{ - Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Emptyindex, Height, LockTime, - Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, - P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, - P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, - Unknownindex, Weight, + Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex, + Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, + P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, + P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, }; mod base; @@ -56,7 +55,7 @@ pub struct StorableVecs { pub txindex_to_txversion: StorableVec, pub txinindex_to_txoutindex: StorableVec, pub txoutindex_to_addressindex: StorableVec, - pub txoutindex_to_amount: StorableVec, + pub txoutindex_to_value: StorableVec, } // const UNSAFE_BLOCKS: usize = 1000; @@ -177,7 +176,7 @@ impl StorableVecs { &path.join("txoutindex_to_addressindex"), Version::from(1), )?, - txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?, + txoutindex_to_value: StorableVec::import(&path.join("txoutindex_to_value"), Version::from(1))?, }) } @@ -350,7 +349,7 @@ impl StorableVecs { &*self.txindex_to_txversion, &*self.txinindex_to_txoutindex, &*self.txoutindex_to_addressindex, - &*self.txoutindex_to_amount, + &*self.txoutindex_to_value, ] } @@ -395,7 +394,7 @@ impl StorableVecs { &mut self.txindex_to_txversion, &mut self.txinindex_to_txoutindex, &mut self.txoutindex_to_addressindex, - &mut self.txoutindex_to_amount, + &mut self.txoutindex_to_value, ] } } diff --git a/indexer/src/structs/mod.rs b/indexer/src/structs/mod.rs index 04b682955..ff4050491 100644 --- a/indexer/src/structs/mod.rs +++ b/indexer/src/structs/mod.rs @@ -2,11 +2,11 @@ mod addressbytes; mod addressindex; mod addresstype; mod addresstypeindex; -mod amount; mod blockhash; mod compressed; mod height; mod locktime; +mod sats; mod timestamp; mod txid; mod txindex; @@ -21,11 +21,11 @@ pub use addressbytes::*; pub use addressindex::*; pub use addresstype::*; pub use addresstypeindex::*; -pub use amount::*; pub use blockhash::*; pub use compressed::*; pub use height::*; pub use locktime::*; +pub use sats::*; pub use timestamp::*; pub use txid::*; pub use txindex::*; diff --git a/indexer/src/structs/amount.rs b/indexer/src/structs/sats.rs similarity index 50% rename from indexer/src/structs/amount.rs rename to indexer/src/structs/sats.rs index 6e0a6acd9..eea28b04f 100644 --- a/indexer/src/structs/amount.rs +++ b/indexer/src/structs/sats.rs @@ -4,7 +4,7 @@ use std::{ }; use derive_deref::{Deref, DerefMut}; -use iterator::bitcoin; +use iterator::bitcoin::Amount; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -27,85 +27,83 @@ use super::Height; KnownLayout, Serialize, )] -pub struct Amount(u64); +pub struct Sats(u64); -impl Amount { +impl Sats { pub const ZERO: Self = Self(0); - pub const ONE_BTC_F32: f32 = 100_000_000.0; - pub const ONE_BTC_F64: f64 = 100_000_000.0; pub fn is_zero(&self) -> bool { *self == Self::ZERO } } -impl Add for Amount { - type Output = Amount; - fn add(self, rhs: Amount) -> Self::Output { - Amount::from(*self + *rhs) +impl Add for Sats { + type Output = Sats; + fn add(self, rhs: Sats) -> Self::Output { + Sats::from(*self + *rhs) } } -impl AddAssign for Amount { +impl AddAssign for Sats { fn add_assign(&mut self, rhs: Self) { *self = *self + rhs; } } -impl Sub for Amount { - type Output = Amount; - fn sub(self, rhs: Amount) -> Self::Output { - Amount::from(*self - *rhs) +impl Sub for Sats { + type Output = Sats; + fn sub(self, rhs: Sats) -> Self::Output { + Sats::from(*self - *rhs) } } -impl SubAssign for Amount { +impl SubAssign for Sats { fn sub_assign(&mut self, rhs: Self) { *self = *self - rhs; } } -impl Mul for Amount { - type Output = Amount; - fn mul(self, rhs: Amount) -> Self::Output { - Amount::from(*self * *rhs) +impl Mul for Sats { + type Output = Sats; + fn mul(self, rhs: Sats) -> Self::Output { + Sats::from(*self * *rhs) } } -impl Mul for Amount { - type Output = Amount; +impl Mul for Sats { + type Output = Sats; fn mul(self, rhs: u64) -> Self::Output { - Amount::from(*self * rhs) + Sats::from(*self * rhs) } } -impl Mul for Amount { - type Output = Amount; +impl Mul for Sats { + type Output = Sats; fn mul(self, rhs: Height) -> Self::Output { - Amount::from(*self * *rhs as u64) + Sats::from(*self * *rhs as u64) } } -impl Sum for Amount { +impl Sum for Sats { fn sum>(iter: I) -> Self { - let sats: u64 = iter.map(|amt| *amt).sum(); - Amount::from(sats) + let sats: u64 = iter.map(|sats| *sats).sum(); + Sats::from(sats) } } -impl From for Amount { +impl From for Sats { fn from(value: u64) -> Self { Self(value) } } -impl From for Amount { - fn from(value: bitcoin::Amount) -> Self { +impl From for Sats { + fn from(value: Amount) -> Self { Self(value.to_sat()) } } -impl From for bitcoin::Amount { - fn from(value: Amount) -> Self { +impl From for Amount { + fn from(value: Sats) -> Self { Self::from_sat(value.0) } } diff --git a/indexer/src/structs/vout.rs b/indexer/src/structs/vout.rs index a5f267823..4cc8ef793 100644 --- a/indexer/src/structs/vout.rs +++ b/indexer/src/structs/vout.rs @@ -4,7 +4,7 @@ use derive_deref::Deref; pub struct Vout(u32); impl Vout { - const ZERO: Self = Vout(0_u32); + const ZERO: Self = Vout(0); pub fn is_zero(&self) -> bool { *self == Self::ZERO diff --git a/pricer/Cargo.toml b/pricer/Cargo.toml index c2ba1eb70..c9db534d0 100644 --- a/pricer/Cargo.toml +++ b/pricer/Cargo.toml @@ -4,3 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] +color-eyre = { workspace = true } +derive_deref = { workspace = true } +indexer = { workspace = true } +logger = { workspace = true } +reqwest = { version = "0.12.12", features = ["blocking", "brotli", "deflate", "gzip", "json", "zstd"] } +jiff = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +zerocopy = { workspace = true } diff --git a/pricer/src/fetchers/binance.rs b/pricer/src/fetchers/binance.rs new file mode 100644 index 000000000..f8ecbb0f3 --- /dev/null +++ b/pricer/src/fetchers/binance.rs @@ -0,0 +1,157 @@ +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::BufReader, + path::Path, + str::FromStr, +}; + +use color_eyre::eyre::{eyre, ContextCompat}; +use indexer::Timestamp; +use logger::info; +use serde_json::Value; + +use crate::{ + fetchers::retry, + structs::{Cents, OHLC}, + Close, Date, Dollars, High, Low, Open, +}; + +pub struct Binance; + +impl Binance { + pub fn fetch_1mn_prices() -> color_eyre::Result> { + info!("Fetching 1mn prices from Binance..."); + + retry( + |_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url("interval=1m&limit=1000"))?.json()?), + 30, + 10, + ) + } + + pub fn fetch_daily_prices() -> color_eyre::Result> { + info!("Fetching daily prices from Kraken..."); + + retry( + |_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url("interval=1d"))?.json()?), + 30, + 10, + ) + } + + pub fn read_har_file(path: &Path) -> color_eyre::Result> { + info!("Reading Binance har file..."); + + fs::create_dir_all(&path)?; + + let path_binance_har = path.join("binance.har"); + + let file = if let Ok(file) = File::open(path_binance_har) { + file + } else { + return Err(eyre!("Missing binance file")); + }; + + let reader = BufReader::new(file); + + let json: BTreeMap = if let Ok(json) = serde_json::from_reader(reader) { + json + } else { + return Ok(Default::default()); + }; + + json.get("log") + .context("Expect object to have log attribute")? + .as_object() + .context("Expect to be an object")? + .get("entries") + .context("Expect object to have entries")? + .as_array() + .context("Expect to be an array")? + .iter() + .filter(|entry| { + entry + .as_object() + .unwrap() + .get("request") + .unwrap() + .as_object() + .unwrap() + .get("url") + .unwrap() + .as_str() + .unwrap() + .contains("/uiKlines") + }) + .map(|entry| { + let response = entry.as_object().unwrap().get("response").unwrap().as_object().unwrap(); + + let content = response.get("content").unwrap().as_object().unwrap(); + + let text = content.get("text"); + + if text.is_none() { + return Ok(BTreeMap::new()); + } + + let text = text.unwrap().as_str().unwrap(); + + Self::json_to_timestamp_to_ohlc(&serde_json::Value::from_str(text).unwrap()) + }) + .try_fold(BTreeMap::default(), |mut all, res| { + all.append(&mut res?); + Ok(all) + }) + } + + fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { + Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) + } + + fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + Self::json_to_btree(json, Self::array_to_date_and_ohlc) + } + + fn json_to_btree(json: &Value, fun: F) -> color_eyre::Result> + where + F: Fn(&Value) -> color_eyre::Result<(K, V)>, + K: Ord, + { + json.as_array() + .context("Expect to be an array")? + .iter() + .map(fun) + .collect::, _>>() + } + + fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> { + let array = array.as_array().context("Expect to be array")?; + + let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32); + + let get_cents = |index: usize| { + Cents::from(Dollars::from( + array.get(index).unwrap().as_str().unwrap().parse::().unwrap(), + )) + }; + + Ok(( + timestamp, + OHLC::from(( + Open::from(get_cents(1)), + High::from(get_cents(2)), + Low::from(get_cents(3)), + Close::from(get_cents(4)), + )), + )) + } + + fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> { + Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) + } + + fn url(query: &str) -> String { + format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}") + } +} diff --git a/pricer/src/price/kibo.rs b/pricer/src/fetchers/kibo.rs similarity index 84% rename from pricer/src/price/kibo.rs rename to pricer/src/fetchers/kibo.rs index 858b7d7ac..8b137090b 100644 --- a/pricer/src/price/kibo.rs +++ b/pricer/src/fetchers/kibo.rs @@ -1,14 +1,10 @@ use std::{collections::BTreeMap, str::FromStr}; -use chrono::NaiveDate; use color_eyre::eyre::ContextCompat; -use log::info; +use logger::info; use serde_json::Value; -use crate::{ - structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC}, - utils::retry, -}; +use crate::structs::{Date, OHLC}; pub struct Kibo; @@ -33,11 +29,9 @@ impl Kibo { |try_index| { let base_url = Self::get_base_url(try_index); - let body: Value = reqwest::blocking::get(format!( - "{base_url}/height-to-price?chunk={}", - chunk_id.to_usize() - ))? - .json()?; + let body: Value = + reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))? + .json()?; let vec = body .as_object() @@ -68,11 +62,9 @@ impl Kibo { |try_index| { let base_url = Self::get_base_url(try_index); - let body: Value = reqwest::blocking::get(format!( - "{base_url}/date-to-price?chunk={}", - chunk_id.to_usize() - ))? - .json()?; + let body: Value = + reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))? + .json()?; Ok(body .as_object() diff --git a/pricer/src/fetchers/kraken.rs b/pricer/src/fetchers/kraken.rs new file mode 100644 index 000000000..2f38725fa --- /dev/null +++ b/pricer/src/fetchers/kraken.rs @@ -0,0 +1,90 @@ +use std::collections::BTreeMap; + +use color_eyre::eyre::ContextCompat; +use indexer::Timestamp; +use logger::info; +use serde_json::Value; + +use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC}; + +pub struct Kraken; + +impl Kraken { + pub fn fetch_1mn_prices() -> color_eyre::Result> { + info!("Fetching 1mn prices from Kraken..."); + + retry( + |_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url(1))?.json()?), + 30, + 10, + ) + } + + pub fn fetch_daily_prices() -> color_eyre::Result> { + info!("Fetching daily prices from Kraken..."); + + retry( + |_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url(1440))?.json()?), + 30, + 10, + ) + } + + fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result> { + Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc) + } + + fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result> { + Self::json_to_btree(json, Self::array_to_date_and_ohlc) + } + + fn json_to_btree(json: &Value, fun: F) -> color_eyre::Result> + where + F: Fn(&Value) -> color_eyre::Result<(K, V)>, + K: Ord, + { + json.as_object() + .context("Expect to be an object")? + .get("result") + .context("Expect object to have result")? + .as_object() + .context("Expect to be an object")? + .get("XXBTZUSD") + .context("Expect to have XXBTZUSD")? + .as_array() + .context("Expect to be an array")? + .iter() + .map(fun) + .collect::, _>>() + } + + fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> { + let array = array.as_array().context("Expect to be array")?; + + let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32); + + let get_cents = |index: usize| { + Cents::from(Dollars::from( + array.get(index).unwrap().as_str().unwrap().parse::().unwrap(), + )) + }; + + Ok(( + timestamp, + OHLC::from(( + Open::from(get_cents(1)), + High::from(get_cents(2)), + Low::from(get_cents(3)), + Close::from(get_cents(4)), + )), + )) + } + + fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> { + Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc)) + } + + fn url(interval: usize) -> String { + format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}") + } +} diff --git a/pricer/src/price/mod.rs b/pricer/src/fetchers/mod.rs similarity index 78% rename from pricer/src/price/mod.rs rename to pricer/src/fetchers/mod.rs index 41cea4435..c6704727b 100644 --- a/pricer/src/price/mod.rs +++ b/pricer/src/fetchers/mod.rs @@ -1,7 +1,9 @@ mod binance; mod kibo; mod kraken; +mod retry; pub use binance::*; pub use kibo::*; pub use kraken::*; +use retry::*; diff --git a/pricer/src/retry.rs b/pricer/src/fetchers/retry.rs similarity index 83% rename from pricer/src/retry.rs rename to pricer/src/fetchers/retry.rs index f200ced47..555e4a31c 100644 --- a/pricer/src/retry.rs +++ b/pricer/src/fetchers/retry.rs @@ -1,5 +1,7 @@ use std::{thread::sleep, time::Duration}; +use logger::info; + pub fn retry( function: impl Fn(usize) -> color_eyre::Result, sleep_in_s: u64, @@ -13,6 +15,7 @@ pub fn retry( if i == retries || res.is_ok() { return res; } else { + info!("Failed, waiting {sleep_in_s} seconds..."); sleep(Duration::from_secs(sleep_in_s)); } diff --git a/pricer/src/lib.rs b/pricer/src/lib.rs index b93cf3ffd..01562e543 100644 --- a/pricer/src/lib.rs +++ b/pricer/src/lib.rs @@ -1,14 +1,5 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} +mod fetchers; +mod structs; -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +pub use fetchers::*; +pub use structs::*; diff --git a/pricer/src/main.rs b/pricer/src/main.rs new file mode 100644 index 000000000..c73ec56b7 --- /dev/null +++ b/pricer/src/main.rs @@ -0,0 +1,16 @@ +// fn main() {} + +use pricer::{Binance, Kraken}; + +fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + + logger::init_log(None); + + // dbg!(Binance::fetch_daily_prices()); + // dbg!(Binance::fetch_1mn_prices()); + // dbg!(Kraken::fetch_daily_prices()); + dbg!(Kraken::fetch_1mn_prices()); + + Ok(()) +} diff --git a/pricer/src/price.rs b/pricer/src/price.rs index ca79bd945..5c98da0a1 100644 --- a/pricer/src/price.rs +++ b/pricer/src/price.rs @@ -9,8 +9,8 @@ use struct_iterable::Iterable; use crate::{ parser::price::{Binance, Kibo, Kraken}, structs::{ - Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, - MapKind, Timestamp, OHLC, + Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, MapKind, Timestamp, + OHLC, }, utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS}, }; @@ -34,54 +34,6 @@ pub struct PriceDatasets { pub high: BiMap, pub low: BiMap, pub close: BiMap, - pub market_cap: BiMap, - pub price_1w_sma: BiMap, - pub price_1w_sma_ratio: RatioDataset, - pub price_1m_sma: BiMap, - pub price_1m_sma_ratio: RatioDataset, - pub price_1y_sma: BiMap, - pub price_1y_sma_ratio: RatioDataset, - pub price_2y_sma: BiMap, - pub price_2y_sma_ratio: RatioDataset, - pub price_4y_sma: BiMap, - pub price_4y_sma_ratio: RatioDataset, - pub price_8d_sma: BiMap, - pub price_8d_sma_ratio: RatioDataset, - pub price_13d_sma: BiMap, - pub price_13d_sma_ratio: RatioDataset, - pub price_21d_sma: BiMap, - pub price_21d_sma_ratio: RatioDataset, - pub price_34d_sma: BiMap, - pub price_34d_sma_ratio: RatioDataset, - pub price_55d_sma: BiMap, - pub price_55d_sma_ratio: RatioDataset, - pub price_89d_sma: BiMap, - pub price_89d_sma_ratio: RatioDataset, - pub price_144d_sma: BiMap, - pub price_144d_sma_ratio: RatioDataset, - pub price_200w_sma: BiMap, - pub price_200w_sma_ratio: RatioDataset, - pub price_1d_total_return: DateMap, - pub price_1m_total_return: DateMap, - pub price_6m_total_return: DateMap, - pub price_1y_total_return: DateMap, - pub price_2y_total_return: DateMap, - pub price_3y_total_return: DateMap, - pub price_4y_total_return: DateMap, - pub price_6y_total_return: DateMap, - pub price_8y_total_return: DateMap, - pub price_10y_total_return: DateMap, - pub price_4y_compound_return: DateMap, - // projection via lowest 4y compound value - pub all_time_high: BiMap, - pub all_time_high_date: DateMap, - pub days_since_all_time_high: DateMap, - pub max_days_between_all_time_highs: DateMap, - pub max_years_between_all_time_highs: DateMap, - pub market_price_to_all_time_high_ratio: BiMap, - pub drawdown: BiMap, - pub sats_per_dollar: BiMap, - // volatility } impl PriceDatasets { @@ -108,116 +60,10 @@ impl PriceDatasets { // --- // Computed // --- - open: BiMap::new_bin(1, MapKind::Computed, &f("open")), - high: BiMap::new_bin(1, MapKind::Computed, &f("high")), - low: BiMap::new_bin(1, MapKind::Computed, &f("low")), + open_cents: BiMap::new_bin(1, MapKind::Computed, &f("open")), + high_cents: BiMap::new_bin(1, MapKind::Computed, &f("high")), + low_cents: BiMap::new_bin(1, MapKind::Computed, &f("low")), close: BiMap::new_bin(1, MapKind::Computed, &f("close")), - market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")), - price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")), - price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?, - price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")), - price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?, - price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")), - price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?, - price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")), - price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?, - price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")), - price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?, - price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")), - price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?, - price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")), - price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?, - price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")), - price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?, - price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")), - price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?, - price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")), - price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?, - price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")), - price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?, - price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")), - price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?, - price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")), - price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?, - price_1d_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_1d_total_return"), - ), - price_1m_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_1m_total_return"), - ), - price_6m_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_6m_total_return"), - ), - price_1y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_1y_total_return"), - ), - price_2y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_2y_total_return"), - ), - price_3y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_3y_total_return"), - ), - price_4y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_4y_total_return"), - ), - price_6y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_6y_total_return"), - ), - price_8y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_8y_total_return"), - ), - price_10y_total_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_10y_total_return"), - ), - price_4y_compound_return: DateMap::new_bin( - 1, - MapKind::Computed, - &f("price_4y_compound_return"), - ), - all_time_high: BiMap::new_bin(1, MapKind::Computed, &f("all_time_high")), - all_time_high_date: DateMap::new_bin(1, MapKind::Computed, &f("all_time_high_date")), - days_since_all_time_high: DateMap::new_bin( - 1, - MapKind::Computed, - &f("days_since_all_time_high"), - ), - max_days_between_all_time_highs: DateMap::new_bin( - 1, - MapKind::Computed, - &f("max_days_between_all_time_highs"), - ), - max_years_between_all_time_highs: DateMap::new_bin( - 2, - MapKind::Computed, - &f("max_years_between_all_time_highs"), - ), - market_price_to_all_time_high_ratio: BiMap::new_bin( - 1, - MapKind::Computed, - &f("market_price_to_all_time_high_ratio"), - ), - drawdown: BiMap::new_bin(1, MapKind::Computed, &f("drawdown")), - sats_per_dollar: BiMap::new_bin(1, MapKind::Computed, &f("sats_per_dollar")), }; s.min_initial_states @@ -240,213 +86,6 @@ impl PriceDatasets { self.close .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close); - - self.market_cap - .multi_insert_multiply(heights, dates, &mut self.close, circulating_supply); - - self.price_1w_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - ONE_WEEK_IN_DAYS, - ); - - self.price_1m_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - ONE_MONTH_IN_DAYS, - ); - - self.price_1y_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - ONE_YEAR_IN_DAYS, - ); - - self.price_2y_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - 2 * ONE_YEAR_IN_DAYS, - ); - - self.price_4y_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - 4 * ONE_YEAR_IN_DAYS, - ); - - self.price_8d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 8); - - self.price_13d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 13); - - self.price_21d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 21); - - self.price_34d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 34); - - self.price_55d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 55); - - self.price_89d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 89); - - self.price_144d_sma - .multi_insert_simple_average(heights, dates, &mut self.close, 144); - - self.price_200w_sma.multi_insert_simple_average( - heights, - dates, - &mut self.close, - 200 * ONE_WEEK_IN_DAYS, - ); - - self.price_1d_total_return - .multi_insert_percentage_change(dates, &mut self.close.date, 1); - self.price_1m_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - ONE_MONTH_IN_DAYS, - ); - self.price_6m_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 6 * ONE_MONTH_IN_DAYS, - ); - self.price_1y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - ONE_YEAR_IN_DAYS, - ); - self.price_2y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 2 * ONE_YEAR_IN_DAYS, - ); - self.price_3y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 3 * ONE_YEAR_IN_DAYS, - ); - self.price_4y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 4 * ONE_YEAR_IN_DAYS, - ); - self.price_6y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 6 * ONE_YEAR_IN_DAYS, - ); - self.price_8y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 8 * ONE_YEAR_IN_DAYS, - ); - self.price_10y_total_return.multi_insert_percentage_change( - dates, - &mut self.close.date, - 10 * ONE_YEAR_IN_DAYS, - ); - - self.price_4y_compound_return - .multi_insert_complex_transform( - dates, - &mut self.close.date, - |(last_value, date, closes, _)| { - let previous_value = date - .checked_sub_days(Days::new(4 * ONE_YEAR_IN_DAYS as u64)) - .and_then(|date| closes.get_or_import(&Date::wrap(date))) - .unwrap_or_default(); - - (((last_value / previous_value).powf(1.0 / 4.0)) - 1.0) * 100.0 - }, - ); - - self.price_1w_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_1w_sma); - self.price_1m_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_1m_sma); - self.price_1y_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_1y_sma); - self.price_2y_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_2y_sma); - self.price_4y_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_4y_sma); - self.price_8d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_8d_sma); - self.price_13d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_13d_sma); - self.price_21d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_21d_sma); - self.price_34d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_34d_sma); - self.price_55d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_55d_sma); - self.price_89d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_89d_sma); - self.price_144d_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_144d_sma); - self.price_200w_sma_ratio - .compute(compute_data, &mut self.close, &mut self.price_200w_sma); - - self.all_time_high - .multi_insert_max(heights, dates, &mut self.high); - - self.market_price_to_all_time_high_ratio - .multi_insert_percentage(heights, dates, &mut self.close, &mut self.all_time_high); - - self.all_time_high_date.multi_insert_complex_transform( - dates, - &mut self.all_time_high.date, - |(value, date, _, map)| { - let high = self.high.date.get_or_import(date).unwrap(); - let is_ath = high == value; - - if is_ath { - *date - } else { - let previous_date = date.checked_sub(1).unwrap(); - *map.get_or_import(&previous_date).as_ref().unwrap_or(date) - } - }, - ); - - self.days_since_all_time_high.multi_insert_simple_transform( - dates, - &mut self.all_time_high_date, - |value, key| key.difference_in_days_between(value), - ); - - self.max_days_between_all_time_highs - .multi_insert_max(dates, &mut self.days_since_all_time_high); - - self.max_years_between_all_time_highs - .multi_insert_simple_transform( - dates, - &mut self.max_days_between_all_time_highs, - |days, _| (days as f64 / ONE_YEAR_IN_DAYS as f64) as f32, - ); - - self.drawdown.multi_insert_simple_transform( - heights, - dates, - &mut self.market_price_to_all_time_high_ratio, - &|v| -(100.0 - v), - ); - - self.sats_per_dollar.multi_insert_simple_transform( - heights, - dates, - &mut self.close, - &|price| Amount::ONE_BTC_F32 / price, - ); } pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result { @@ -469,17 +108,9 @@ impl PriceDatasets { #[allow(clippy::map_entry)] if !self.kibo_by_date.contains_key(&chunk_id) - || self - .kibo_by_date - .get(&chunk_id) - .unwrap() - .last_key_value() - .unwrap() - .0 - < date + || self.kibo_by_date.get(&chunk_id).unwrap().last_key_value().unwrap().0 < date { - self.kibo_by_date - .insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?); + self.kibo_by_date.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?); } self.kibo_by_date @@ -491,16 +122,7 @@ impl PriceDatasets { } fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result { - if self.kraken_daily.is_none() - || self - .kraken_daily - .as_ref() - .unwrap() - .last_key_value() - .unwrap() - .0 - < date - { + if self.kraken_daily.is_none() || self.kraken_daily.as_ref().unwrap().last_key_value().unwrap().0 < date { self.kraken_daily.replace(Kraken::fetch_daily_prices()?); } @@ -513,16 +135,7 @@ impl PriceDatasets { } fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result { - if self.binance_daily.is_none() - || self - .binance_daily - .as_ref() - .unwrap() - .last_key_value() - .unwrap() - .0 - < date - { + if self.binance_daily.is_none() || self.binance_daily.as_ref().unwrap().last_key_value().unwrap().0 < date { self.binance_daily.replace(Binance::fetch_daily_prices()?); } @@ -593,8 +206,7 @@ How to fix this: #[allow(clippy::map_entry)] if !self.kibo_by_height.contains_key(&chunk_id) - || ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) - <= height.to_usize()) + || ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) <= height.to_usize()) { self.kibo_by_height .insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?); @@ -613,16 +225,7 @@ How to fix this: timestamp: Timestamp, previous_timestamp: Option, ) -> color_eyre::Result { - if self.kraken_1mn.is_none() - || self - .kraken_1mn - .as_ref() - .unwrap() - .last_key_value() - .unwrap() - .0 - <= ×tamp - { + if self.kraken_1mn.is_none() || self.kraken_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?); } @@ -634,25 +237,11 @@ How to fix this: timestamp: Timestamp, previous_timestamp: Option, ) -> color_eyre::Result { - if self.binance_1mn.is_none() - || self - .binance_1mn - .as_ref() - .unwrap() - .last_key_value() - .unwrap() - .0 - <= ×tamp - { + if self.binance_1mn.is_none() || self.binance_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= ×tamp { self.binance_1mn.replace(Binance::fetch_1mn_prices()?); } - Self::find_height_ohlc( - &self.binance_1mn, - timestamp, - previous_timestamp, - "binance 1m", - ) + Self::find_height_ohlc(&self.binance_1mn, timestamp, previous_timestamp, "binance 1m") } fn get_from_har_binance( @@ -666,12 +255,7 @@ How to fix this: .replace(Binance::read_har_file(config).unwrap_or_default()); } - Self::find_height_ohlc( - &self.binance_har, - timestamp, - previous_timestamp, - "binance har", - ) + Self::find_height_ohlc(&self.binance_har, timestamp, previous_timestamp, "binance har") } fn find_height_ohlc( @@ -684,10 +268,9 @@ How to fix this: let err = Error::msg(format!("Couldn't find timestamp in {name}")); - let previous_ohlc = previous_timestamp - .map_or(Some(OHLC::default()), |previous_timestamp| { - tree.get(&previous_timestamp).cloned() - }); + let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| { + tree.get(&previous_timestamp).cloned() + }); let last_ohlc = tree.get(×tamp); diff --git a/pricer/src/price/binance.rs b/pricer/src/price/binance.rs deleted file mode 100644 index 0967ad1e0..000000000 --- a/pricer/src/price/binance.rs +++ /dev/null @@ -1,213 +0,0 @@ -#![allow(dead_code)] - -use std::{collections::BTreeMap, fs}; - -use color_eyre::eyre::ContextCompat; -use itertools::Itertools; -use log::info; -use serde_json::Value; - -use crate::{ - io::Json, - structs::{Config, Date, Timestamp, OHLC}, - utils::retry, -}; - -pub struct Binance; - -impl Binance { - pub fn read_har_file(config: &Config) -> color_eyre::Result> { - info!("binance: read har file"); - - let path = config.path_inputs(); - - fs::create_dir_all(&path)?; - - let path_binance_har = path.join("binance.har"); - - let json: BTreeMap = Json::import(&path_binance_har).unwrap_or_default(); - - Ok(json - .get("log") - .context("Expect object to have log attribute")? - .as_object() - .context("Expect to be an object")? - .get("entries") - .context("Expect object to have entries")? - .as_array() - .context("Expect to be an array")? - .iter() - .filter(|entry| { - entry - .as_object() - .unwrap() - .get("request") - .unwrap() - .as_object() - .unwrap() - .get("url") - .unwrap() - .as_str() - .unwrap() - .contains("/uiKlines") - }) - .flat_map(|entry| { - let response = entry - .as_object() - .unwrap() - .get("response") - .unwrap() - .as_object() - .unwrap(); - - let content = response.get("content").unwrap().as_object().unwrap(); - - let text = content.get("text"); - - if text.is_none() { - return vec![]; - } - - let text = text.unwrap().as_str().unwrap(); - - let arrays: Value = serde_json::from_str(text).unwrap(); - - arrays - .as_array() - .unwrap() - .iter() - .map(|array| { - let array = array.as_array().unwrap(); - - let timestamp = (array.first().unwrap().as_u64().unwrap() / 1000) as u32; - - let get_f32 = |index: usize| { - array - .get(index) - .unwrap() - .as_str() - .unwrap() - .parse::() - .unwrap() - }; - - ( - timestamp, - OHLC { - open: get_f32(1), - high: get_f32(2), - low: get_f32(3), - close: get_f32(4), - }, - ) - }) - .collect_vec() - }) - .collect::>()) - } - - pub fn fetch_1mn_prices() -> color_eyre::Result> { - info!("binance: fetch 1mn"); - - retry( - |_| { - let body: Value = reqwest::blocking::get( - "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000", - )? - .json()?; - - Ok(body - .as_array() - .context("Expect to be an array")? - .iter() - .map(|value| -> color_eyre::Result<_> { - // [timestamp, open, high, low, close, volume, ...] - let array = value.as_array().context("Expect to be array")?; - - let timestamp = (array - .first() - .context("Expect to have first")? - .as_u64() - .context("Expect to be convertible to u64")? - / 1_000) as u32; - - let get_f32 = |index: usize| -> color_eyre::Result { - Ok(array - .get(index) - .context("Expect to have index")? - .as_str() - .context("Expect to have &str")? - .parse::()?) - }; - - Ok(( - timestamp, - OHLC { - open: get_f32(1)?, - high: get_f32(2)?, - low: get_f32(3)?, - close: get_f32(4)?, - }, - )) - }) - .collect::, _>>()?) - }, - 30, - 10, - ) - } - - pub fn fetch_daily_prices() -> color_eyre::Result> { - info!("binance: fetch 1d"); - - retry( - |_| { - let body: Value = reqwest::blocking::get( - "https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d", - )? - .json()?; - - Ok(body - .as_array() - .context("Expect to be an array")? - .iter() - .map(|value| -> color_eyre::Result<_> { - // [timestamp, open, high, low, close, volume, ...] - let array = value.as_array().context("Expect to be array")?; - - let date = Timestamp::from( - (array - .first() - .context("Expect to have first")? - .as_u64() - .context("Expect to be convertible to u64")? - / 1_000) as u32, - ) - .to_date(); - - let get_f32 = |index: usize| -> color_eyre::Result { - Ok(array - .get(index) - .context("Expect to have index")? - .as_str() - .context("Expect to have &str")? - .parse::()?) - }; - - Ok(( - date, - OHLC { - open: get_f32(1)?, - high: get_f32(2)?, - low: get_f32(3)?, - close: get_f32(4)?, - }, - )) - }) - .collect::, _>>()?) - }, - 30, - 10, - ) - } -} diff --git a/pricer/src/price/kraken.rs b/pricer/src/price/kraken.rs deleted file mode 100644 index 99566fee0..000000000 --- a/pricer/src/price/kraken.rs +++ /dev/null @@ -1,133 +0,0 @@ -use std::collections::BTreeMap; - -use color_eyre::eyre::ContextCompat; -use log::info; -use serde_json::Value; - -use crate::{ - structs::{Date, Timestamp, OHLC}, - utils::retry, -}; - -pub struct Kraken; - -impl Kraken { - pub fn fetch_1mn_prices() -> color_eyre::Result> { - info!("kraken: fetch 1mn"); - - retry( - |_| { - let body: Value = reqwest::blocking::get( - "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1", - )? - .json()?; - - Ok(body - .as_object() - .context("Expect to be an object")? - .get("result") - .context("Expect object to have result")? - .as_object() - .context("Expect to be an object")? - .get("XXBTZUSD") - .context("Expect to have XXBTZUSD")? - .as_array() - .context("Expect to be an array")? - .iter() - .map(|value| -> color_eyre::Result<_> { - let array = value.as_array().context("Expect as_array to work")?; - - let timestamp = array - .first() - .context("Expect first to work")? - .as_u64() - .expect("Expect as_u64 to work") - as u32; - - let get_f32 = |index: usize| -> color_eyre::Result { - Ok(array - .get(index) - .context("Expect get index to work")? - .as_str() - .context("Expect as_str to work")? - .parse::()?) - }; - - Ok(( - timestamp, - OHLC { - open: get_f32(1)?, - high: get_f32(2)?, - low: get_f32(3)?, - close: get_f32(4)?, - }, - )) - }) - .collect::, _>>()?) - }, - 30, - 10, - ) - } - - pub fn fetch_daily_prices() -> color_eyre::Result> { - info!("fetch kraken daily"); - - retry( - |_| { - let body: Value = reqwest::blocking::get( - "https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440", - )? - .json()?; - - Ok(body - .as_object() - .context("Expect to be an object")? - .get("result") - .context("Expect object to have result")? - .as_object() - .context("Expect to be an object")? - .get("XXBTZUSD") - .context("Expect to have XXBTZUSD")? - .as_array() - .context("Expect to be an array")? - .iter() - .map(|value| -> color_eyre::Result<_> { - let array = value.as_array().context("Expect as_array to work")?; - - let date = Timestamp::from( - array - .first() - .context("Expect first to work")? - .as_u64() - .context("Expect as_u64 to work")? - as u32, - ) - .to_date(); - - let get_f32 = |index: usize| -> color_eyre::Result { - Ok(array - .get(index) - .context("Expect get index to work")? - .as_str() - .context("Expect as_str to work")? - .parse::()?) - }; - - Ok(( - date, - OHLC { - open: get_f32(1)?, - high: get_f32(2)?, - low: get_f32(3)?, - close: get_f32(4)?, - }, - )) - }) - .collect::, _>>()?) - }, - 30, - 10, - ) - } -} diff --git a/pricer/src/structs/cents.rs b/pricer/src/structs/cents.rs new file mode 100644 index 000000000..839c5fcea --- /dev/null +++ b/pricer/src/structs/cents.rs @@ -0,0 +1,14 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use super::Dollars; + +#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)] +pub struct Cents(u64); + +impl From for Cents { + fn from(value: Dollars) -> Self { + Self((*value * 100.0).floor() as u64) + } +} diff --git a/pricer/src/structs/close.rs b/pricer/src/structs/close.rs new file mode 100644 index 000000000..c3a15b543 --- /dev/null +++ b/pricer/src/structs/close.rs @@ -0,0 +1,12 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +#[repr(C)] +pub struct Close(T); +impl From for Close { + fn from(value: T) -> Self { + Self(value) + } +} diff --git a/computer/src/structs/date.rs b/pricer/src/structs/date.rs similarity index 95% rename from computer/src/structs/date.rs rename to pricer/src/structs/date.rs index 997559a0b..e4c422888 100644 --- a/computer/src/structs/date.rs +++ b/pricer/src/structs/date.rs @@ -5,7 +5,7 @@ use indexer::Timestamp; use jiff::{civil::Date as Date_, tz::TimeZone, Span}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct Date(u32); impl Date { diff --git a/computer/src/structs/dateindex.rs b/pricer/src/structs/dateindex.rs similarity index 100% rename from computer/src/structs/dateindex.rs rename to pricer/src/structs/dateindex.rs diff --git a/pricer/src/structs/dollars.rs b/pricer/src/structs/dollars.rs new file mode 100644 index 000000000..321106bba --- /dev/null +++ b/pricer/src/structs/dollars.rs @@ -0,0 +1,18 @@ +use derive_deref::Deref; + +use super::Cents; + +#[derive(Debug, Default, Clone, Copy, Deref)] +pub struct Dollars(f64); + +impl From for Dollars { + fn from(value: f64) -> Self { + Self(value) + } +} + +impl From for Dollars { + fn from(value: Cents) -> Self { + Self((*value as f64) / 100.0) + } +} diff --git a/pricer/src/structs/high.rs b/pricer/src/structs/high.rs new file mode 100644 index 000000000..51947090a --- /dev/null +++ b/pricer/src/structs/high.rs @@ -0,0 +1,12 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +#[repr(C)] +pub struct High(T); +impl From for High { + fn from(value: T) -> Self { + Self(value) + } +} diff --git a/pricer/src/structs/low.rs b/pricer/src/structs/low.rs new file mode 100644 index 000000000..3c4988fea --- /dev/null +++ b/pricer/src/structs/low.rs @@ -0,0 +1,12 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +#[repr(C)] +pub struct Low(T); +impl From for Low { + fn from(value: T) -> Self { + Self(value) + } +} diff --git a/pricer/src/structs/mod.rs b/pricer/src/structs/mod.rs new file mode 100644 index 000000000..5d1dfcf2b --- /dev/null +++ b/pricer/src/structs/mod.rs @@ -0,0 +1,19 @@ +mod cents; +mod close; +mod date; +mod dateindex; +mod dollars; +mod high; +mod low; +mod open; + +pub use cents::*; +pub use close::*; +pub use date::*; +pub use dateindex::*; +pub use dollars::*; +pub use high::*; +pub use low::*; +pub use open::*; + +pub type OHLC = (Open, High, Low, Close); diff --git a/pricer/src/structs/open.rs b/pricer/src/structs/open.rs new file mode 100644 index 000000000..b6632c90d --- /dev/null +++ b/pricer/src/structs/open.rs @@ -0,0 +1,12 @@ +use derive_deref::Deref; +use serde::Serialize; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)] +#[repr(C)] +pub struct Open(T); +impl From for Open { + fn from(value: T) -> Self { + Self(value) + } +} diff --git a/python/example.py b/python/example.py index a07c64a32..bffdf2c0f 100644 --- a/python/example.py +++ b/python/example.py @@ -2,11 +2,26 @@ # We're aiming to read the first 21 values from the height_to_timestamp vec import sys +# import struct import datetime with open("../_outputs/indexes/vecs/height_to_timestamp/vec", "rb") as file: for x in range(0, 21): - bytes = file.read(4) # Need to check the rust side to find the size, at least for now - number = int.from_bytes(bytes, sys.byteorder) + b = file.read(4) # Need to check the rust side to find the size, at least for now + number = int.from_bytes(b, sys.byteorder) date = datetime.date.fromtimestamp(number) print(date) + +# print(int.from_bytes([21], sys.byteorder)) # 21 u8 native endian +# print(int.from_bytes([21, 0], sys.byteorder)) # 21 u16 native endian +# print(int.from_bytes([21, 0, 0, 0], sys.byteorder)) # 21 u32 native endian +# print(int.from_bytes([21, 0, 0, 0, 0, 0, 0, 0], sys.byteorder)) # 21 u64/usize native endian + +# # check i8, ... + +# print(struct.unpack('f', bytes([0, 0, 168, 65]))) # 21.0 f32 native endian +# print(struct.unpack('d', bytes([0, 0, 0, 0, 0, 0, 53, 64]))) # 21.0 f64 native endian +# print(struct.unpack('f', bytes([65, 168, 0, 0]))) # 21.0 f32 big endian +# print(struct.unpack('>d', bytes([64, 53, 0, 0, 0, 0, 0, 0]))) # 21.0 f64 big endian diff --git a/server/Cargo.toml b/server/Cargo.toml index d925f55c5..e2ad61572 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -12,11 +12,9 @@ indexer = { workspace = true } jiff = { workspace = true } logger = { workspace = true } oxc = { version = "0.50.0", features = ["codegen", "minifier"] } -regex = "1.11.1" reqwest = { version = "0.12.12", features = ["blocking", "json"] } serde = { workspace = true } serde_json = { workspace = true } storable_vec = { workspace = true } tokio = { version = "1.43.0", features = ["full"] } tower-http = { version = "0.6.2", features = ["compression-full"] } -zstd = "0.13.2" diff --git a/server/src/api/vecs/mod.rs b/server/src/api/vecs/mod.rs index 2d8a39df7..c728b55ef 100644 --- a/server/src/api/vecs/mod.rs +++ b/server/src/api/vecs/mod.rs @@ -15,14 +15,14 @@ use crate::{log_result, traits::HeaderMapExtended}; use super::AppState; mod format; -mod id_to_i_to_vec; mod index; mod query; +mod tree; use format::Format; -pub use id_to_i_to_vec::*; use index::Index; use query::QueryS; +pub use tree::*; pub async fn handler( headers: HeaderMap, diff --git a/server/src/api/vecs/id_to_i_to_vec.rs b/server/src/api/vecs/tree.rs similarity index 100% rename from server/src/api/vecs/id_to_i_to_vec.rs rename to server/src/api/vecs/tree.rs