Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-10 15:03:32 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-13 19:00:52 +01:00
parent 443a32dc81
commit a1006dddb5
37 changed files with 547 additions and 880 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+14 -3
View File
@@ -262,14 +262,12 @@ dependencies = [
"jiff",
"logger",
"oxc",
"regex",
"reqwest",
"serde",
"serde_json",
"storable_vec",
"tokio",
"tower-http",
"zstd",
]
[[package]]
@@ -400,7 +398,7 @@ dependencies = [
"derive_deref",
"exit",
"fjall",
"jiff",
"pricer",
"storable_vec",
"zerocopy 0.8.17",
]
@@ -2111,6 +2109,17 @@ dependencies = [
[[package]]
name = "pricer"
version = "0.1.0"
dependencies = [
"bindex",
"color-eyre",
"derive_deref",
"jiff",
"logger",
"reqwest",
"serde",
"serde_json",
"zerocopy 0.8.17",
]
[[package]]
name = "proc-macro2"
@@ -2240,6 +2249,7 @@ version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [
"async-compression",
"base64 0.22.1",
"bytes",
"encoding_rs",
@@ -2270,6 +2280,7 @@ dependencies = [
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower",
"tower-service",
"url",
Cargo.toml
+1
View File
@@ -23,6 +23,7 @@ iterator = { path = "iterator", package = "biter" }
jiff = "0.2.0"
logger = { path = "logger" }
rayon = "1.10.0"
pricer = { path = "pricer" }
rlimit = { version = "0.10.2" }
serde = { version = "1.0.217", features = ["derive"] }
serde_json = { version = "1.0.138", features = ["float_roundtrip"] }
computer/Cargo.toml
+1 -1
View File
@@ -10,6 +10,6 @@ derive_deref = { workspace = true }
exit = { workspace = true }
fjall = { workspace = true }
indexer = { workspace = true }
jiff = { workspace = true }
pricer = { workspace = true }
storable_vec = { workspace = true }
zerocopy = { workspace = true }
computer/src/lib.rs
+1
View File
@@ -7,6 +7,7 @@ pub use iterator::rpc;
mod storage;
mod structs;
use pricer::Date;
use storable_vec::SINGLE_THREAD;
use storage::{Fjalls, StorableVecs};
pub use structs::*;
computer/src/storage/storable_vecs.rs
+6 -8
View File
@@ -1,12 +1,10 @@
use std::{fs, path::Path};
use indexer::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex};
use pricer::{Date, Dateindex};
use storable_vec::{StorableVec, Version};
use crate::{
structs::{Date, Feerate},
Dateindex,
};
use crate::structs::Feerate;
// mod base;
@@ -30,16 +28,16 @@ pub struct StorableVecs<const MODE: u8> {
// pub height_to_subsidy: StorableVec<Height, u32, MODE>,
// pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
// pub height_to_txcount: StorableVec<Height, u32, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Sats, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate, MODE>,
pub txindex_to_inputs_count: StorableVec<Txindex, u32, MODE>,
pub txindex_to_inputs_sum: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_inputs_sum: StorableVec<Txindex, Sats, MODE>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_outputs_count: StorableVec<Txindex, u32, MODE>,
pub txindex_to_outputs_sum: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_outputs_sum: StorableVec<Txindex, Sats, MODE>,
}
impl<const MODE: u8> StorableVecs<MODE> {
computer/src/structs/bitcoin.rs
+14
View File
@@ -0,0 +1,14 @@
use indexer::Sats;
#[derive(Debug, Default, Clone, Copy)]
pub struct Bitcoin(f64);
impl Bitcoin {
const ONE: Self = Self(100_000_000.0);
}
impl From<Sats> for Bitcoin {
fn from(value: Sats) -> Self {
Self((*value as f64) / Self::ONE.0)
}
}
computer/src/structs/mod.rs
+4 -4
View File
@@ -1,11 +1,11 @@
mod addressindextxoutindex;
mod date;
mod dateindex;
mod bitcoin;
mod feerate;
mod ohlc;
mod unit;
pub use addressindextxoutindex::*;
pub use date::*;
pub use dateindex::*;
pub use bitcoin::*;
pub use feerate::*;
pub use ohlc::*;
pub use unit::*;
computer/src/structs/ohlc.rs
+36
View File
@@ -0,0 +1,36 @@
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Cents, Close, High, Low, Open};
// #[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
// #[repr(C)]
// pub struct OHLCCents(OHLC<Cents>);
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
#[repr(C)]
pub struct OHLCCents(Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>);
impl OHLCCents {
pub fn open(&self) -> Open<Cents> {
self.0
}
pub fn high(&self) -> High<Cents> {
self.1
}
pub fn low(&self) -> Low<Cents> {
self.2
}
pub fn close(&self) -> Close<Cents> {
self.3
}
}
impl From<(Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)> for OHLCCents {
fn from(value: (Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>)) -> Self {
Self(value.0, value.1, value.2, value.3)
}
}
indexer/src/lib.rs
+3 -3
View File
@@ -92,7 +92,7 @@ impl Indexer<CACHED_GETS> {
iterator::new(bitcoin_dir, Some(height.into()), Some(400_000), rpc)
.iter()
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
info!("Processing block {_height}...");
info!("Indexing block {_height}...");
let blockhash = BlockHash::from(blockhash);
height = Height::from(_height);
@@ -440,13 +440,13 @@ impl Indexer<CACHED_GETS> {
(txout, txindex, vout, addresstype, addressbytes_res, addressindex_opt, _tx),
)|
-> color_eyre::Result<()> {
let amount = Amount::from(txout.value);
let sats = Sats::from(txout.value);
if vout.is_zero() {
vecs.txindex_to_first_txoutindex.push_if_needed(txindex, txoutindex)?;
}
vecs.txoutindex_to_amount.push_if_needed(txoutindex, amount)?;
vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?;
let mut addressindex = addressindex_global;
indexer/src/storage/storable_vecs/mod.rs
+8 -9
View File
@@ -5,11 +5,10 @@ use rayon::prelude::*;
use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Emptyindex, Height, LockTime,
Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes,
P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex,
P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex,
Unknownindex, Weight,
Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, Multisigindex,
Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
};
mod base;
@@ -56,7 +55,7 @@ pub struct StorableVecs<const MODE: u8> {
pub txindex_to_txversion: StorableVec<Txindex, TxVersion, MODE>,
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex, MODE>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex, MODE>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount, MODE>,
pub txoutindex_to_value: StorableVec<Txoutindex, Sats, MODE>,
}
// const UNSAFE_BLOCKS: usize = 1000;
@@ -177,7 +176,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
&path.join("txoutindex_to_addressindex"),
Version::from(1),
)?,
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?,
txoutindex_to_value: StorableVec::import(&path.join("txoutindex_to_value"), Version::from(1))?,
})
}
@@ -350,7 +349,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
&*self.txindex_to_txversion,
&*self.txinindex_to_txoutindex,
&*self.txoutindex_to_addressindex,
&*self.txoutindex_to_amount,
&*self.txoutindex_to_value,
]
}
@@ -395,7 +394,7 @@ impl<const MODE: u8> StorableVecs<MODE> {
&mut self.txindex_to_txversion,
&mut self.txinindex_to_txoutindex,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_amount,
&mut self.txoutindex_to_value,
]
}
}
indexer/src/structs/mod.rs
+2 -2
View File
@@ -2,11 +2,11 @@ mod addressbytes;
mod addressindex;
mod addresstype;
mod addresstypeindex;
mod amount;
mod blockhash;
mod compressed;
mod height;
mod locktime;
mod sats;
mod timestamp;
mod txid;
mod txindex;
@@ -21,11 +21,11 @@ pub use addressbytes::*;
pub use addressindex::*;
pub use addresstype::*;
pub use addresstypeindex::*;
pub use amount::*;
pub use blockhash::*;
pub use compressed::*;
pub use height::*;
pub use locktime::*;
pub use sats::*;
pub use timestamp::*;
pub use txid::*;
pub use txindex::*;
indexer/src/structs/amount.rs → indexer/src/structs/sats.rs
+31 -33
View File
@@ -4,7 +4,7 @@ use std::{
};
use derive_deref::{Deref, DerefMut};
use iterator::bitcoin;
use iterator::bitcoin::Amount;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -27,85 +27,83 @@ use super::Height;
KnownLayout,
Serialize,
)]
pub struct Amount(u64);
pub struct Sats(u64);
impl Amount {
impl Sats {
pub const ZERO: Self = Self(0);
pub const ONE_BTC_F32: f32 = 100_000_000.0;
pub const ONE_BTC_F64: f64 = 100_000_000.0;
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
}
}
impl Add for Amount {
type Output = Amount;
fn add(self, rhs: Amount) -> Self::Output {
Amount::from(*self + *rhs)
impl Add for Sats {
type Output = Sats;
fn add(self, rhs: Sats) -> Self::Output {
Sats::from(*self + *rhs)
}
}
impl AddAssign for Amount {
impl AddAssign for Sats {
fn add_assign(&mut self, rhs: Self) {
*self = *self + rhs;
}
}
impl Sub for Amount {
type Output = Amount;
fn sub(self, rhs: Amount) -> Self::Output {
Amount::from(*self - *rhs)
impl Sub for Sats {
type Output = Sats;
fn sub(self, rhs: Sats) -> Self::Output {
Sats::from(*self - *rhs)
}
}
impl SubAssign for Amount {
impl SubAssign for Sats {
fn sub_assign(&mut self, rhs: Self) {
*self = *self - rhs;
}
}
impl Mul<Amount> for Amount {
type Output = Amount;
fn mul(self, rhs: Amount) -> Self::Output {
Amount::from(*self * *rhs)
impl Mul<Sats> for Sats {
type Output = Sats;
fn mul(self, rhs: Sats) -> Self::Output {
Sats::from(*self * *rhs)
}
}
impl Mul<u64> for Amount {
type Output = Amount;
impl Mul<u64> for Sats {
type Output = Sats;
fn mul(self, rhs: u64) -> Self::Output {
Amount::from(*self * rhs)
Sats::from(*self * rhs)
}
}
impl Mul<Height> for Amount {
type Output = Amount;
impl Mul<Height> for Sats {
type Output = Sats;
fn mul(self, rhs: Height) -> Self::Output {
Amount::from(*self * *rhs as u64)
Sats::from(*self * *rhs as u64)
}
}
impl Sum for Amount {
impl Sum for Sats {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
let sats: u64 = iter.map(|amt| *amt).sum();
Amount::from(sats)
let sats: u64 = iter.map(|sats| *sats).sum();
Sats::from(sats)
}
}
impl From<u64> for Amount {
impl From<u64> for Sats {
fn from(value: u64) -> Self {
Self(value)
}
}
impl From<bitcoin::Amount> for Amount {
fn from(value: bitcoin::Amount) -> Self {
impl From<Amount> for Sats {
fn from(value: Amount) -> Self {
Self(value.to_sat())
}
}
impl From<Amount> for bitcoin::Amount {
fn from(value: Amount) -> Self {
impl From<Sats> for Amount {
fn from(value: Sats) -> Self {
Self::from_sat(value.0)
}
}
indexer/src/structs/vout.rs
+1 -1
View File
@@ -4,7 +4,7 @@ use derive_deref::Deref;
pub struct Vout(u32);
impl Vout {
const ZERO: Self = Vout(0_u32);
const ZERO: Self = Vout(0);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
pricer/Cargo.toml
+9
View File
@@ -4,3 +4,12 @@ version = "0.1.0"
edition = "2021"
[dependencies]
color-eyre = { workspace = true }
derive_deref = { workspace = true }
indexer = { workspace = true }
logger = { workspace = true }
reqwest = { version = "0.12.12", features = ["blocking", "brotli", "deflate", "gzip", "json", "zstd"] }
jiff = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
pricer/src/fetchers/binance.rs
+157
View File
@@ -0,0 +1,157 @@
use std::{
collections::BTreeMap,
fs::{self, File},
io::BufReader,
path::Path,
str::FromStr,
};
use color_eyre::eyre::{eyre, ContextCompat};
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use crate::{
fetchers::retry,
structs::{Cents, OHLC},
Close, Date, Dollars, High, Low, Open,
};
pub struct Binance;
impl Binance {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Binance...");
retry(
|_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url("interval=1m&limit=1000"))?.json()?),
30,
10,
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(
|_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url("interval=1d"))?.json()?),
30,
10,
)
}
pub fn read_har_file(path: &Path) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Reading Binance har file...");
fs::create_dir_all(&path)?;
let path_binance_har = path.join("binance.har");
let file = if let Ok(file) = File::open(path_binance_har) {
file
} else {
return Err(eyre!("Missing binance file"));
};
let reader = BufReader::new(file);
let json: BTreeMap<String, Value> = if let Ok(json) = serde_json::from_reader(reader) {
json
} else {
return Ok(Default::default());
};
json.get("log")
.context("Expect object to have log attribute")?
.as_object()
.context("Expect to be an object")?
.get("entries")
.context("Expect object to have entries")?
.as_array()
.context("Expect to be an array")?
.iter()
.filter(|entry| {
entry
.as_object()
.unwrap()
.get("request")
.unwrap()
.as_object()
.unwrap()
.get("url")
.unwrap()
.as_str()
.unwrap()
.contains("/uiKlines")
})
.map(|entry| {
let response = entry.as_object().unwrap().get("response").unwrap().as_object().unwrap();
let content = response.get("content").unwrap().as_object().unwrap();
let text = content.get("text");
if text.is_none() {
return Ok(BTreeMap::new());
}
let text = text.unwrap().as_str().unwrap();
Self::json_to_timestamp_to_ohlc(&serde_json::Value::from_str(text).unwrap())
})
.try_fold(BTreeMap::default(), |mut all, res| {
all.append(&mut res?);
Ok(all)
})
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
K: Ord,
{
json.as_array()
.context("Expect to be an array")?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> {
let array = array.as_array().context("Expect to be array")?;
let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32);
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
))
};
Ok((
timestamp,
OHLC::from((
Open::from(get_cents(1)),
High::from(get_cents(2)),
Low::from(get_cents(3)),
Close::from(get_cents(4)),
)),
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}
fn url(query: &str) -> String {
format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}")
}
}
pricer/src/price/kibo.rs → pricer/src/fetchers/kibo.rs
+8 -16
View File
@@ -1,14 +1,10 @@
use std::{collections::BTreeMap, str::FromStr};
use chrono::NaiveDate;
use color_eyre::eyre::ContextCompat;
use log::info;
use logger::info;
use serde_json::Value;
use crate::{
structs::{Date, DateMapChunkId, HeightMapChunkId, MapChunkId, OHLC},
utils::retry,
};
use crate::structs::{Date, OHLC};
pub struct Kibo;
@@ -33,11 +29,9 @@ impl Kibo {
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value = reqwest::blocking::get(format!(
"{base_url}/height-to-price?chunk={}",
chunk_id.to_usize()
))?
.json()?;
let body: Value =
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
let vec = body
.as_object()
@@ -68,11 +62,9 @@ impl Kibo {
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value = reqwest::blocking::get(format!(
"{base_url}/date-to-price?chunk={}",
chunk_id.to_usize()
))?
.json()?;
let body: Value =
reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
Ok(body
.as_object()
pricer/src/fetchers/kraken.rs
+90
View File
@@ -0,0 +1,90 @@
use std::collections::BTreeMap;
use color_eyre::eyre::ContextCompat;
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC};
pub struct Kraken;
impl Kraken {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Kraken...");
retry(
|_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url(1))?.json()?),
30,
10,
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(
|_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url(1440))?.json()?),
30,
10,
)
}
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
K: Ord,
{
json.as_object()
.context("Expect to be an object")?
.get("result")
.context("Expect object to have result")?
.as_object()
.context("Expect to be an object")?
.get("XXBTZUSD")
.context("Expect to have XXBTZUSD")?
.as_array()
.context("Expect to be an array")?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> {
let array = array.as_array().context("Expect to be array")?;
let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32);
let get_cents = |index: usize| {
Cents::from(Dollars::from(
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
))
};
Ok((
timestamp,
OHLC::from((
Open::from(get_cents(1)),
High::from(get_cents(2)),
Low::from(get_cents(3)),
Close::from(get_cents(4)),
)),
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}
fn url(interval: usize) -> String {
format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}")
}
}
pricer/src/price/mod.rs → pricer/src/fetchers/mod.rs
+2
View File
@@ -1,7 +1,9 @@
mod binance;
mod kibo;
mod kraken;
mod retry;
pub use binance::*;
pub use kibo::*;
pub use kraken::*;
use retry::*;
pricer/src/retry.rs → pricer/src/fetchers/retry.rs
+3
View File
@@ -1,5 +1,7 @@
use std::{thread::sleep, time::Duration};
use logger::info;
pub fn retry<T>(
function: impl Fn(usize) -> color_eyre::Result<T>,
sleep_in_s: u64,
@@ -13,6 +15,7 @@ pub fn retry<T>(
if i == retries || res.is_ok() {
return res;
} else {
info!("Failed, waiting {sleep_in_s} seconds...");
sleep(Duration::from_secs(sleep_in_s));
}
pricer/src/lib.rs
+4 -13
View File
@@ -1,14 +1,5 @@
pub fn add(left: u64, right: u64) -> u64 {
left + right
}
mod fetchers;
mod structs;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}
pub use fetchers::*;
pub use structs::*;
pricer/src/main.rs
+16
View File
@@ -0,0 +1,16 @@
// fn main() {}
use pricer::{Binance, Kraken};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
logger::init_log(None);
// dbg!(Binance::fetch_daily_prices());
// dbg!(Binance::fetch_1mn_prices());
// dbg!(Kraken::fetch_daily_prices());
dbg!(Kraken::fetch_1mn_prices());
Ok(())
}
pricer/src/price.rs
+17 -434
View File
@@ -9,8 +9,8 @@ use struct_iterable::Iterable;
use crate::{
parser::price::{Binance, Kibo, Kraken},
structs::{
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey,
MapKind, Timestamp, OHLC,
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, MapKind, Timestamp,
OHLC,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
@@ -34,54 +34,6 @@ pub struct PriceDatasets {
pub high: BiMap<f32>,
pub low: BiMap<f32>,
pub close: BiMap<f32>,
pub market_cap: BiMap<f32>,
pub price_1w_sma: BiMap<f32>,
pub price_1w_sma_ratio: RatioDataset,
pub price_1m_sma: BiMap<f32>,
pub price_1m_sma_ratio: RatioDataset,
pub price_1y_sma: BiMap<f32>,
pub price_1y_sma_ratio: RatioDataset,
pub price_2y_sma: BiMap<f32>,
pub price_2y_sma_ratio: RatioDataset,
pub price_4y_sma: BiMap<f32>,
pub price_4y_sma_ratio: RatioDataset,
pub price_8d_sma: BiMap<f32>,
pub price_8d_sma_ratio: RatioDataset,
pub price_13d_sma: BiMap<f32>,
pub price_13d_sma_ratio: RatioDataset,
pub price_21d_sma: BiMap<f32>,
pub price_21d_sma_ratio: RatioDataset,
pub price_34d_sma: BiMap<f32>,
pub price_34d_sma_ratio: RatioDataset,
pub price_55d_sma: BiMap<f32>,
pub price_55d_sma_ratio: RatioDataset,
pub price_89d_sma: BiMap<f32>,
pub price_89d_sma_ratio: RatioDataset,
pub price_144d_sma: BiMap<f32>,
pub price_144d_sma_ratio: RatioDataset,
pub price_200w_sma: BiMap<f32>,
pub price_200w_sma_ratio: RatioDataset,
pub price_1d_total_return: DateMap<f32>,
pub price_1m_total_return: DateMap<f32>,
pub price_6m_total_return: DateMap<f32>,
pub price_1y_total_return: DateMap<f32>,
pub price_2y_total_return: DateMap<f32>,
pub price_3y_total_return: DateMap<f32>,
pub price_4y_total_return: DateMap<f32>,
pub price_6y_total_return: DateMap<f32>,
pub price_8y_total_return: DateMap<f32>,
pub price_10y_total_return: DateMap<f32>,
pub price_4y_compound_return: DateMap<f32>,
// projection via lowest 4y compound value
pub all_time_high: BiMap<f32>,
pub all_time_high_date: DateMap<Date>,
pub days_since_all_time_high: DateMap<u32>,
pub max_days_between_all_time_highs: DateMap<u32>,
pub max_years_between_all_time_highs: DateMap<f32>,
pub market_price_to_all_time_high_ratio: BiMap<f32>,
pub drawdown: BiMap<f32>,
pub sats_per_dollar: BiMap<f32>,
// volatility
}
impl PriceDatasets {
@@ -108,116 +60,10 @@ impl PriceDatasets {
// ---
// Computed
// ---
open: BiMap::new_bin(1, MapKind::Computed, &f("open")),
high: BiMap::new_bin(1, MapKind::Computed, &f("high")),
low: BiMap::new_bin(1, MapKind::Computed, &f("low")),
open_cents: BiMap::new_bin(1, MapKind::Computed, &f("open")),
high_cents: BiMap::new_bin(1, MapKind::Computed, &f("high")),
low_cents: BiMap::new_bin(1, MapKind::Computed, &f("low")),
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
market_cap: BiMap::new_bin(1, MapKind::Computed, &f("market_cap")),
price_1w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1w_sma")),
price_1w_sma_ratio: RatioDataset::import(&path_dataset, "price_1w_sma", config)?,
price_1m_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1m_sma")),
price_1m_sma_ratio: RatioDataset::import(&path_dataset, "price_1m_sma", config)?,
price_1y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_1y_sma")),
price_1y_sma_ratio: RatioDataset::import(&path_dataset, "price_1y_sma", config)?,
price_2y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_2y_sma")),
price_2y_sma_ratio: RatioDataset::import(&path_dataset, "price_2y_sma", config)?,
price_4y_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_4y_sma")),
price_4y_sma_ratio: RatioDataset::import(&path_dataset, "price_4y_sma", config)?,
price_8d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_8d_sma")),
price_8d_sma_ratio: RatioDataset::import(&path_dataset, "price_8d_sma", config)?,
price_13d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_13d_sma")),
price_13d_sma_ratio: RatioDataset::import(&path_dataset, "price_13d_sma", config)?,
price_21d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_21d_sma")),
price_21d_sma_ratio: RatioDataset::import(&path_dataset, "price_21d_sma", config)?,
price_34d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_34d_sma")),
price_34d_sma_ratio: RatioDataset::import(&path_dataset, "price_34d_sma", config)?,
price_55d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_55d_sma")),
price_55d_sma_ratio: RatioDataset::import(&path_dataset, "price_55d_sma", config)?,
price_89d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_89d_sma")),
price_89d_sma_ratio: RatioDataset::import(&path_dataset, "price_89d_sma", config)?,
price_144d_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_144d_sma")),
price_144d_sma_ratio: RatioDataset::import(&path_dataset, "price_144d_sma", config)?,
price_200w_sma: BiMap::new_bin(1, MapKind::Computed, &f("price_200w_sma")),
price_200w_sma_ratio: RatioDataset::import(&path_dataset, "price_200w_sma", config)?,
price_1d_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1d_total_return"),
),
price_1m_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1m_total_return"),
),
price_6m_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_6m_total_return"),
),
price_1y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_1y_total_return"),
),
price_2y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_2y_total_return"),
),
price_3y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_3y_total_return"),
),
price_4y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_4y_total_return"),
),
price_6y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_6y_total_return"),
),
price_8y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_8y_total_return"),
),
price_10y_total_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_10y_total_return"),
),
price_4y_compound_return: DateMap::new_bin(
1,
MapKind::Computed,
&f("price_4y_compound_return"),
),
all_time_high: BiMap::new_bin(1, MapKind::Computed, &f("all_time_high")),
all_time_high_date: DateMap::new_bin(1, MapKind::Computed, &f("all_time_high_date")),
days_since_all_time_high: DateMap::new_bin(
1,
MapKind::Computed,
&f("days_since_all_time_high"),
),
max_days_between_all_time_highs: DateMap::new_bin(
1,
MapKind::Computed,
&f("max_days_between_all_time_highs"),
),
max_years_between_all_time_highs: DateMap::new_bin(
2,
MapKind::Computed,
&f("max_years_between_all_time_highs"),
),
market_price_to_all_time_high_ratio: BiMap::new_bin(
1,
MapKind::Computed,
&f("market_price_to_all_time_high_ratio"),
),
drawdown: BiMap::new_bin(1, MapKind::Computed, &f("drawdown")),
sats_per_dollar: BiMap::new_bin(1, MapKind::Computed, &f("sats_per_dollar")),
};
s.min_initial_states
@@ -240,213 +86,6 @@ impl PriceDatasets {
self.close
.multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close);
self.market_cap
.multi_insert_multiply(heights, dates, &mut self.close, circulating_supply);
self.price_1w_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
ONE_WEEK_IN_DAYS,
);
self.price_1m_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
ONE_MONTH_IN_DAYS,
);
self.price_1y_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
ONE_YEAR_IN_DAYS,
);
self.price_2y_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
2 * ONE_YEAR_IN_DAYS,
);
self.price_4y_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
4 * ONE_YEAR_IN_DAYS,
);
self.price_8d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 8);
self.price_13d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 13);
self.price_21d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 21);
self.price_34d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 34);
self.price_55d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 55);
self.price_89d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 89);
self.price_144d_sma
.multi_insert_simple_average(heights, dates, &mut self.close, 144);
self.price_200w_sma.multi_insert_simple_average(
heights,
dates,
&mut self.close,
200 * ONE_WEEK_IN_DAYS,
);
self.price_1d_total_return
.multi_insert_percentage_change(dates, &mut self.close.date, 1);
self.price_1m_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
ONE_MONTH_IN_DAYS,
);
self.price_6m_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
6 * ONE_MONTH_IN_DAYS,
);
self.price_1y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
ONE_YEAR_IN_DAYS,
);
self.price_2y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
2 * ONE_YEAR_IN_DAYS,
);
self.price_3y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
3 * ONE_YEAR_IN_DAYS,
);
self.price_4y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
4 * ONE_YEAR_IN_DAYS,
);
self.price_6y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
6 * ONE_YEAR_IN_DAYS,
);
self.price_8y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
8 * ONE_YEAR_IN_DAYS,
);
self.price_10y_total_return.multi_insert_percentage_change(
dates,
&mut self.close.date,
10 * ONE_YEAR_IN_DAYS,
);
self.price_4y_compound_return
.multi_insert_complex_transform(
dates,
&mut self.close.date,
|(last_value, date, closes, _)| {
let previous_value = date
.checked_sub_days(Days::new(4 * ONE_YEAR_IN_DAYS as u64))
.and_then(|date| closes.get_or_import(&Date::wrap(date)))
.unwrap_or_default();
(((last_value / previous_value).powf(1.0 / 4.0)) - 1.0) * 100.0
},
);
self.price_1w_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_1w_sma);
self.price_1m_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_1m_sma);
self.price_1y_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_1y_sma);
self.price_2y_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_2y_sma);
self.price_4y_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_4y_sma);
self.price_8d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_8d_sma);
self.price_13d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_13d_sma);
self.price_21d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_21d_sma);
self.price_34d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_34d_sma);
self.price_55d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_55d_sma);
self.price_89d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_89d_sma);
self.price_144d_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_144d_sma);
self.price_200w_sma_ratio
.compute(compute_data, &mut self.close, &mut self.price_200w_sma);
self.all_time_high
.multi_insert_max(heights, dates, &mut self.high);
self.market_price_to_all_time_high_ratio
.multi_insert_percentage(heights, dates, &mut self.close, &mut self.all_time_high);
self.all_time_high_date.multi_insert_complex_transform(
dates,
&mut self.all_time_high.date,
|(value, date, _, map)| {
let high = self.high.date.get_or_import(date).unwrap();
let is_ath = high == value;
if is_ath {
*date
} else {
let previous_date = date.checked_sub(1).unwrap();
*map.get_or_import(&previous_date).as_ref().unwrap_or(date)
}
},
);
self.days_since_all_time_high.multi_insert_simple_transform(
dates,
&mut self.all_time_high_date,
|value, key| key.difference_in_days_between(value),
);
self.max_days_between_all_time_highs
.multi_insert_max(dates, &mut self.days_since_all_time_high);
self.max_years_between_all_time_highs
.multi_insert_simple_transform(
dates,
&mut self.max_days_between_all_time_highs,
|days, _| (days as f64 / ONE_YEAR_IN_DAYS as f64) as f32,
);
self.drawdown.multi_insert_simple_transform(
heights,
dates,
&mut self.market_price_to_all_time_high_ratio,
&|v| -(100.0 - v),
);
self.sats_per_dollar.multi_insert_simple_transform(
heights,
dates,
&mut self.close,
&|price| Amount::ONE_BTC_F32 / price,
);
}
pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLC> {
@@ -469,17 +108,9 @@ impl PriceDatasets {
#[allow(clippy::map_entry)]
if !self.kibo_by_date.contains_key(&chunk_id)
|| self
.kibo_by_date
.get(&chunk_id)
.unwrap()
.last_key_value()
.unwrap()
.0
< date
|| self.kibo_by_date.get(&chunk_id).unwrap().last_key_value().unwrap().0 < date
{
self.kibo_by_date
.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?);
self.kibo_by_date.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?);
}
self.kibo_by_date
@@ -491,16 +122,7 @@ impl PriceDatasets {
}
fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.kraken_daily.is_none()
|| self
.kraken_daily
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
< date
{
if self.kraken_daily.is_none() || self.kraken_daily.as_ref().unwrap().last_key_value().unwrap().0 < date {
self.kraken_daily.replace(Kraken::fetch_daily_prices()?);
}
@@ -513,16 +135,7 @@ impl PriceDatasets {
}
fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.binance_daily.is_none()
|| self
.binance_daily
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
< date
{
if self.binance_daily.is_none() || self.binance_daily.as_ref().unwrap().last_key_value().unwrap().0 < date {
self.binance_daily.replace(Binance::fetch_daily_prices()?);
}
@@ -593,8 +206,7 @@ How to fix this:
#[allow(clippy::map_entry)]
if !self.kibo_by_height.contains_key(&chunk_id)
|| ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len())
<= height.to_usize())
|| ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) <= height.to_usize())
{
self.kibo_by_height
.insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?);
@@ -613,16 +225,7 @@ How to fix this:
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.kraken_1mn.is_none()
|| self
.kraken_1mn
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= &timestamp
{
if self.kraken_1mn.is_none() || self.kraken_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?);
}
@@ -634,25 +237,11 @@ How to fix this:
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.binance_1mn.is_none()
|| self
.binance_1mn
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= &timestamp
{
if self.binance_1mn.is_none() || self.binance_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self.binance_1mn.replace(Binance::fetch_1mn_prices()?);
}
Self::find_height_ohlc(
&self.binance_1mn,
timestamp,
previous_timestamp,
"binance 1m",
)
Self::find_height_ohlc(&self.binance_1mn, timestamp, previous_timestamp, "binance 1m")
}
fn get_from_har_binance(
@@ -666,12 +255,7 @@ How to fix this:
.replace(Binance::read_har_file(config).unwrap_or_default());
}
Self::find_height_ohlc(
&self.binance_har,
timestamp,
previous_timestamp,
"binance har",
)
Self::find_height_ohlc(&self.binance_har, timestamp, previous_timestamp, "binance har")
}
fn find_height_ohlc(
@@ -684,10 +268,9 @@ How to fix this:
let err = Error::msg(format!("Couldn't find timestamp in {name}"));
let previous_ohlc = previous_timestamp
.map_or(Some(OHLC::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
});
let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
});
let last_ohlc = tree.get(&timestamp);
pricer/src/price/binance.rs
-213
View File
@@ -1,213 +0,0 @@
#![allow(dead_code)]
use std::{collections::BTreeMap, fs};
use color_eyre::eyre::ContextCompat;
use itertools::Itertools;
use log::info;
use serde_json::Value;
use crate::{
io::Json,
structs::{Config, Date, Timestamp, OHLC},
utils::retry,
};
pub struct Binance;
impl Binance {
pub fn read_har_file(config: &Config) -> color_eyre::Result<BTreeMap<u32, OHLC>> {
info!("binance: read har file");
let path = config.path_inputs();
fs::create_dir_all(&path)?;
let path_binance_har = path.join("binance.har");
let json: BTreeMap<String, Value> = Json::import(&path_binance_har).unwrap_or_default();
Ok(json
.get("log")
.context("Expect object to have log attribute")?
.as_object()
.context("Expect to be an object")?
.get("entries")
.context("Expect object to have entries")?
.as_array()
.context("Expect to be an array")?
.iter()
.filter(|entry| {
entry
.as_object()
.unwrap()
.get("request")
.unwrap()
.as_object()
.unwrap()
.get("url")
.unwrap()
.as_str()
.unwrap()
.contains("/uiKlines")
})
.flat_map(|entry| {
let response = entry
.as_object()
.unwrap()
.get("response")
.unwrap()
.as_object()
.unwrap();
let content = response.get("content").unwrap().as_object().unwrap();
let text = content.get("text");
if text.is_none() {
return vec![];
}
let text = text.unwrap().as_str().unwrap();
let arrays: Value = serde_json::from_str(text).unwrap();
arrays
.as_array()
.unwrap()
.iter()
.map(|array| {
let array = array.as_array().unwrap();
let timestamp = (array.first().unwrap().as_u64().unwrap() / 1000) as u32;
let get_f32 = |index: usize| {
array
.get(index)
.unwrap()
.as_str()
.unwrap()
.parse::<f32>()
.unwrap()
};
(
timestamp,
OHLC {
open: get_f32(1),
high: get_f32(2),
low: get_f32(3),
close: get_f32(4),
},
)
})
.collect_vec()
})
.collect::<BTreeMap<_, _>>())
}
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
info!("binance: fetch 1mn");
retry(
|_| {
let body: Value = reqwest::blocking::get(
"https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000",
)?
.json()?;
Ok(body
.as_array()
.context("Expect to be an array")?
.iter()
.map(|value| -> color_eyre::Result<_> {
// [timestamp, open, high, low, close, volume, ...]
let array = value.as_array().context("Expect to be array")?;
let timestamp = (array
.first()
.context("Expect to have first")?
.as_u64()
.context("Expect to be convertible to u64")?
/ 1_000) as u32;
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
Ok(array
.get(index)
.context("Expect to have index")?
.as_str()
.context("Expect to have &str")?
.parse::<f32>()?)
};
Ok((
timestamp,
OHLC {
open: get_f32(1)?,
high: get_f32(2)?,
low: get_f32(3)?,
close: get_f32(4)?,
},
))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
},
30,
10,
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("binance: fetch 1d");
retry(
|_| {
let body: Value = reqwest::blocking::get(
"https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d",
)?
.json()?;
Ok(body
.as_array()
.context("Expect to be an array")?
.iter()
.map(|value| -> color_eyre::Result<_> {
// [timestamp, open, high, low, close, volume, ...]
let array = value.as_array().context("Expect to be array")?;
let date = Timestamp::from(
(array
.first()
.context("Expect to have first")?
.as_u64()
.context("Expect to be convertible to u64")?
/ 1_000) as u32,
)
.to_date();
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
Ok(array
.get(index)
.context("Expect to have index")?
.as_str()
.context("Expect to have &str")?
.parse::<f32>()?)
};
Ok((
date,
OHLC {
open: get_f32(1)?,
high: get_f32(2)?,
low: get_f32(3)?,
close: get_f32(4)?,
},
))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
},
30,
10,
)
}
}
pricer/src/price/kraken.rs
-133
View File
@@ -1,133 +0,0 @@
use std::collections::BTreeMap;
use color_eyre::eyre::ContextCompat;
use log::info;
use serde_json::Value;
use crate::{
structs::{Date, Timestamp, OHLC},
utils::retry,
};
pub struct Kraken;
impl Kraken {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
info!("kraken: fetch 1mn");
retry(
|_| {
let body: Value = reqwest::blocking::get(
"https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1",
)?
.json()?;
Ok(body
.as_object()
.context("Expect to be an object")?
.get("result")
.context("Expect object to have result")?
.as_object()
.context("Expect to be an object")?
.get("XXBTZUSD")
.context("Expect to have XXBTZUSD")?
.as_array()
.context("Expect to be an array")?
.iter()
.map(|value| -> color_eyre::Result<_> {
let array = value.as_array().context("Expect as_array to work")?;
let timestamp = array
.first()
.context("Expect first to work")?
.as_u64()
.expect("Expect as_u64 to work")
as u32;
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
Ok(array
.get(index)
.context("Expect get index to work")?
.as_str()
.context("Expect as_str to work")?
.parse::<f32>()?)
};
Ok((
timestamp,
OHLC {
open: get_f32(1)?,
high: get_f32(2)?,
low: get_f32(3)?,
close: get_f32(4)?,
},
))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
},
30,
10,
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("fetch kraken daily");
retry(
|_| {
let body: Value = reqwest::blocking::get(
"https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440",
)?
.json()?;
Ok(body
.as_object()
.context("Expect to be an object")?
.get("result")
.context("Expect object to have result")?
.as_object()
.context("Expect to be an object")?
.get("XXBTZUSD")
.context("Expect to have XXBTZUSD")?
.as_array()
.context("Expect to be an array")?
.iter()
.map(|value| -> color_eyre::Result<_> {
let array = value.as_array().context("Expect as_array to work")?;
let date = Timestamp::from(
array
.first()
.context("Expect first to work")?
.as_u64()
.context("Expect as_u64 to work")?
as u32,
)
.to_date();
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
Ok(array
.get(index)
.context("Expect get index to work")?
.as_str()
.context("Expect as_str to work")?
.parse::<f32>()?)
};
Ok((
date,
OHLC {
open: get_f32(1)?,
high: get_f32(2)?,
low: get_f32(3)?,
close: get_f32(4)?,
},
))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
},
30,
10,
)
}
}
pricer/src/structs/cents.rs
+14
View File
@@ -0,0 +1,14 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Dollars;
#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
pub struct Cents(u64);
impl From<Dollars> for Cents {
fn from(value: Dollars) -> Self {
Self((*value * 100.0).floor() as u64)
}
}
pricer/src/structs/close.rs
+12
View File
@@ -0,0 +1,12 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
#[repr(C)]
pub struct Close<T>(T);
impl<T> From<T> for Close<T> {
fn from(value: T) -> Self {
Self(value)
}
}
computer/src/structs/date.rs → pricer/src/structs/date.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use indexer::Timestamp;
use jiff::{civil::Date as Date_, tz::TimeZone, Span};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Date(u32);
impl Date {
computer/src/structs/dateindex.rs → pricer/src/structs/dateindex.rs
View File
pricer/src/structs/dollars.rs
+18
View File
@@ -0,0 +1,18 @@
use derive_deref::Deref;
use super::Cents;
#[derive(Debug, Default, Clone, Copy, Deref)]
pub struct Dollars(f64);
impl From<f64> for Dollars {
fn from(value: f64) -> Self {
Self(value)
}
}
impl From<Cents> for Dollars {
fn from(value: Cents) -> Self {
Self((*value as f64) / 100.0)
}
}
pricer/src/structs/high.rs
+12
View File
@@ -0,0 +1,12 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
#[repr(C)]
pub struct High<T>(T);
impl<T> From<T> for High<T> {
fn from(value: T) -> Self {
Self(value)
}
}
pricer/src/structs/low.rs
+12
View File
@@ -0,0 +1,12 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
#[repr(C)]
pub struct Low<T>(T);
impl<T> From<T> for Low<T> {
fn from(value: T) -> Self {
Self(value)
}
}
pricer/src/structs/mod.rs
+19
View File
@@ -0,0 +1,19 @@
mod cents;
mod close;
mod date;
mod dateindex;
mod dollars;
mod high;
mod low;
mod open;
pub use cents::*;
pub use close::*;
pub use date::*;
pub use dateindex::*;
pub use dollars::*;
pub use high::*;
pub use low::*;
pub use open::*;
pub type OHLC = (Open<Cents>, High<Cents>, Low<Cents>, Close<Cents>);
pricer/src/structs/open.rs
+12
View File
@@ -0,0 +1,12 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
#[repr(C)]
pub struct Open<T>(T);
impl<T> From<T> for Open<T> {
fn from(value: T) -> Self {
Self(value)
}
}
python/example.py
+17 -2
View File
@@ -2,11 +2,26 @@
# We're aiming to read the first 21 values from the height_to_timestamp vec
import sys
# import struct
import datetime
with open("../_outputs/indexes/vecs/height_to_timestamp/vec", "rb") as file:
for x in range(0, 21):
bytes = file.read(4) # Need to check the rust side to find the size, at least for now
number = int.from_bytes(bytes, sys.byteorder)
b = file.read(4) # Need to check the rust side to find the size, at least for now
number = int.from_bytes(b, sys.byteorder)
date = datetime.date.fromtimestamp(number)
print(date)
# print(int.from_bytes([21], sys.byteorder)) # 21 u8 native endian
# print(int.from_bytes([21, 0], sys.byteorder)) # 21 u16 native endian
# print(int.from_bytes([21, 0, 0, 0], sys.byteorder)) # 21 u32 native endian
# print(int.from_bytes([21, 0, 0, 0, 0, 0, 0, 0], sys.byteorder)) # 21 u64/usize native endian
# # check i8, ...
# print(struct.unpack('f', bytes([0, 0, 168, 65]))) # 21.0 f32 native endian
# print(struct.unpack('d', bytes([0, 0, 0, 0, 0, 0, 53, 64]))) # 21.0 f64 native endian
# print(struct.unpack('<f', bytes([0, 0, 168, 65]))) # 21.0 f32 little endian
# print(struct.unpack('<d', bytes([0, 0, 0, 0, 0, 0, 53, 64]))) # 21.0 f64 little endian
# print(struct.unpack('>f', bytes([65, 168, 0, 0]))) # 21.0 f32 big endian
# print(struct.unpack('>d', bytes([64, 53, 0, 0, 0, 0, 0, 0]))) # 21.0 f64 big endian
server/Cargo.toml
-2
View File
@@ -12,11 +12,9 @@ indexer = { workspace = true }
jiff = { workspace = true }
logger = { workspace = true }
oxc = { version = "0.50.0", features = ["codegen", "minifier"] }
regex = "1.11.1"
reqwest = { version = "0.12.12", features = ["blocking", "json"] }
serde = { workspace = true }
serde_json = { workspace = true }
storable_vec = { workspace = true }
tokio = { version = "1.43.0", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
zstd = "0.13.2"
server/src/api/vecs/mod.rs
+2 -2
View File
@@ -15,14 +15,14 @@ use crate::{log_result, traits::HeaderMapExtended};
use super::AppState;
mod format;
mod id_to_i_to_vec;
mod index;
mod query;
mod tree;
use format::Format;
pub use id_to_i_to_vec::*;
use index::Index;
use query::QueryS;
pub use tree::*;
pub async fn handler(
headers: HeaderMap,
server/src/api/vecs/id_to_i_to_vec.rs → server/src/api/vecs/tree.rs
View File