mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-04-25 07:09:59 -07:00
global: snapshot
This commit is contained in:
nym21
157
pricer/src/fetchers/binance.rs
Normal file
157
pricer/src/fetchers/binance.rs
Normal file
@@ -0,0 +1,157 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::{self, File},
|
||||
io::BufReader,
|
||||
path::Path,
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::{eyre, ContextCompat};
|
||||
use indexer::Timestamp;
|
||||
use logger::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
fetchers::retry,
|
||||
structs::{Cents, OHLC},
|
||||
Close, Date, Dollars, High, Low, Open,
|
||||
};
|
||||
|
||||
pub struct Binance;
|
||||
|
||||
impl Binance {
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
|
||||
info!("Fetching 1mn prices from Binance...");
|
||||
|
||||
retry(
|
||||
|_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url("interval=1m&limit=1000"))?.json()?),
|
||||
30,
|
||||
10,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
info!("Fetching daily prices from Kraken...");
|
||||
|
||||
retry(
|
||||
|_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url("interval=1d"))?.json()?),
|
||||
30,
|
||||
10,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn read_har_file(path: &Path) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
|
||||
info!("Reading Binance har file...");
|
||||
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
let path_binance_har = path.join("binance.har");
|
||||
|
||||
let file = if let Ok(file) = File::open(path_binance_har) {
|
||||
file
|
||||
} else {
|
||||
return Err(eyre!("Missing binance file"));
|
||||
};
|
||||
|
||||
let reader = BufReader::new(file);
|
||||
|
||||
let json: BTreeMap<String, Value> = if let Ok(json) = serde_json::from_reader(reader) {
|
||||
json
|
||||
} else {
|
||||
return Ok(Default::default());
|
||||
};
|
||||
|
||||
json.get("log")
|
||||
.context("Expect object to have log attribute")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("entries")
|
||||
.context("Expect object to have entries")?
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.filter(|entry| {
|
||||
entry
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.get("request")
|
||||
.unwrap()
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.get("url")
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("/uiKlines")
|
||||
})
|
||||
.map(|entry| {
|
||||
let response = entry.as_object().unwrap().get("response").unwrap().as_object().unwrap();
|
||||
|
||||
let content = response.get("content").unwrap().as_object().unwrap();
|
||||
|
||||
let text = content.get("text");
|
||||
|
||||
if text.is_none() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
|
||||
let text = text.unwrap().as_str().unwrap();
|
||||
|
||||
Self::json_to_timestamp_to_ohlc(&serde_json::Value::from_str(text).unwrap())
|
||||
})
|
||||
.try_fold(BTreeMap::default(), |mut all, res| {
|
||||
all.append(&mut res?);
|
||||
Ok(all)
|
||||
})
|
||||
}
|
||||
|
||||
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
|
||||
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
|
||||
}
|
||||
|
||||
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
|
||||
}
|
||||
|
||||
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
|
||||
where
|
||||
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
|
||||
K: Ord,
|
||||
{
|
||||
json.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(fun)
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()
|
||||
}
|
||||
|
||||
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> {
|
||||
let array = array.as_array().context("Expect to be array")?;
|
||||
|
||||
let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32);
|
||||
|
||||
let get_cents = |index: usize| {
|
||||
Cents::from(Dollars::from(
|
||||
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
|
||||
))
|
||||
};
|
||||
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC::from((
|
||||
Open::from(get_cents(1)),
|
||||
High::from(get_cents(2)),
|
||||
Low::from(get_cents(3)),
|
||||
Close::from(get_cents(4)),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> {
|
||||
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
|
||||
}
|
||||
|
||||
fn url(query: &str) -> String {
|
||||
format!("https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&{query}")
|
||||
}
|
||||
}
|
||||
110
pricer/src/fetchers/kibo.rs
Normal file
110
pricer/src/fetchers/kibo.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use std::{collections::BTreeMap, str::FromStr};
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use logger::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::structs::{Date, OHLC};
|
||||
|
||||
pub struct Kibo;
|
||||
|
||||
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
|
||||
const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api";
|
||||
|
||||
const RETRIES: usize = 10;
|
||||
|
||||
impl Kibo {
|
||||
fn get_base_url(try_index: usize) -> &'static str {
|
||||
if try_index < RETRIES / 2 {
|
||||
KIBO_OFFICIAL_URL
|
||||
} else {
|
||||
KIBO_OFFICIAL_BACKUP_URL
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
|
||||
info!("kibo: fetch height prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
let base_url = Self::get_base_url(try_index);
|
||||
|
||||
let body: Value =
|
||||
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))?
|
||||
.json()?;
|
||||
|
||||
let vec = body
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
.context("Expect object to have dataset")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("map")
|
||||
.context("Expect to have map")?
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(vec)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
info!("kibo: fetch date prices");
|
||||
|
||||
retry(
|
||||
|try_index| {
|
||||
let base_url = Self::get_base_url(try_index);
|
||||
|
||||
let body: Value =
|
||||
reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))?
|
||||
.json()?;
|
||||
|
||||
Ok(body
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
.context("Expect object to have dataset")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("map")
|
||||
.context("Expect to have map")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.iter()
|
||||
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
|
||||
let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
|
||||
Ok((date, Self::value_to_ohlc(value)?))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLC> {
|
||||
let ohlc = value.as_object().context("Expect as_object to work")?;
|
||||
|
||||
let get_value = |key: &str| -> color_eyre::Result<f32> {
|
||||
Ok(ohlc
|
||||
.get(key)
|
||||
.context("Expect get key to work")?
|
||||
.as_f64()
|
||||
.context("Expect as_f64 to work")? as f32)
|
||||
};
|
||||
|
||||
Ok(OHLC {
|
||||
open: get_value("open")?,
|
||||
high: get_value("high")?,
|
||||
low: get_value("low")?,
|
||||
close: get_value("close")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
90
pricer/src/fetchers/kraken.rs
Normal file
90
pricer/src/fetchers/kraken.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use indexer::Timestamp;
|
||||
use logger::info;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC};
|
||||
|
||||
pub struct Kraken;
|
||||
|
||||
impl Kraken {
|
||||
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
|
||||
info!("Fetching 1mn prices from Kraken...");
|
||||
|
||||
retry(
|
||||
|_| Self::json_to_timestamp_to_ohlc(&reqwest::blocking::get(Self::url(1))?.json()?),
|
||||
30,
|
||||
10,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
info!("Fetching daily prices from Kraken...");
|
||||
|
||||
retry(
|
||||
|_| Self::json_to_date_to_ohlc(&reqwest::blocking::get(Self::url(1440))?.json()?),
|
||||
30,
|
||||
10,
|
||||
)
|
||||
}
|
||||
|
||||
fn json_to_timestamp_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
|
||||
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
|
||||
}
|
||||
|
||||
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
|
||||
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
|
||||
}
|
||||
|
||||
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
|
||||
where
|
||||
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
|
||||
K: Ord,
|
||||
{
|
||||
json.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("result")
|
||||
.context("Expect object to have result")?
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("XXBTZUSD")
|
||||
.context("Expect to have XXBTZUSD")?
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(fun)
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()
|
||||
}
|
||||
|
||||
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLC)> {
|
||||
let array = array.as_array().context("Expect to be array")?;
|
||||
|
||||
let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32);
|
||||
|
||||
let get_cents = |index: usize| {
|
||||
Cents::from(Dollars::from(
|
||||
array.get(index).unwrap().as_str().unwrap().parse::<f64>().unwrap(),
|
||||
))
|
||||
};
|
||||
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC::from((
|
||||
Open::from(get_cents(1)),
|
||||
High::from(get_cents(2)),
|
||||
Low::from(get_cents(3)),
|
||||
Close::from(get_cents(4)),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLC)> {
|
||||
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
|
||||
}
|
||||
|
||||
fn url(interval: usize) -> String {
|
||||
format!("https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval={interval}")
|
||||
}
|
||||
}
|
||||
9
pricer/src/fetchers/mod.rs
Normal file
9
pricer/src/fetchers/mod.rs
Normal file
@@ -0,0 +1,9 @@
|
||||
mod binance;
|
||||
mod kibo;
|
||||
mod kraken;
|
||||
mod retry;
|
||||
|
||||
pub use binance::*;
|
||||
pub use kibo::*;
|
||||
pub use kraken::*;
|
||||
use retry::*;
|
||||
24
pricer/src/fetchers/retry.rs
Normal file
24
pricer/src/fetchers/retry.rs
Normal file
@@ -0,0 +1,24 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use logger::info;
|
||||
|
||||
pub fn retry<T>(
|
||||
function: impl Fn(usize) -> color_eyre::Result<T>,
|
||||
sleep_in_s: u64,
|
||||
retries: usize,
|
||||
) -> color_eyre::Result<T> {
|
||||
let mut i = 0;
|
||||
|
||||
loop {
|
||||
let res = function(i);
|
||||
|
||||
if i == retries || res.is_ok() {
|
||||
return res;
|
||||
} else {
|
||||
info!("Failed, waiting {sleep_in_s} seconds...");
|
||||
sleep(Duration::from_secs(sleep_in_s));
|
||||
}
|
||||
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user