Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-26 07:39:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: add auto fetch price from main instance

Browse Source
This commit is contained in:
k
2024-07-22 15:58:02 +02:00
parent 232276d106
commit 9871fdffc9
9 changed files with 183 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
parser/src/bitcoin/daemon.rs
View File

@@ -96,7 +96,7 @@ impl BitcoinDaemon {
fn get_blockchain_info(&self) -> BlockchainInfo {
retry(
|| {
|_| {
// bitcoin-cli -datadir=/Users/k/Developer/bitcoin getblockchaininfo
let output = Command::new("bitcoin-cli")
.arg(self.datadir_arg())
@@ -122,7 +122,7 @@ impl BitcoinDaemon {
Ok(BlockchainInfo { headers, blocks })
},
1,
u64::MAX,
usize::MAX,
)
.unwrap()
}

70
parser/src/datasets/price/mod.rs
View File

@@ -9,8 +9,11 @@ use color_eyre::eyre::Error;
pub use ohlc::*;
use crate::{
price::{Binance, Kraken},
structs::{AnyBiMap, AnyDateMap, BiMap, Date, DateMap, Height, MapKey},
price::{Binance, Kraken, Satonomics},
structs::{
AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId,
MapKey,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
@@ -24,7 +27,8 @@ pub struct PriceDatasets {
kraken_1mn: Option<BTreeMap<u32, OHLC>>,
binance_1mn: Option<BTreeMap<u32, OHLC>>,
binance_har: Option<BTreeMap<u32, OHLC>>,
satonomics_by_height: BTreeMap<usize, Option<BTreeMap<usize, OHLC>>>,
satonomics_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
satonomics_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
// Inserted
pub ohlcs: BiMap<OHLC>,
@@ -89,6 +93,7 @@ impl PriceDatasets {
kraken_1mn: None,
kraken_daily: None,
satonomics_by_height: BTreeMap::default(),
satonomics_by_date: BTreeMap::default(),
ohlcs: BiMap::new_json(1, price_path),
closes: BiMap::new_bin(1, &f("close")),
@@ -304,7 +309,9 @@ impl PriceDatasets {
if self.ohlcs.date.is_key_safe(date) {
Ok(self.ohlcs.date.get(&date).unwrap().to_owned())
} else {
let ohlc = self.get_from_daily_kraken(&date)?;
let ohlc = self
.get_from_date_satonomics(&date)
.or_else(|_| self.get_from_daily_kraken(&date))?;
self.ohlcs.date.insert(date, ohlc);
@@ -312,12 +319,27 @@ impl PriceDatasets {
}
}
fn get_from_date_satonomics(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
let chunk_id = date.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_date.contains_key(&chunk_id) {
self.satonomics_by_date
.insert(chunk_id, Satonomics::fetch_date_prices(chunk_id)?);
}
self.satonomics_by_date
.get(&chunk_id)
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date in satonomics"))
}
fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.kraken_daily.is_none() {
self.kraken_daily.replace(
Kraken::fetch_daily_prices()
.unwrap_or_else(|_| Binance::fetch_daily_prices().unwrap()),
);
self.kraken_daily
.replace(Kraken::fetch_daily_prices().or_else(|_| Binance::fetch_daily_prices())?);
}
self.kraken_daily
@@ -325,7 +347,7 @@ impl PriceDatasets {
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date in daily kraken"))
.ok_or(Error::msg("Couldn't find date"))
}
pub fn get_height_ohlc(
@@ -358,15 +380,17 @@ impl PriceDatasets {
let previous_timestamp = previous_timestamp.map(clean_timestamp);
let ohlc = self
.get_from_1mn_kraken(timestamp, previous_timestamp)
.get_from_height_satonomics(&height)
.unwrap_or_else(|_| {
self.get_from_1mn_binance(timestamp, previous_timestamp)
self.get_from_1mn_kraken(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_har_binance(timestamp, previous_timestamp)
self.get_from_1mn_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
let date = Date::from_timestamp(timestamp);
self.get_from_har_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
let date = Date::from_timestamp(timestamp);
panic!(
panic!(
"Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
@@ -381,6 +405,7 @@ How to fix this:
9. Move the file to 'parser/imports/binance.har'
"
)
})
})
})
});
@@ -390,6 +415,23 @@ How to fix this:
Ok(ohlc)
}
fn get_from_height_satonomics(&mut self, height: &Height) -> color_eyre::Result<OHLC> {
let chunk_id = height.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_height.contains_key(&chunk_id) {
self.satonomics_by_height
.insert(chunk_id, Satonomics::fetch_height_prices(chunk_id)?);
}
self.satonomics_by_height
.get(&chunk_id)
.unwrap()
.get(height.to_serialized_key().to_usize())
.cloned()
.ok_or(Error::msg("Couldn't find height in satonomics"))
}
fn get_from_1mn_kraken(
&mut self,
timestamp: u32,

4
parser/src/price/binance.rs
View File

@@ -107,7 +107,7 @@ impl Binance {
log("binance: fetch 1mn");
retry(
|| {
|_| {
let body: Value = reqwest::blocking::get(
"https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1m&limit=1000",
)?
@@ -154,7 +154,7 @@ impl Binance {
log("binance: fetch 1d");
retry(
|| {
|_| {
let body: Value = reqwest::blocking::get(
"https://api.binance.com/api/v3/uiKlines?symbol=BTCUSDT&interval=1d",
)?

4
parser/src/price/kraken.rs
View File

@@ -16,7 +16,7 @@ impl Kraken {
log("kraken: fetch 1mn");
retry(
|| {
|_| {
let body: Value = reqwest::blocking::get(
"https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1",
)?
@@ -70,7 +70,7 @@ impl Kraken {
log("fetch kraken daily");
retry(
|| {
|_| {
let body: Value = reqwest::blocking::get(
"https://api.kraken.com/0/public/OHLC?pair=XBTUSD&interval=1440",
)?

2
parser/src/price/mod.rs
View File

@@ -1,5 +1,7 @@
mod binance;
mod kraken;
mod satonomics;
pub use binance::*;
pub use kraken::*;
pub use satonomics::*;

113
parser/src/price/satonomics.rs Normal file
View File

@@ -0,0 +1,113 @@
use std::{collections::BTreeMap, str::FromStr};
use chrono::NaiveDate;
use color_eyre::eyre::ContextCompat;
use itertools::Itertools;
use serde_json::Value;
use crate::{
datasets::OHLC,
structs::{Date, DateMapChunkId, HeightMapChunkId},
utils::{log, retry},
MapChunkId,
};
pub struct Satonomics;
const SATONOMICS_OFFICIAL_URL: &str = "https://api.satonomics.xyz";
const SATONOMICS_OFFICIAL_BACKUP_URL: &str = "https://api-bkp.satonomics.xyz";
const RETRIES: usize = 10;
impl Satonomics {
fn get_base_url(try_index: usize) -> &'static str {
if try_index < RETRIES / 2 {
SATONOMICS_OFFICIAL_URL
} else {
SATONOMICS_OFFICIAL_BACKUP_URL
}
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
log("satonomics: fetch height prices");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value = reqwest::blocking::get(format!(
"{base_url}/height-to-price?chunk={}",
chunk_id.to_usize()
))?
.json()?;
Ok(body
.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
.as_object()
.context("Expect to be an object")?
.get("map")
.context("Expect to have map")?
.as_array()
.context("Expect to be an array")?
.iter()
.map(Self::value_to_ohlc)
.collect_vec())
},
10,
RETRIES,
)
}
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
log("satonomics: date height prices");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value = reqwest::blocking::get(format!(
"{base_url}/date-to-price?chunk={}",
chunk_id.to_usize()
))?
.json()?;
Ok(body
.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
.as_object()
.context("Expect to be an object")?
.get("map")
.context("Expect to have map")?
.as_object()
.context("Expect to be an object")?
.iter()
.map(|(serialized_date, value)| {
let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap());
(date, Self::value_to_ohlc(value))
})
.collect::<BTreeMap<_, _>>())
},
10,
RETRIES,
)
}
fn value_to_ohlc(value: &Value) -> OHLC {
let ohlc = value.as_object().unwrap();
let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32;
OHLC {
open: get_value("open"),
high: get_value("high"),
low: get_value("low"),
close: get_value("close"),
}
}
}

6
parser/src/utils/retry.rs
View File

@@ -1,9 +1,9 @@
use std::{thread::sleep, time::Duration};
pub fn retry<T>(
function: impl Fn() -> color_eyre::Result<T>,
function: impl Fn(usize) -> color_eyre::Result<T>,
sleep_in_s: u64,
retries: u64,
retries: usize,
) -> color_eyre::Result<T> {
if retries < 1 {
unreachable!()
@@ -12,7 +12,7 @@ pub fn retry<T>(
let mut i = 0;
loop {
let res = function();
let res = function(i);
if i == retries || res.is_ok() {
return res;