Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-18 02:39:43 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser: price fetch fixes

Browse Source
This commit is contained in:
k
2024-09-19 10:49:26 +02:00
parent 1acfcf088c
commit 7b296e4863
11 changed files with 80 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parser/src/datasets/price/mod.rs
+27 -28
View File
@@ -9,7 +9,7 @@ use color_eyre::eyre::Error;
pub use ohlc::*;
use crate::{
price::{Binance, Kraken, Satonomics},
price::{Binance, Kibo, Kraken},
structs::{
AnyBiMap, AnyDateMap, BiMap, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId,
MapKey,
@@ -28,8 +28,8 @@ pub struct PriceDatasets {
binance_1mn: Option<BTreeMap<u32, OHLC>>,
binance_daily: Option<BTreeMap<Date, OHLC>>,
binance_har: Option<BTreeMap<u32, OHLC>>,
satonomics_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
satonomics_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
kibo_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
kibo_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
// Inserted
pub ohlcs: BiMap<OHLC>,
@@ -94,8 +94,8 @@ impl PriceDatasets {
binance_har: None,
kraken_1mn: None,
kraken_daily: None,
satonomics_by_height: BTreeMap::default(),
satonomics_by_date: BTreeMap::default(),
kibo_by_height: BTreeMap::default(),
kibo_by_date: BTreeMap::default(),
ohlcs: BiMap::new_json(1, price_path),
closes: BiMap::new_bin(1, &f("close")),
@@ -314,7 +314,7 @@ impl PriceDatasets {
let ohlc = self
.get_from_daily_kraken(&date)
.or_else(|_| self.get_from_daily_binance(&date))
.or_else(|_| self.get_from_date_satonomics(&date))?;
.or_else(|_| self.get_from_date_kibo(&date))?;
self.ohlcs.date.insert(date, ohlc);
@@ -322,25 +322,25 @@ impl PriceDatasets {
}
}
fn get_from_date_satonomics(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
let chunk_id = date.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_date.contains_key(&chunk_id)
if !self.kibo_by_date.contains_key(&chunk_id)
|| self
.satonomics_by_date
.kibo_by_date
.get(&chunk_id)
.unwrap()
.last_key_value()
.unwrap()
.0
<= date
< date
{
self.satonomics_by_date
.insert(chunk_id, Satonomics::fetch_date_prices(chunk_id)?);
self.kibo_by_date
.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?);
}
self.satonomics_by_date
self.kibo_by_date
.get(&chunk_id)
.unwrap()
.get(date)
@@ -379,7 +379,7 @@ impl PriceDatasets {
.last_key_value()
.unwrap()
.0
<= date
< date
{
self.binance_daily.replace(Binance::fetch_daily_prices()?);
}
@@ -428,12 +428,11 @@ impl PriceDatasets {
.unwrap_or_else(|_| {
self.get_from_har_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_height_satonomics(&height)
.unwrap_or_else(|_| {
let date = Date::from_timestamp(timestamp);
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
let date = Date::from_timestamp(timestamp);
panic!(
"Can't find the price for: height: {height} - date: {date}
panic!(
"Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot
@@ -446,8 +445,8 @@ How to fix this:
8. Export to a har file (if there is no explicit button, click on the cog button)
9. Move the file to 'parser/imports/binance.har'
"
)
})
)
})
})
})
});
@@ -457,24 +456,24 @@ How to fix this:
Ok(ohlc)
}
fn get_from_height_satonomics(&mut self, height: &Height) -> color_eyre::Result<OHLC> {
fn get_from_height_kibo(&mut self, height: &Height) -> color_eyre::Result<OHLC> {
let chunk_id = height.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_height.contains_key(&chunk_id)
|| ((chunk_id.to_usize() + self.satonomics_by_height.get(&chunk_id).unwrap().len())
if !self.kibo_by_height.contains_key(&chunk_id)
|| ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len())
<= height.to_usize())
{
self.satonomics_by_height
.insert(chunk_id, Satonomics::fetch_height_prices(chunk_id)?);
self.kibo_by_height
.insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?);
}
self.satonomics_by_height
self.kibo_by_height
.get(&chunk_id)
.unwrap()
.get(height.to_serialized_key().to_usize())
.cloned()
.ok_or(Error::msg("Couldn't find height in satonomics"))
.ok_or(Error::msg("Couldn't find height in kibo"))
}
fn get_from_1mn_kraken(
parser/src/price/binance.rs
+4 -3
View File
@@ -122,7 +122,7 @@ impl Binance {
// [timestamp, open, high, low, close, volume, ...]
let array = value.as_array().unwrap();
let timestamp = (array.first().unwrap().as_u64().unwrap() / 1000) as u32;
let timestamp = (array.first().unwrap().as_u64().unwrap() / 1_000) as u32;
let get_f32 = |index: usize| {
array
@@ -169,8 +169,9 @@ impl Binance {
// [timestamp, open, high, low, close, volume, ...]
let array = value.as_array().unwrap();
let date =
Date::from_timestamp(array.first().unwrap().as_u64().unwrap() as u32);
let date = Date::from_timestamp(
(array.first().unwrap().as_u64().unwrap() / 1_000) as u32,
);
let get_f32 = |index: usize| {
array
parser/src/price/satonomics.rs → parser/src/price/kibo.rs
+8 -8
View File
@@ -12,24 +12,24 @@ use crate::{
MapChunkId,
};
pub struct Satonomics;
pub struct Kibo;
const SATONOMICS_OFFICIAL_URL: &str = "https://api.satonomics.xyz";
const SATONOMICS_OFFICIAL_BACKUP_URL: &str = "https://api-bkp.satonomics.xyz";
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api";
const RETRIES: usize = 10;
impl Satonomics {
impl Kibo {
fn get_base_url(try_index: usize) -> &'static str {
if try_index < RETRIES / 2 {
SATONOMICS_OFFICIAL_URL
KIBO_OFFICIAL_URL
} else {
SATONOMICS_OFFICIAL_BACKUP_URL
KIBO_OFFICIAL_BACKUP_URL
}
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
log("satonomics: fetch height prices");
log("kibo: fetch height prices");
retry(
|try_index| {
@@ -62,7 +62,7 @@ impl Satonomics {
}
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
log("satonomics: date height prices");
log("kibo: fetch date prices");
retry(
|try_index| {
parser/src/price/mod.rs
+2 -2
View File
@@ -1,7 +1,7 @@
mod binance;
mod kibo;
mod kraken;
mod satonomics;
pub use binance::*;
pub use kibo::*;
pub use kraken::*;
pub use satonomics::*;