Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 07:09:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

pricer: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-14 19:02:46 +01:00
parent a1006dddb5
commit ed10dccfe2
22 changed files with 692 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
pricer/src/fetchers/binance.rs
View File

@@ -2,7 +2,7 @@ use std::{
collections::BTreeMap,
fs::{self, File},
io::BufReader,
path::Path,
path::{Path, PathBuf},
str::FromStr,
};
@@ -10,17 +10,48 @@ use color_eyre::eyre::{eyre, ContextCompat};
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use storable_vec::STATELESS;
use crate::{
fetchers::retry,
structs::{Cents, OHLC},
Close, Date, Dollars, High, Low, Open,
Close, Date, Dollars, High, Low, Open, Pricer,
};
pub struct Binance;
pub struct Binance {
path: PathBuf,
_1mn: Option<BTreeMap<Timestamp, OHLC>>,
_1d: Option<BTreeMap<Date, OHLC>>,
har: Option<BTreeMap<Timestamp, OHLC>>,
}
impl Binance {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn init(path: &Path) -> Self {
Self {
path: path.to_owned(),
_1mn: None,
_1d: None,
har: None,
}
}
fn get_from_1mn(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?);
}
Pricer::<STATELESS>::find_height_ohlc(
&self._1mn.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance 1mn",
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Binance...");
retry(
@@ -30,7 +61,20 @@ impl Binance {
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date {
self._1d.replace(Self::fetch_1d()?);
}
self._1d
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
}
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(
@@ -40,10 +84,28 @@ impl Binance {
)
}
pub fn read_har_file(path: &Path) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn get_from_har_binance(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.har.is_none() {
self.har.replace(self.read_har().unwrap_or_default());
}
Pricer::<STATELESS>::find_height_ohlc(
&self.har.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance har",
)
}
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Reading Binance har file...");
fs::create_dir_all(&path)?;
let path = &self.path;
fs::create_dir_all(path)?;
let path_binance_har = path.join("binance.har");

102
pricer/src/fetchers/kibo.rs
View File

@@ -1,12 +1,21 @@
use std::{collections::BTreeMap, str::FromStr};
use color_eyre::eyre::ContextCompat;
use indexer::Height;
use logger::info;
use serde_json::Value;
use crate::structs::{Date, OHLC};
use crate::{
fetchers::retry,
structs::{Date, OHLC},
Cents, Close, Dollars, High, Low, Open,
};
pub struct Kibo;
#[derive(Default)]
pub struct Kibo {
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLC>>,
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLC>>,
}
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api";
@@ -22,16 +31,32 @@ impl Kibo {
}
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
info!("kibo: fetch height prices");
pub fn get_from_height_kibo(&mut self, height: Height) -> color_eyre::Result<OHLC> {
#[allow(clippy::map_entry)]
if !self.height_to_ohlc_vec.contains_key(&height)
|| ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len()) <= usize::from(height))
{
self.height_to_ohlc_vec
.insert(height, Self::fetch_height_prices(height)?);
}
self.height_to_ohlc_vec
.get(&height)
.unwrap()
.get(usize::from(height))
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLC>> {
info!("Fetching Kibo height prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value =
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", height))?.json()?;
let vec = body
.as_object()
@@ -55,19 +80,41 @@ impl Kibo {
)
}
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("kibo: fetch date prices");
pub fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
let year = date.year();
#[allow(clippy::map_entry)]
if !self.year_to_date_to_ohlc.contains_key(&year)
|| self
.year_to_date_to_ohlc
.get(&year)
.unwrap()
.last_key_value()
.unwrap()
.0
< date
{
self.year_to_date_to_ohlc.insert(year, Self::fetch_date_prices(year)?);
}
self.year_to_date_to_ohlc
.get(&year)
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo"))
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching Kibo date prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value =
reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
let body: Value = reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", year))?.json()?;
Ok(body
.as_object()
body.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
@@ -79,10 +126,10 @@ impl Kibo {
.context("Expect to be an object")?
.iter()
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
let date = Date::from(jiff::civil::Date::from_str(serialized_date).unwrap());
Ok((date, Self::value_to_ohlc(value)?))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
.collect::<Result<BTreeMap<_, _>, _>>()
},
30,
RETRIES,
@@ -92,19 +139,20 @@ impl Kibo {
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLC> {
let ohlc = value.as_object().context("Expect as_object to work")?;
let get_value = |key: &str| -> color_eyre::Result<f32> {
Ok(ohlc
.get(key)
.context("Expect get key to work")?
.as_f64()
.context("Expect as_f64 to work")? as f32)
let get_value = |key: &str| -> color_eyre::Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(key)
.context("Expect get key to work")?
.as_f64()
.context("Expect as_f64 to work")?,
)))
};
Ok(OHLC {
open: get_value("open")?,
high: get_value("high")?,
low: get_value("low")?,
close: get_value("close")?,
})
Ok((
Open::from(get_value("open")?),
High::from(get_value("high")?),
Low::from(get_value("low")?),
Close::from(get_value("close")?),
))
}
}

36
pricer/src/fetchers/kraken.rs
View File

@@ -4,13 +4,29 @@ use color_eyre::eyre::ContextCompat;
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use storable_vec::STATELESS;
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC};
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, Pricer, OHLC};
pub struct Kraken;
#[derive(Default)]
pub struct Kraken {
_1mn: Option<BTreeMap<Timestamp, OHLC>>,
_1d: Option<BTreeMap<Date, OHLC>>,
}
impl Kraken {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn get_from_1mn(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?);
}
Pricer::<STATELESS>::find_height_ohlc(&self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
}
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Kraken...");
retry(
@@ -20,7 +36,19 @@ impl Kraken {
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date {
self._1d.replace(Kraken::fetch_1d()?);
}
self._1d
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
}
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(