Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-09 06:31:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

pricer: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-14 19:02:46 +01:00
parent a1006dddb5
commit ed10dccfe2
22 changed files with 692 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -29,6 +29,7 @@ docker/kibo
# Types
paths.d.ts
vecid-to-indexes.d.ts
# Outputs
_outputs
Cargo.lock
Generated
+1
View File
@@ -2118,6 +2118,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
"storable_vec",
"zerocopy 0.8.17",
]
computer/src/structs/mod.rs
+2 -2
View File
@@ -1,11 +1,11 @@
mod addressindextxoutindex;
mod bitcoin;
mod feerate;
mod ohlc;
// mod ohlc;
mod unit;
pub use addressindextxoutindex::*;
pub use bitcoin::*;
pub use feerate::*;
pub use ohlc::*;
// pub use ohlc::*;
pub use unit::*;
pricer/Cargo.toml
+9 -1
View File
@@ -8,8 +8,16 @@ color-eyre = { workspace = true }
derive_deref = { workspace = true }
indexer = { workspace = true }
logger = { workspace = true }
reqwest = { version = "0.12.12", features = ["blocking", "brotli", "deflate", "gzip", "json", "zstd"] }
reqwest = { version = "0.12.12", features = [
"blocking",
"brotli",
"deflate",
"gzip",
"json",
"zstd",
] }
jiff = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
storable_vec = { workspace = true }
zerocopy = { workspace = true }
pricer/src/fetchers/binance.rs
+69 -7
View File
@@ -2,7 +2,7 @@ use std::{
collections::BTreeMap,
fs::{self, File},
io::BufReader,
path::Path,
path::{Path, PathBuf},
str::FromStr,
};
@@ -10,17 +10,48 @@ use color_eyre::eyre::{eyre, ContextCompat};
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use storable_vec::STATELESS;
use crate::{
fetchers::retry,
structs::{Cents, OHLC},
Close, Date, Dollars, High, Low, Open,
Close, Date, Dollars, High, Low, Open, Pricer,
};
pub struct Binance;
pub struct Binance {
path: PathBuf,
_1mn: Option<BTreeMap<Timestamp, OHLC>>,
_1d: Option<BTreeMap<Date, OHLC>>,
har: Option<BTreeMap<Timestamp, OHLC>>,
}
impl Binance {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn init(path: &Path) -> Self {
Self {
path: path.to_owned(),
_1mn: None,
_1d: None,
har: None,
}
}
fn get_from_1mn(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?);
}
Pricer::<STATELESS>::find_height_ohlc(
&self._1mn.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance 1mn",
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Binance...");
retry(
@@ -30,7 +61,20 @@ impl Binance {
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date {
self._1d.replace(Self::fetch_1d()?);
}
self._1d
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
}
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(
@@ -40,10 +84,28 @@ impl Binance {
)
}
pub fn read_har_file(path: &Path) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn get_from_har_binance(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.har.is_none() {
self.har.replace(self.read_har().unwrap_or_default());
}
Pricer::<STATELESS>::find_height_ohlc(
&self.har.as_ref().unwrap(),
timestamp,
previous_timestamp,
"binance har",
)
}
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Reading Binance har file...");
fs::create_dir_all(&path)?;
let path = &self.path;
fs::create_dir_all(path)?;
let path_binance_har = path.join("binance.har");
pricer/src/fetchers/kibo.rs
+75 -27
View File
@@ -1,12 +1,21 @@
use std::{collections::BTreeMap, str::FromStr};
use color_eyre::eyre::ContextCompat;
use indexer::Height;
use logger::info;
use serde_json::Value;
use crate::structs::{Date, OHLC};
use crate::{
fetchers::retry,
structs::{Date, OHLC},
Cents, Close, Dollars, High, Low, Open,
};
pub struct Kibo;
#[derive(Default)]
pub struct Kibo {
height_to_ohlc_vec: BTreeMap<Height, Vec<OHLC>>,
year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLC>>,
}
const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
const KIBO_OFFICIAL_BACKUP_URL: &str = "https://backup.kibo.money/api";
@@ -22,16 +31,32 @@ impl Kibo {
}
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {
info!("kibo: fetch height prices");
pub fn get_from_height_kibo(&mut self, height: Height) -> color_eyre::Result<OHLC> {
#[allow(clippy::map_entry)]
if !self.height_to_ohlc_vec.contains_key(&height)
|| ((usize::from(height) + self.height_to_ohlc_vec.get(&height).unwrap().len()) <= usize::from(height))
{
self.height_to_ohlc_vec
.insert(height, Self::fetch_height_prices(height)?);
}
self.height_to_ohlc_vec
.get(&height)
.unwrap()
.get(usize::from(height))
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find height in kibo"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLC>> {
info!("Fetching Kibo height prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value =
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
reqwest::blocking::get(format!("{base_url}/height-to-price?chunk={}", height))?.json()?;
let vec = body
.as_object()
@@ -55,19 +80,41 @@ impl Kibo {
)
}
pub fn fetch_date_prices(chunk_id: DateMapChunkId) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("kibo: fetch date prices");
pub fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
let year = date.year();
#[allow(clippy::map_entry)]
if !self.year_to_date_to_ohlc.contains_key(&year)
|| self
.year_to_date_to_ohlc
.get(&year)
.unwrap()
.last_key_value()
.unwrap()
.0
< date
{
self.year_to_date_to_ohlc.insert(year, Self::fetch_date_prices(year)?);
}
self.year_to_date_to_ohlc
.get(&year)
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date in kibo"))
}
fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching Kibo date prices...");
retry(
|try_index| {
let base_url = Self::get_base_url(try_index);
let body: Value =
reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", chunk_id.to_usize()))?
.json()?;
let body: Value = reqwest::blocking::get(format!("{base_url}/date-to-price?chunk={}", year))?.json()?;
Ok(body
.as_object()
body.as_object()
.context("Expect to be an object")?
.get("dataset")
.context("Expect object to have dataset")?
@@ -79,10 +126,10 @@ impl Kibo {
.context("Expect to be an object")?
.iter()
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
let date = Date::from(jiff::civil::Date::from_str(serialized_date).unwrap());
Ok((date, Self::value_to_ohlc(value)?))
})
.collect::<Result<BTreeMap<_, _>, _>>()?)
.collect::<Result<BTreeMap<_, _>, _>>()
},
30,
RETRIES,
@@ -92,19 +139,20 @@ impl Kibo {
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLC> {
let ohlc = value.as_object().context("Expect as_object to work")?;
let get_value = |key: &str| -> color_eyre::Result<f32> {
Ok(ohlc
.get(key)
.context("Expect get key to work")?
.as_f64()
.context("Expect as_f64 to work")? as f32)
let get_value = |key: &str| -> color_eyre::Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(key)
.context("Expect get key to work")?
.as_f64()
.context("Expect as_f64 to work")?,
)))
};
Ok(OHLC {
open: get_value("open")?,
high: get_value("high")?,
low: get_value("low")?,
close: get_value("close")?,
})
Ok((
Open::from(get_value("open")?),
High::from(get_value("high")?),
Low::from(get_value("low")?),
Close::from(get_value("close")?),
))
}
}
pricer/src/fetchers/kraken.rs
+32 -4
View File
@@ -4,13 +4,29 @@ use color_eyre::eyre::ContextCompat;
use indexer::Timestamp;
use logger::info;
use serde_json::Value;
use storable_vec::STATELESS;
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, OHLC};
use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, Pricer, OHLC};
pub struct Kraken;
#[derive(Default)]
pub struct Kraken {
_1mn: Option<BTreeMap<Timestamp, OHLC>>,
_1d: Option<BTreeMap<Date, OHLC>>,
}
impl Kraken {
pub fn fetch_1mn_prices() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
pub fn get_from_1mn(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self._1mn.is_none() || self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self._1mn.replace(Self::fetch_1mn()?);
}
Pricer::<STATELESS>::find_height_ohlc(&self._1mn.as_ref().unwrap(), timestamp, previous_timestamp, "kraken 1m")
}
fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLC>> {
info!("Fetching 1mn prices from Kraken...");
retry(
@@ -20,7 +36,19 @@ impl Kraken {
)
}
pub fn fetch_daily_prices() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 < date {
self._1d.replace(Kraken::fetch_1d()?);
}
self._1d
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
}
fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLC>> {
info!("Fetching daily prices from Kraken...");
retry(
pricer/src/lib.rs
+272
View File
@@ -1,5 +1,277 @@
use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
};
use color_eyre::eyre::Error;
mod fetchers;
mod structs;
pub use fetchers::*;
use indexer::{Height, Indexer, Timestamp};
use storable_vec::{AnyJsonStorableVec, AnyStorableVec, StorableVec, Version, SINGLE_THREAD};
pub use structs::*;
pub struct Pricer<const MODE: u8> {
path: PathBuf,
binance: Binance,
kraken: Kraken,
kibo: Kibo,
pub dateindex_to_close_in_cents: StorableVec<Dateindex, Close<Cents>, MODE>,
pub dateindex_to_close_in_dollars: StorableVec<Dateindex, Close<Dollars>, MODE>,
pub dateindex_to_high_in_cents: StorableVec<Dateindex, High<Cents>, MODE>,
pub dateindex_to_high_in_dollars: StorableVec<Dateindex, High<Dollars>, MODE>,
pub dateindex_to_low_in_cents: StorableVec<Dateindex, Low<Cents>, MODE>,
pub dateindex_to_low_in_dollars: StorableVec<Dateindex, Low<Dollars>, MODE>,
pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>, MODE>,
pub dateindex_to_open_in_dollars: StorableVec<Dateindex, Open<Dollars>, MODE>,
pub height_to_close_in_cents: StorableVec<Height, Close<Cents>, MODE>,
pub height_to_close_in_dollars: StorableVec<Height, Close<Dollars>, MODE>,
pub height_to_high_in_cents: StorableVec<Height, High<Cents>, MODE>,
pub height_to_high_in_dollars: StorableVec<Height, High<Dollars>, MODE>,
pub height_to_low_in_cents: StorableVec<Height, Low<Cents>, MODE>,
pub height_to_low_in_dollars: StorableVec<Height, Low<Dollars>, MODE>,
pub height_to_open_in_cents: StorableVec<Height, Open<Cents>, MODE>,
pub height_to_open_in_dollars: StorableVec<Height, Open<Dollars>, MODE>,
}
impl<const MODE: u8> Pricer<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
path: path.to_owned(),
binance: Binance::init(path),
kraken: Kraken::default(),
kibo: Kibo::default(),
// binance_1mn: None,
// binance_daily: None,
// binance_har: None,
// kraken_1mn: None,
// kraken_daily: None,
// kibo_by_height: BTreeMap::default(),
// kibo_by_date: BTreeMap::default(),
dateindex_to_close_in_cents: StorableVec::import(
&path.join("dateindex_to_close_in_cents"),
Version::from(1),
)?,
dateindex_to_close_in_dollars: StorableVec::import(
&path.join("dateindex_to_close_in_dollars"),
Version::from(1),
)?,
dateindex_to_high_in_cents: StorableVec::import(
&path.join("dateindex_to_high_in_cents"),
Version::from(1),
)?,
dateindex_to_high_in_dollars: StorableVec::import(
&path.join("dateindex_to_high_in_dollars"),
Version::from(1),
)?,
dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?,
dateindex_to_low_in_dollars: StorableVec::import(
&path.join("dateindex_to_low_in_dollars"),
Version::from(1),
)?,
dateindex_to_open_in_cents: StorableVec::import(
&path.join("dateindex_to_open_in_cents"),
Version::from(1),
)?,
dateindex_to_open_in_dollars: StorableVec::import(
&path.join("dateindex_to_open_in_dollars"),
Version::from(1),
)?,
height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?,
height_to_close_in_dollars: StorableVec::import(
&path.join("height_to_close_in_dollars"),
Version::from(1),
)?,
height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?,
height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?,
height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?,
height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?,
height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?,
height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?,
})
}
pub fn compute_if_needed(&mut self, indexer: &mut Indexer<SINGLE_THREAD>) {
// TODO: Remove all outdated
indexer
.vecs
.height_to_timestamp
.iter_from(Height::default(), |v| Ok(()));
// self.open
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open);
// self.high
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high);
// self.low
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low);
// self.close
// .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close);
}
fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLC> {
if self.ohlc.date.is_key_safe(date) {
Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned())
} else {
let ohlc = self
.get_from_daily_kraken(&date)
.or_else(|_| self.get_from_daily_binance(&date))
.or_else(|_| self.get_from_date_kibo(&date))?;
self.ohlc.date.insert(date, ohlc);
Ok(ohlc)
}
}
fn get_height_ohlc(
&mut self,
height: Height,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if let Some(ohlc) = self.ohlc.height.get_or_import(&height) {
return Ok(ohlc);
}
let timestamp = timestamp.to_floored_seconds();
if previous_timestamp.is_none() && !height.is_first() {
panic!("Shouldn't be possible");
}
let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds());
let ohlc = self
.get_from_1mn_kraken(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_1mn_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_har_binance(timestamp, previous_timestamp, config)
.unwrap_or_else(|_| {
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
let date = timestamp.to_date();
panic!(
"Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot
2. Select 1mn interval
3. Open the inspector/dev tools
4. Go to the Network Tab
5. Filter URLs by 'uiKlines'
6. Go back to the chart and scroll until you pass the date mentioned few lines ago
7. Go back to the dev tools
8. Export to a har file (if there is no explicit button, click on the cog button)
9. Move the file to 'parser/imports/binance.har'
"
)
})
})
})
});
// self.ohlc.height.insert(height, ohlc);
Ok(ohlc)
}
fn find_height_ohlc(
tree: &BTreeMap<Timestamp, OHLC>,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
name: &str,
) -> color_eyre::Result<OHLC> {
let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
});
let last_ohlc = tree.get(&timestamp);
if previous_ohlc.is_none() || last_ohlc.is_none() {
return Err(Error::msg(format!("Couldn't find timestamp in {name}")));
}
let previous_ohlc = previous_ohlc.unwrap();
let mut final_ohlc = (
Open::from(previous_ohlc.3),
High::from(previous_ohlc.3),
Low::from(previous_ohlc.3),
previous_ohlc.3,
);
let start = previous_timestamp.unwrap_or(Timestamp::from(0));
let end = timestamp;
// Otherwise it's a re-org
if start < end {
tree.range(start..=end).skip(1).for_each(|(_, ohlc)| {
if ohlc.1 > final_ohlc.1 {
final_ohlc.1 = ohlc.1
}
if ohlc.2 < final_ohlc.2 {
final_ohlc.2 = ohlc.2
}
final_ohlc.3 = ohlc.3;
});
}
Ok(final_ohlc)
}
pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 16] {
[
&self.dateindex_to_close_in_cents as &dyn AnyJsonStorableVec,
&self.dateindex_to_close_in_dollars,
&self.dateindex_to_high_in_cents,
&self.dateindex_to_high_in_dollars,
&self.dateindex_to_low_in_cents,
&self.dateindex_to_low_in_dollars,
&self.dateindex_to_open_in_cents,
&self.dateindex_to_open_in_dollars,
&self.height_to_close_in_cents,
&self.height_to_close_in_dollars,
&self.height_to_high_in_cents,
&self.height_to_high_in_dollars,
&self.height_to_low_in_cents,
&self.height_to_low_in_dollars,
&self.height_to_open_in_cents,
&self.height_to_open_in_dollars,
]
}
pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 16] {
[
&mut self.dateindex_to_close_in_cents as &mut dyn AnyStorableVec,
&mut self.dateindex_to_close_in_dollars,
&mut self.dateindex_to_high_in_cents,
&mut self.dateindex_to_high_in_dollars,
&mut self.dateindex_to_low_in_cents,
&mut self.dateindex_to_low_in_dollars,
&mut self.dateindex_to_open_in_cents,
&mut self.dateindex_to_open_in_dollars,
&mut self.height_to_close_in_cents,
&mut self.height_to_close_in_dollars,
&mut self.height_to_high_in_cents,
&mut self.height_to_high_in_dollars,
&mut self.height_to_low_in_cents,
&mut self.height_to_low_in_dollars,
&mut self.height_to_open_in_cents,
&mut self.height_to_open_in_dollars,
]
}
}
pricer/src/main.rs
+7 -4
View File
@@ -1,16 +1,19 @@
// fn main() {}
use pricer::{Binance, Kraken};
use indexer::Height;
use pricer::{Binance, Kibo, Kraken};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
logger::init_log(None);
// dbg!(Binance::fetch_daily_prices());
dbg!(Binance::fetch_1d_prices()?);
// dbg!(Binance::fetch_1mn_prices());
// dbg!(Kraken::fetch_daily_prices());
dbg!(Kraken::fetch_1mn_prices());
dbg!(Kraken::fetch_1d()?);
// dbg!(Kraken::fetch_1mn_prices()?);
dbg!(Kibo::fetch_date_prices(2025)?);
dbg!(Kibo::fetch_height_prices(Height::from(880_000_u32))?);
Ok(())
}
pricer/src/price.rs
-316
View File
@@ -1,316 +0,0 @@
use std::collections::BTreeMap;
use allocative::Allocative;
use chrono::Days;
use color_eyre::eyre::Error;
use struct_iterable::Iterable;
use crate::{
parser::price::{Binance, Kibo, Kraken},
structs::{
Amount, BiMap, Config, Date, DateMap, DateMapChunkId, Height, HeightMapChunkId, MapKey, MapKind, Timestamp,
OHLC,
},
utils::{ONE_MONTH_IN_DAYS, ONE_WEEK_IN_DAYS, ONE_YEAR_IN_DAYS},
};
use super::{AnyDataset, ComputeData, MinInitialStates, RatioDataset};
#[derive(Allocative, Iterable)]
pub struct PriceDatasets {
min_initial_states: MinInitialStates,
kraken_daily: Option<BTreeMap<Date, OHLC>>,
kraken_1mn: Option<BTreeMap<u32, OHLC>>,
binance_1mn: Option<BTreeMap<u32, OHLC>>,
binance_daily: Option<BTreeMap<Date, OHLC>>,
binance_har: Option<BTreeMap<u32, OHLC>>,
kibo_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
kibo_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
pub ohlc: BiMap<OHLC>,
pub open: BiMap<f32>,
pub high: BiMap<f32>,
pub low: BiMap<f32>,
pub close: BiMap<f32>,
}
impl PriceDatasets {
pub fn import(config: &Config) -> color_eyre::Result<Self> {
let path_dataset = config.path_datasets();
let f = |s: &str| path_dataset.join(s);
let mut s = Self {
min_initial_states: MinInitialStates::default(),
binance_1mn: None,
binance_daily: None,
binance_har: None,
kraken_1mn: None,
kraken_daily: None,
kibo_by_height: BTreeMap::default(),
kibo_by_date: BTreeMap::default(),
// ---
// Inserted
// ---
ohlc: BiMap::new_json(1, MapKind::Inserted, &config.path_price()),
// ---
// Computed
// ---
open_cents: BiMap::new_bin(1, MapKind::Computed, &f("open")),
high_cents: BiMap::new_bin(1, MapKind::Computed, &f("high")),
low_cents: BiMap::new_bin(1, MapKind::Computed, &f("low")),
close: BiMap::new_bin(1, MapKind::Computed, &f("close")),
};
s.min_initial_states
.consume(MinInitialStates::compute_from_dataset(&s, config));
Ok(s)
}
pub fn compute(&mut self, compute_data: &ComputeData, circulating_supply: &mut BiMap<f64>) {
let &ComputeData { dates, heights, .. } = compute_data;
self.open
.multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open);
self.high
.multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.high);
self.low
.multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.low);
self.close
.multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.close);
}
pub fn get_date_ohlc(&mut self, date: Date) -> color_eyre::Result<OHLC> {
if self.ohlc.date.is_key_safe(date) {
Ok(self.ohlc.date.get_or_import(&date).unwrap().to_owned())
} else {
let ohlc = self
.get_from_daily_kraken(&date)
.or_else(|_| self.get_from_daily_binance(&date))
.or_else(|_| self.get_from_date_kibo(&date))?;
self.ohlc.date.insert(date, ohlc);
Ok(ohlc)
}
}
fn get_from_date_kibo(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
let chunk_id = date.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.kibo_by_date.contains_key(&chunk_id)
|| self.kibo_by_date.get(&chunk_id).unwrap().last_key_value().unwrap().0 < date
{
self.kibo_by_date.insert(chunk_id, Kibo::fetch_date_prices(chunk_id)?);
}
self.kibo_by_date
.get(&chunk_id)
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date in satonomics"))
}
fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.kraken_daily.is_none() || self.kraken_daily.as_ref().unwrap().last_key_value().unwrap().0 < date {
self.kraken_daily.replace(Kraken::fetch_daily_prices()?);
}
self.kraken_daily
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date"))
}
fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.binance_daily.is_none() || self.binance_daily.as_ref().unwrap().last_key_value().unwrap().0 < date {
self.binance_daily.replace(Binance::fetch_daily_prices()?);
}
self.binance_daily
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date"))
}
pub fn get_height_ohlc(
&mut self,
height: Height,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
config: &Config,
) -> color_eyre::Result<OHLC> {
if let Some(ohlc) = self.ohlc.height.get_or_import(&height) {
return Ok(ohlc);
}
let timestamp = timestamp.to_floored_seconds();
if previous_timestamp.is_none() && !height.is_first() {
panic!("Shouldn't be possible");
}
let previous_timestamp = previous_timestamp.map(|t| t.to_floored_seconds());
let ohlc = self
.get_from_1mn_kraken(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_1mn_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_har_binance(timestamp, previous_timestamp, config)
.unwrap_or_else(|_| {
self.get_from_height_kibo(&height).unwrap_or_else(|_| {
let date = timestamp.to_date();
panic!(
"Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot
2. Select 1mn interval
3. Open the inspector/dev tools
4. Go to the Network Tab
5. Filter URLs by 'uiKlines'
6. Go back to the chart and scroll until you pass the date mentioned few lines ago
7. Go back to the dev tools
8. Export to a har file (if there is no explicit button, click on the cog button)
9. Move the file to 'parser/imports/binance.har'
"
)
})
})
})
});
self.ohlc.height.insert(height, ohlc);
Ok(ohlc)
}
fn get_from_height_kibo(&mut self, height: &Height) -> color_eyre::Result<OHLC> {
let chunk_id = height.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.kibo_by_height.contains_key(&chunk_id)
|| ((chunk_id.to_usize() + self.kibo_by_height.get(&chunk_id).unwrap().len()) <= height.to_usize())
{
self.kibo_by_height
.insert(chunk_id, Kibo::fetch_height_prices(chunk_id)?);
}
self.kibo_by_height
.get(&chunk_id)
.unwrap()
.get(height.to_serialized_key().to_usize())
.cloned()
.ok_or(Error::msg("Couldn't find height in kibo"))
}
fn get_from_1mn_kraken(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.kraken_1mn.is_none() || self.kraken_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?);
}
Self::find_height_ohlc(&self.kraken_1mn, timestamp, previous_timestamp, "kraken 1m")
}
fn get_from_1mn_binance(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLC> {
if self.binance_1mn.is_none() || self.binance_1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp {
self.binance_1mn.replace(Binance::fetch_1mn_prices()?);
}
Self::find_height_ohlc(&self.binance_1mn, timestamp, previous_timestamp, "binance 1m")
}
fn get_from_har_binance(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
config: &Config,
) -> color_eyre::Result<OHLC> {
if self.binance_har.is_none() {
self.binance_har
.replace(Binance::read_har_file(config).unwrap_or_default());
}
Self::find_height_ohlc(&self.binance_har, timestamp, previous_timestamp, "binance har")
}
fn find_height_ohlc(
tree: &Option<BTreeMap<u32, OHLC>>,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
name: &str,
) -> color_eyre::Result<OHLC> {
let tree = tree.as_ref().unwrap();
let err = Error::msg(format!("Couldn't find timestamp in {name}"));
let previous_ohlc = previous_timestamp.map_or(Some(OHLC::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
});
let last_ohlc = tree.get(&timestamp);
if previous_ohlc.is_none() || last_ohlc.is_none() {
return Err(err);
}
let previous_ohlc = previous_ohlc.unwrap();
let mut final_ohlc = OHLC {
open: previous_ohlc.close,
high: previous_ohlc.close,
low: previous_ohlc.close,
close: previous_ohlc.close,
};
let start = previous_timestamp.unwrap_or_default();
let end = timestamp;
// Otherwise it's a re-org
if start < end {
tree.range(&*start..=&*end).skip(1).for_each(|(_, ohlc)| {
if ohlc.high > final_ohlc.high {
final_ohlc.high = ohlc.high
}
if ohlc.low < final_ohlc.low {
final_ohlc.low = ohlc.low
}
final_ohlc.close = ohlc.close;
});
}
Ok(final_ohlc)
}
}
impl AnyDataset for PriceDatasets {
fn get_min_initial_states(&self) -> &MinInitialStates {
&self.min_initial_states
}
}
pricer/src/structs/cents.rs
+16 -1
View File
@@ -4,7 +4,22 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Dollars;
#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Deref,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Serialize,
)]
pub struct Cents(u64);
impl From<Dollars> for Cents {
pricer/src/structs/close.rs
+16 -1
View File
@@ -2,7 +2,22 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Deref,
Serialize,
)]
#[repr(C)]
pub struct Close<T>(T);
impl<T> From<T> for Close<T> {
pricer/src/structs/date.rs
+15 -37
View File
@@ -1,18 +1,17 @@
use std::ops::Add;
use color_eyre::eyre::eyre;
use indexer::Timestamp;
use jiff::{civil::Date as Date_, tz::TimeZone, Span};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Dateindex;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Date(u32);
impl Date {
const INDEX_ZERO: Self = Self(20090103);
const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3);
const INDEX_ONE: Self = Self(20090109);
const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9);
pub const INDEX_ZERO: Self = Self(20090103);
pub const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3);
pub const INDEX_ONE: Self = Self(20090109);
pub const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9);
pub fn year(&self) -> u16 {
(self.0 / 1_00_00) as u16
@@ -51,33 +50,12 @@ impl From<Timestamp> for Date {
}
}
// impl TryFrom<Date> for usize {
// type Error = color_eyre::Report;
// fn try_from(value: Date) -> Result<Self, Self::Error> {
// let value_ = Date_::from(value);
// if value_ < Date::INDEX_ZERO_ {
// Err(eyre!("Date is too early"))
// } else if value == Date::INDEX_ZERO {
// Ok(0)
// } else if value_ < Date::INDEX_ONE_ {
// Err(eyre!("Date is between first and second"))
// } else if value == Date::INDEX_ONE {
// Ok(1)
// } else {
// Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1)
// }
// }
// }
// impl From<usize> for Date {
// fn from(value: usize) -> Self {
// Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap())
// }
// }
// impl Add<usize> for Date {
// type Output = Self;
// fn add(self, rhs: usize) -> Self::Output {
// Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap())
// }
// }
impl From<Dateindex> for Date {
fn from(value: Dateindex) -> Self {
Self::from(
Self::INDEX_ZERO_
.checked_add(Span::new().days(i64::from(value)))
.unwrap(),
)
}
}
pricer/src/structs/dateindex.rs
+27
View File
@@ -1,7 +1,10 @@
use std::ops::Add;
use color_eyre::eyre::eyre;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Date;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Dateindex(u16);
@@ -17,9 +20,33 @@ impl From<usize> for Dateindex {
}
}
impl From<Dateindex> for i64 {
fn from(value: Dateindex) -> Self {
value.0 as i64
}
}
impl Add<usize> for Dateindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u16)
}
}
impl TryFrom<Date> for Dateindex {
type Error = color_eyre::Report;
fn try_from(value: Date) -> Result<Self, Self::Error> {
let value_ = jiff::civil::Date::from(value);
if value_ < Date::INDEX_ZERO_ {
Err(eyre!("Date is too early"))
} else if value == Date::INDEX_ZERO {
Ok(Self(0))
} else if value_ < Date::INDEX_ONE_ {
Err(eyre!("Date is between first and second"))
} else if value == Date::INDEX_ONE {
Ok(Self(1))
} else {
Ok(Self(Date::INDEX_ONE_.until(value_)?.get_days() as u16 + 1))
}
}
}
pricer/src/structs/dollars.rs
+3 -1
View File
@@ -1,8 +1,10 @@
use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Cents;
#[derive(Debug, Default, Clone, Copy, Deref)]
#[derive(Debug, Default, Clone, Copy, Deref, FromBytes, Immutable, IntoBytes, KnownLayout, Serialize)]
pub struct Dollars(f64);
impl From<f64> for Dollars {
pricer/src/structs/high.rs
+27 -1
View File
@@ -2,7 +2,24 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
use super::Close;
#[derive(
Debug,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Deref,
Serialize,
)]
#[repr(C)]
pub struct High<T>(T);
impl<T> From<T> for High<T> {
@@ -10,3 +27,12 @@ impl<T> From<T> for High<T> {
Self(value)
}
}
impl<T> From<Close<T>> for High<T>
where
T: Copy,
{
fn from(value: Close<T>) -> Self {
Self(*value)
}
}
pricer/src/structs/low.rs
+27 -1
View File
@@ -2,7 +2,24 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
use super::Close;
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Deref,
Serialize,
)]
#[repr(C)]
pub struct Low<T>(T);
impl<T> From<T> for Low<T> {
@@ -10,3 +27,12 @@ impl<T> From<T> for Low<T> {
Self(value)
}
}
impl<T> From<Close<T>> for Low<T>
where
T: Copy,
{
fn from(value: Close<T>) -> Self {
Self(*value)
}
}
pricer/src/structs/open.rs
+27 -1
View File
@@ -2,7 +2,24 @@ use derive_deref::Deref;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout, Deref, Serialize)]
use super::Close;
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
Deref,
Serialize,
)]
#[repr(C)]
pub struct Open<T>(T);
impl<T> From<T> for Open<T> {
@@ -10,3 +27,12 @@ impl<T> From<T> for Open<T> {
Self(value)
}
}
impl<T> From<Close<T>> for Open<T>
where
T: Copy,
{
fn from(value: Close<T>) -> Self {
Self(*value)
}
}
server/src/api/vecs/index.rs
+20
View File
@@ -17,6 +17,26 @@ pub enum Index {
Txoutindex,
}
impl Index {
pub fn all() -> [Self; 13] {
[
Self::Addressindex,
Self::Dateindex,
Self::Height,
Self::P2PK33index,
Self::P2PK65index,
Self::P2PKHindex,
Self::P2SHindex,
Self::P2TRindex,
Self::P2WPKHindex,
Self::P2WSHindex,
Self::Txindex,
Self::Txinindex,
Self::Txoutindex,
]
}
}
impl TryFrom<&str> for Index {
type Error = ();
fn try_from(value: &str) -> Result<Self, Self::Error> {
server/src/api/vecs/tree.rs
+41 -1
View File
@@ -1,8 +1,10 @@
use std::collections::BTreeMap;
use std::{collections::BTreeMap, fs, io};
use derive_deref::{Deref, DerefMut};
use storable_vec::AnyJsonStorableVec;
use crate::WEBSITE_DEV_PATH;
use super::index::Index;
#[derive(Default, Deref, DerefMut)]
@@ -32,6 +34,44 @@ impl VecIdToIndexToVec {
panic!()
}
}
pub fn generate_dts_file(&self) -> io::Result<()> {
if !fs::exists(WEBSITE_DEV_PATH)? {
return Ok(());
}
let path = format!("{WEBSITE_DEV_PATH}/scripts/types/vecid-to-indexes.d.ts");
let mut contents = Index::all()
.into_iter()
.enumerate()
.map(|(i_of_i, i)| format!("type {} = {};", i, i_of_i))
.collect::<Vec<_>>()
.join("\n");
contents += "\n\ninterface VecIdToIndexes {\n";
self.iter().for_each(|(id, index_to_vec)| {
let indexes = index_to_vec
.keys()
.map(|i| i.to_string())
.collect::<Vec<_>>()
.join(", ");
contents += &format!(
" {}: [{indexes}]\n",
if id.contains("-") {
format!("\"{id}\"")
} else {
id.to_owned()
}
);
});
contents.push('}');
fs::write(path, contents)
}
}
#[derive(Default, Deref, DerefMut)]
server/src/files/file.rs
+1 -2
View File
@@ -16,12 +16,11 @@ use reqwest::StatusCode;
use crate::{
log_result,
traits::{HeaderMapExtended, ModifiedState, ResponseExtended},
WEBSITE_DEV_PATH,
};
use super::minify::minify_js;
const WEBSITE_DEV_PATH: &str = "../website/";
pub async fn file_handler(headers: HeaderMap, path: extract::Path<String>) -> Response {
any_handler(headers, Some(path))
}
server/src/lib.rs
+4 -3
View File
@@ -23,10 +23,9 @@ pub struct AppState {
computer: &'static Computer<STATELESS>,
}
pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) -> color_eyre::Result<()> {
// pub async fn main(routes: Routes, config: Config) -> color_eyre::Result<()> {
// routes.generate_dts_file();
pub const WEBSITE_DEV_PATH: &str = "../website/";
pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) -> color_eyre::Result<()> {
let indexer = Box::leak(Box::new(indexer));
let computer = Box::leak(Box::new(computer));
let vecs = Box::leak(Box::new(VecIdToIndexToVec::default()));
@@ -37,6 +36,8 @@ pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) ->
.into_iter()
.for_each(|vec| vecs.insert(vec));
vecs.generate_dts_file()?;
let state = AppState {
vecs,
indexer,