Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: one big snapshot

Browse Source
This commit is contained in:
nym21
2025-08-02 16:59:22 +02:00
parent aa8b47a3dd
commit f7aa9424db
252 changed files with 6283 additions and 5264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/brk_fetcher/Cargo.toml
View File

@@ -8,9 +8,9 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_error = { workspace = true }
brk_logger = { workspace = true }
color-eyre = { workspace = true }
brk_structs = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
serde_json = { workspace = true }

6
crates/brk_fetcher/README.md
View File

@@ -23,12 +23,6 @@
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
<a href="https://x.com/brkdotorg">
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
</a>
</p>
A crate that can fetch the Bitcoin price, either by date or height, from Binance, Kraken and the main instance of BRK.

7
crates/brk_fetcher/examples/main.rs
View File

@@ -1,9 +1,8 @@
use brk_core::{Date, Height};
use brk_error::Result;
use brk_fetcher::{BRK, Binance, Fetcher, Kraken};
use brk_structs::{Date, Height};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
fn main() -> Result<()> {
brk_logger::init(None);
let mut brk = BRK::default();

46
crates/brk_fetcher/src/binance.rs
View File

@@ -6,8 +6,8 @@ use std::{
str::FromStr,
};
use brk_core::{Cents, OHLCCents, Timestamp};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_error::{Error, Result};
use brk_structs::{Cents, OHLCCents, Timestamp};
use log::info;
use serde_json::Value;
@@ -35,7 +35,7 @@ impl Binance {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
@@ -65,7 +65,7 @@ impl Binance {
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Binance...");
retry(
@@ -81,7 +81,7 @@ impl Binance {
)
}
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Self::fetch_1d()?);
}
@@ -91,10 +91,10 @@ impl Binance {
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
.ok_or(Error::Str("Couldn't find date"))
}
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Binance...");
retry(
@@ -104,9 +104,9 @@ impl Binance {
)
}
fn read_har(&self) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn read_har(&self) -> Result<BTreeMap<Timestamp, OHLCCents>> {
if self.path.is_none() {
return Err(eyre!("Path missing"));
return Err(Error::Str("Path missing"));
}
info!("Reading Binance har file...");
@@ -120,7 +120,7 @@ impl Binance {
let file = if let Ok(file) = File::open(path_binance_har) {
file
} else {
return Err(eyre!("Missing binance file"));
return Err(Error::Str("Missing binance file"));
};
let reader = BufReader::new(file);
@@ -132,13 +132,13 @@ impl Binance {
};
json.get("log")
.context("Expect object to have log attribute")?
.ok_or(Error::Str("Expect object to have log attribute"))?
.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("entries")
.context("Expect object to have entries")?
.ok_or(Error::Str("Expect object to have entries"))?
.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.filter(|entry| {
entry
@@ -181,30 +181,28 @@ impl Binance {
})
}
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
fn json_to_date_to_ohlc(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
F: Fn(&Value) -> Result<(K, V)>,
K: Ord,
{
json.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> {
let array = array.as_array().context("Expect to be array")?;
fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> {
let array = array.as_array().ok_or(Error::Str("Expect to be array"))?;
let timestamp = Timestamp::from((array.first().unwrap().as_u64().unwrap() / 1_000) as u32);
@@ -231,7 +229,7 @@ impl Binance {
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> {
fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}

32
crates/brk_fetcher/src/brk.rs
View File

@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use color_eyre::eyre::{ContextCompat, eyre};
use brk_error::{Error, Result};
use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use log::info;
use serde_json::Value;
@@ -19,7 +19,7 @@ const RETRIES: usize = 10;
const CHUNK_SIZE: usize = 10_000;
impl BRK {
pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
pub fn get_from_height(&mut self, height: Height) -> Result<OHLCCents> {
let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
#[allow(clippy::map_entry)]
@@ -39,10 +39,10 @@ impl BRK {
.unwrap()
.get(usize::from(height.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find height in BRK"))
.ok_or(Error::Str("Couldn't find height in BRK"))
}
fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
fn fetch_height_prices(height: Height) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK height {height} prices...");
retry(
@@ -56,7 +56,7 @@ impl BRK {
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
@@ -66,7 +66,7 @@ impl BRK {
)
}
pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_date(&mut self, date: Date) -> Result<OHLCCents> {
let dateindex = DateIndex::try_from(date)?;
let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
@@ -88,10 +88,10 @@ impl BRK {
.unwrap()
.get(usize::from(dateindex.checked_sub(key).unwrap()))
.cloned()
.ok_or(eyre!("Couldn't find date in BRK"))
.ok_or(Error::Str("Couldn't find date in BRK"))
}
fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result<Vec<OHLCCents>> {
fn fetch_date_prices(dateindex: DateIndex) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK dateindex {dateindex} prices...");
retry(
@@ -105,7 +105,7 @@ impl BRK {
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
@@ -115,15 +115,17 @@ impl BRK {
)
}
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
let ohlc = value.as_array().context("Expect as_array to work")?;
fn value_to_ohlc(value: &Value) -> Result<OHLCCents> {
let ohlc = value
.as_array()
.ok_or(Error::Str("Expect as_array to work"))?;
let get_value = |index: usize| -> color_eyre::Result<_> {
let get_value = |index: usize| -> Result<_> {
Ok(Cents::from(Dollars::from(
ohlc.get(index)
.context("Expect index key to work")?
.ok_or(Error::Str("Expect index key to work"))?
.as_f64()
.context("Expect as_f64 to work")?,
.ok_or(Error::Str("Expect as_f64 to work"))?,
)))
};

40
crates/brk_fetcher/src/kraken.rs
View File

@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use brk_core::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::ContextCompat;
use brk_error::{Error, Result};
use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use log::info;
use serde_json::Value;
@@ -18,7 +18,7 @@ impl Kraken {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
@@ -32,7 +32,7 @@ impl Kraken {
)
}
pub fn fetch_1mn() -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Kraken...");
retry(
@@ -42,7 +42,7 @@ impl Kraken {
)
}
pub fn get_from_1d(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Kraken::fetch_1d()?);
}
@@ -51,10 +51,10 @@ impl Kraken {
.unwrap()
.get(date)
.cloned()
.ok_or(color_eyre::eyre::Error::msg("Couldn't find date"))
.ok_or(Error::Str("Couldn't find date"))
}
pub fn fetch_1d() -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
retry(
@@ -64,38 +64,36 @@ impl Kraken {
)
}
fn json_to_timestamp_to_ohlc(
json: &Value,
) -> color_eyre::Result<BTreeMap<Timestamp, OHLCCents>> {
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_timestamp_and_ohlc)
}
fn json_to_date_to_ohlc(json: &Value) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
fn json_to_date_to_ohlc(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
Self::json_to_btree(json, Self::array_to_date_and_ohlc)
}
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> color_eyre::Result<BTreeMap<K, V>>
fn json_to_btree<F, K, V>(json: &Value, fun: F) -> Result<BTreeMap<K, V>>
where
F: Fn(&Value) -> color_eyre::Result<(K, V)>,
F: Fn(&Value) -> Result<(K, V)>,
K: Ord,
{
json.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("result")
.context("Expect object to have result")?
.ok_or(Error::Str("Expect object to have result"))?
.as_object()
.context("Expect to be an object")?
.ok_or(Error::Str("Expect to be an object"))?
.get("XXBTZUSD")
.context("Expect to have XXBTZUSD")?
.ok_or(Error::Str("Expect to have XXBTZUSD"))?
.as_array()
.context("Expect to be an array")?
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(fun)
.collect::<Result<BTreeMap<_, _>, _>>()
}
fn array_to_timestamp_and_ohlc(array: &Value) -> color_eyre::Result<(Timestamp, OHLCCents)> {
let array = array.as_array().context("Expect to be array")?;
fn array_to_timestamp_and_ohlc(array: &Value) -> Result<(Timestamp, OHLCCents)> {
let array = array.as_array().ok_or(Error::Str("Expect to be array"))?;
let timestamp = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32);
@@ -122,7 +120,7 @@ impl Kraken {
))
}
fn array_to_date_and_ohlc(array: &Value) -> color_eyre::Result<(Date, OHLCCents)> {
fn array_to_date_and_ohlc(array: &Value) -> Result<(Date, OHLCCents)> {
Self::array_to_timestamp_and_ohlc(array).map(|(t, ohlc)| (Date::from(t), ohlc))
}

18
crates/brk_fetcher/src/lib.rs
View File

@@ -5,8 +5,8 @@
use std::{collections::BTreeMap, path::Path, thread::sleep, time::Duration};
use brk_core::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use color_eyre::eyre::Error;
use brk_error::{Error, Result};
use brk_structs::{Close, Date, Dollars, Height, High, Low, OHLCCents, Open, Timestamp};
use log::info;
mod binance;
@@ -29,7 +29,7 @@ pub struct Fetcher {
}
impl Fetcher {
pub fn import(hars_path: Option<&Path>) -> color_eyre::Result<Self> {
pub fn import(hars_path: Option<&Path>) -> Result<Self> {
Ok(Self {
binance: Binance::init(hars_path),
kraken: Kraken::default(),
@@ -37,11 +37,11 @@ impl Fetcher {
})
}
pub fn get_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
pub fn get_date(&mut self, date: Date) -> Result<OHLCCents> {
self.get_date_(date, 0)
}
fn get_date_(&mut self, date: Date, tries: usize) -> color_eyre::Result<OHLCCents> {
fn get_date_(&mut self, date: Date, tries: usize) -> Result<OHLCCents> {
self.kraken
.get_from_1d(&date)
.or_else(|_| {
@@ -72,7 +72,7 @@ impl Fetcher {
height: Height,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
self.get_height_(height, timestamp, previous_timestamp, 0)
}
@@ -82,7 +82,7 @@ impl Fetcher {
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
tries: usize,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
let timestamp = timestamp.floor_seconds();
if previous_timestamp.is_none() && height != Height::ZERO {
@@ -149,7 +149,7 @@ How to fix this:
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
name: &str,
) -> color_eyre::Result<OHLCCents> {
) -> Result<OHLCCents> {
let previous_ohlc = previous_timestamp
.map_or(Some(OHLCCents::default()), |previous_timestamp| {
tree.get(&previous_timestamp).cloned()
@@ -158,7 +158,7 @@ How to fix this:
let last_ohlc = tree.get(&timestamp);
if previous_ohlc.is_none() || last_ohlc.is_none() {
return Err(Error::msg(format!("Couldn't find timestamp in {name}")));
return Err(Error::String(format!("Couldn't find timestamp in {name}")));
}
let previous_ohlc = previous_ohlc.unwrap();

7
crates/brk_fetcher/src/retry.rs
View File

@@ -1,12 +1,9 @@
use std::{fmt::Debug, thread::sleep, time::Duration};
use brk_error::Result;
use log::info;
pub fn retry<T>(
function: impl Fn(usize) -> color_eyre::Result<T>,
sleep_in_s: u64,
retries: usize,
) -> color_eyre::Result<T>
pub fn retry<T>(function: impl Fn(usize) -> Result<T>, sleep_in_s: u64, retries: usize) -> Result<T>
where
T: Debug,
{