Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2026-01-31 17:39:48 +01:00
parent 8dd350264a
commit ff5bb770d7
116 changed files with 13312 additions and 9530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
crates/brk_fetcher/src/binance.rs
View File

@@ -6,7 +6,7 @@ use std::{
};
use brk_error::{Error, Result};
use brk_types::{Date, Height, OHLCCents, Timestamp};
use brk_types::{Date, Height, OHLCCentsUnsigned, Timestamp};
use serde_json::Value;
use tracing::info;
@@ -18,9 +18,9 @@ use crate::{
#[derive(Clone)]
pub struct Binance {
path: Option<PathBuf>,
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
har: Option<BTreeMap<Timestamp, OHLCCents>>,
_1mn: Option<BTreeMap<Timestamp, OHLCCentsUnsigned>>,
_1d: Option<BTreeMap<Date, OHLCCentsUnsigned>>,
har: Option<BTreeMap<Timestamp, OHLCCentsUnsigned>>,
}
impl Binance {
@@ -37,7 +37,7 @@ impl Binance {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Result<OHLCCents> {
) -> Result<OHLCCentsUnsigned> {
// Try live API data first
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
@@ -69,7 +69,7 @@ impl Binance {
)
}
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCentsUnsigned>> {
default_retry(|_| {
let url = Self::url("interval=1m&limit=1000");
info!("Fetching {url} ...");
@@ -79,7 +79,7 @@ impl Binance {
})
}
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCentsUnsigned> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Self::fetch_1d()?);
}
@@ -92,7 +92,7 @@ impl Binance {
.ok_or(Error::NotFound("Couldn't find date".into()))
}
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCentsUnsigned>> {
default_retry(|_| {
let url = Self::url("interval=1d");
info!("Fetching {url} ...");
@@ -102,7 +102,7 @@ impl Binance {
})
}
fn read_har(&self) -> Result<BTreeMap<Timestamp, OHLCCents>> {
fn read_har(&self) -> Result<BTreeMap<Timestamp, OHLCCentsUnsigned>> {
if self.path.is_none() {
return Err(Error::NotFound("HAR path not configured".into()));
}
@@ -179,7 +179,7 @@ impl Binance {
})
}
fn parse_ohlc_array(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
fn parse_ohlc_array(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCentsUnsigned>> {
let result = json
.as_array()
.ok_or(Error::Parse("Expected JSON array".into()))?
@@ -193,7 +193,7 @@ impl Binance {
Ok(result)
}
fn parse_date_ohlc_array(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
fn parse_date_ohlc_array(json: &Value) -> Result<BTreeMap<Date, OHLCCentsUnsigned>> {
Self::parse_ohlc_array(json).map(|map| {
map.into_iter()
.map(|(ts, ohlc)| (date_from_timestamp(ts), ohlc))
@@ -218,7 +218,7 @@ impl PriceSource for Binance {
"Binance"
}
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCents>> {
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_1d(&date))
}
@@ -226,11 +226,11 @@ impl PriceSource for Binance {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Option<Result<OHLCCents>> {
) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_1mn(timestamp, previous_timestamp))
}
fn get_height(&mut self, _height: Height) -> Option<Result<OHLCCents>> {
fn get_height(&mut self, _height: Height) -> Option<Result<OHLCCentsUnsigned>> {
None // Binance doesn't support height-based queries
}

28
crates/brk_fetcher/src/brk.rs
View File

@@ -2,8 +2,8 @@ use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_types::{
Cents, CheckedSub, Close, Date, DateIndex, Dollars, Height, High, Low, OHLCCents, Open,
Timestamp,
CentsUnsigned, CheckedSub, Close, Date, DateIndex, Dollars, Height, High, Low,
OHLCCentsUnsigned, Open, Timestamp,
};
use serde_json::Value;
use tracing::info;
@@ -13,15 +13,15 @@ use crate::{PriceSource, check_response, default_retry};
#[derive(Default, Clone)]
#[allow(clippy::upper_case_acronyms)]
pub struct BRK {
height_to_ohlc: BTreeMap<Height, Vec<OHLCCents>>,
dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
height_to_ohlc: BTreeMap<Height, Vec<OHLCCentsUnsigned>>,
dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCentsUnsigned>>,
}
const API_URL: &str = "https://bitview.space/api/vecs";
const CHUNK_SIZE: usize = 10_000;
impl BRK {
pub fn get_from_height(&mut self, height: Height) -> Result<OHLCCents> {
pub fn get_from_height(&mut self, height: Height) -> Result<OHLCCentsUnsigned> {
let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
#[allow(clippy::map_entry)]
@@ -40,7 +40,7 @@ impl BRK {
.ok_or(Error::NotFound("Couldn't find height in BRK".into()))
}
fn fetch_height_prices(height: Height) -> Result<Vec<OHLCCents>> {
fn fetch_height_prices(height: Height) -> Result<Vec<OHLCCentsUnsigned>> {
default_retry(|_| {
let url = format!(
"{API_URL}/height-to-price-ohlc?from={}&to={}",
@@ -60,7 +60,7 @@ impl BRK {
})
}
pub fn get_from_date(&mut self, date: Date) -> Result<OHLCCents> {
pub fn get_from_date(&mut self, date: Date) -> Result<OHLCCentsUnsigned> {
let dateindex = DateIndex::try_from(date)?;
let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
@@ -81,7 +81,7 @@ impl BRK {
.ok_or(Error::NotFound("Couldn't find date in BRK".into()))
}
fn fetch_date_prices(dateindex: DateIndex) -> Result<Vec<OHLCCents>> {
fn fetch_date_prices(dateindex: DateIndex) -> Result<Vec<OHLCCentsUnsigned>> {
default_retry(|_| {
let url = format!(
"{API_URL}/dateindex-to-price-ohlc?from={}&to={}",
@@ -101,13 +101,13 @@ impl BRK {
})
}
fn value_to_ohlc(value: &Value) -> Result<OHLCCents> {
fn value_to_ohlc(value: &Value) -> Result<OHLCCentsUnsigned> {
let ohlc = value
.as_array()
.ok_or(Error::Parse("Expected OHLC array".into()))?;
let get_value = |index: usize| -> Result<_> {
Ok(Cents::from(Dollars::from(
Ok(CentsUnsigned::from(Dollars::from(
ohlc.get(index)
.ok_or(Error::Parse("Missing OHLC value at index".into()))?
.as_f64()
@@ -115,7 +115,7 @@ impl BRK {
)))
};
Ok(OHLCCents::from((
Ok(OHLCCentsUnsigned::from((
Open::new(get_value(0)?),
High::new(get_value(1)?),
Low::new(get_value(2)?),
@@ -134,7 +134,7 @@ impl PriceSource for BRK {
"BRK"
}
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCents>> {
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_date(date))
}
@@ -142,11 +142,11 @@ impl PriceSource for BRK {
&mut self,
_timestamp: Timestamp,
_previous_timestamp: Option<Timestamp>,
) -> Option<Result<OHLCCents>> {
) -> Option<Result<OHLCCentsUnsigned>> {
None // BRK doesn't support timestamp-based queries
}
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCents>> {
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_height(height))
}

24
crates/brk_fetcher/src/kraken.rs
View File

@@ -1,7 +1,7 @@
use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_types::{Date, Height, OHLCCents, Timestamp};
use brk_types::{Date, Height, OHLCCentsUnsigned, Timestamp};
use serde_json::Value;
use tracing::info;
@@ -12,8 +12,8 @@ use crate::{
#[derive(Default, Clone)]
pub struct Kraken {
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
_1mn: Option<BTreeMap<Timestamp, OHLCCentsUnsigned>>,
_1d: Option<BTreeMap<Date, OHLCCentsUnsigned>>,
}
impl Kraken {
@@ -21,7 +21,7 @@ impl Kraken {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Result<OHLCCents> {
) -> Result<OHLCCentsUnsigned> {
if self._1mn.is_none()
|| self._1mn.as_ref().unwrap().last_key_value().unwrap().0 <= &timestamp
{
@@ -35,7 +35,7 @@ impl Kraken {
)
}
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCentsUnsigned>> {
default_retry(|_| {
let url = Self::url(1);
info!("Fetching {url} ...");
@@ -45,7 +45,7 @@ impl Kraken {
})
}
fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCentsUnsigned> {
if self._1d.is_none() || self._1d.as_ref().unwrap().last_key_value().unwrap().0 <= date {
self._1d.replace(Self::fetch_1d()?);
}
@@ -57,7 +57,7 @@ impl Kraken {
.ok_or(Error::NotFound("Couldn't find date".into()))
}
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCentsUnsigned>> {
default_retry(|_| {
let url = Self::url(1440);
info!("Fetching {url} ...");
@@ -68,7 +68,7 @@ impl Kraken {
}
/// Parse Kraken's nested JSON response: { result: { XXBTZUSD: [...] } }
fn parse_ohlc_response(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
fn parse_ohlc_response(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCentsUnsigned>> {
let result = json
.get("result")
.and_then(|r| r.get("XXBTZUSD"))
@@ -84,7 +84,7 @@ impl Kraken {
Ok(result)
}
fn parse_date_ohlc_response(json: &Value) -> Result<BTreeMap<Date, OHLCCents>> {
fn parse_date_ohlc_response(json: &Value) -> Result<BTreeMap<Date, OHLCCentsUnsigned>> {
Self::parse_ohlc_response(json).map(|map| {
map.into_iter()
.map(|(ts, ohlc)| (date_from_timestamp(ts), ohlc))
@@ -109,7 +109,7 @@ impl PriceSource for Kraken {
"Kraken"
}
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCents>> {
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_1d(&date))
}
@@ -117,11 +117,11 @@ impl PriceSource for Kraken {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Option<Result<OHLCCents>> {
) -> Option<Result<OHLCCentsUnsigned>> {
Some(self.get_from_1mn(timestamp, previous_timestamp))
}
fn get_height(&mut self, _height: Height) -> Option<Result<OHLCCents>> {
fn get_height(&mut self, _height: Height) -> Option<Result<OHLCCentsUnsigned>> {
None // Kraken doesn't support height-based queries
}

14
crates/brk_fetcher/src/lib.rs
View File

@@ -3,7 +3,7 @@
use std::{path::Path, thread::sleep, time::Duration};
use brk_error::{Error, Result};
use brk_types::{Date, Height, OHLCCents, Timestamp};
use brk_types::{Date, Height, OHLCCentsUnsigned, Timestamp};
use tracing::info;
mod binance;
@@ -66,9 +66,9 @@ impl Fetcher {
}
/// Try fetching from each source in order, return first success
fn try_sources<F>(&mut self, mut fetch: F) -> Option<Result<OHLCCents>>
fn try_sources<F>(&mut self, mut fetch: F) -> Option<Result<OHLCCentsUnsigned>>
where
F: FnMut(&mut dyn PriceSource) -> Option<Result<OHLCCents>>,
F: FnMut(&mut dyn PriceSource) -> Option<Result<OHLCCentsUnsigned>>,
{
if let Some(Ok(ohlc)) = fetch(&mut self.binance) {
return Some(Ok(ohlc));
@@ -82,7 +82,7 @@ impl Fetcher {
None
}
pub fn get_date(&mut self, date: Date) -> Result<OHLCCents> {
pub fn get_date(&mut self, date: Date) -> Result<OHLCCentsUnsigned> {
self.fetch_with_retry(
|source| source.get_date(date),
|| format!("Failed to fetch price for date {date}"),
@@ -94,7 +94,7 @@ impl Fetcher {
height: Height,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Result<OHLCCents> {
) -> Result<OHLCCentsUnsigned> {
let timestamp = timestamp.floor_seconds();
let previous_timestamp = previous_timestamp.map(|t| t.floor_seconds());
@@ -133,9 +133,9 @@ How to fix this:
}
/// Try each source in order, with retries on total failure
fn fetch_with_retry<F, E>(&mut self, mut fetch: F, error_message: E) -> Result<OHLCCents>
fn fetch_with_retry<F, E>(&mut self, mut fetch: F, error_message: E) -> Result<OHLCCentsUnsigned>
where
F: FnMut(&mut dyn PriceSource) -> Option<Result<OHLCCents>>,
F: FnMut(&mut dyn PriceSource) -> Option<Result<OHLCCentsUnsigned>>,
E: Fn() -> String,
{
for retry in 0..=MAX_RETRIES {

20
crates/brk_fetcher/src/ohlc.rs
View File

@@ -1,11 +1,11 @@
use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_types::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use brk_types::{CentsUnsigned, Close, Date, Dollars, High, Low, OHLCCentsUnsigned, Open, Timestamp};
/// Parse OHLC value from a JSON array element at given index
pub fn parse_cents(array: &[serde_json::Value], index: usize) -> Cents {
Cents::from(Dollars::from(
pub fn parse_cents(array: &[serde_json::Value], index: usize) -> CentsUnsigned {
CentsUnsigned::from(Dollars::from(
array
.get(index)
.and_then(|v| v.as_str())
@@ -14,9 +14,9 @@ pub fn parse_cents(array: &[serde_json::Value], index: usize) -> Cents {
))
}
/// Build OHLCCents from array indices 1-4 (open, high, low, close)
pub fn ohlc_from_array(array: &[serde_json::Value]) -> OHLCCents {
OHLCCents::from((
/// Build OHLCCentsUnsigned from array indices 1-4 (open, high, low, close)
pub fn ohlc_from_array(array: &[serde_json::Value]) -> OHLCCentsUnsigned {
OHLCCentsUnsigned::from((
Open::new(parse_cents(array, 1)),
High::new(parse_cents(array, 2)),
Low::new(parse_cents(array, 3)),
@@ -27,13 +27,13 @@ pub fn ohlc_from_array(array: &[serde_json::Value]) -> OHLCCents {
/// Compute OHLC for a block from a time series of minute data.
/// Aggregates all candles between previous_timestamp and timestamp.
pub fn compute_ohlc_from_range(
tree: &BTreeMap<Timestamp, OHLCCents>,
tree: &BTreeMap<Timestamp, OHLCCentsUnsigned>,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
source_name: &str,
) -> Result<OHLCCents> {
) -> Result<OHLCCentsUnsigned> {
let previous_ohlc = previous_timestamp
.map_or(Some(OHLCCents::default()), |t| tree.get(&t).cloned());
.map_or(Some(OHLCCentsUnsigned::default()), |t| tree.get(&t).cloned());
let last_ohlc = tree.get(&timestamp);
@@ -44,7 +44,7 @@ pub fn compute_ohlc_from_range(
}
let previous_ohlc = previous_ohlc.unwrap();
let mut result = OHLCCents::from(previous_ohlc.close);
let mut result = OHLCCentsUnsigned::from(previous_ohlc.close);
let start = previous_timestamp.unwrap_or(Timestamp::new(0));
let end = timestamp;

14
crates/brk_fetcher/src/source.rs
View File

@@ -1,7 +1,7 @@
use std::time::{Duration, Instant};
use brk_error::{Error, Result};
use brk_types::{Date, Height, OHLCCents, Timestamp};
use brk_types::{Date, Height, OHLCCentsUnsigned, Timestamp};
use tracing::info;
/// Default cooldown period for unhealthy sources (5 minutes)
@@ -12,17 +12,17 @@ pub trait PriceSource {
fn name(&self) -> &'static str;
/// Fetch daily OHLC for a date. Returns None if this source doesn't support date queries.
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCents>>;
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCentsUnsigned>>;
/// Fetch minute OHLC for a timestamp range. Returns None if unsupported.
fn get_1mn(
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Option<Result<OHLCCents>>;
) -> Option<Result<OHLCCentsUnsigned>>;
/// Fetch OHLC by block height. Returns None if unsupported.
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCents>>;
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCentsUnsigned>>;
/// Check if the source is reachable
fn ping(&self) -> Result<()>;
@@ -115,7 +115,7 @@ impl<T: PriceSource> PriceSource for TrackedSource<T> {
self.source.name()
}
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCents>> {
fn get_date(&mut self, date: Date) -> Option<Result<OHLCCentsUnsigned>> {
self.try_fetch(|s| s.get_date(date))
}
@@ -123,11 +123,11 @@ impl<T: PriceSource> PriceSource for TrackedSource<T> {
&mut self,
timestamp: Timestamp,
previous_timestamp: Option<Timestamp>,
) -> Option<Result<OHLCCents>> {
) -> Option<Result<OHLCCentsUnsigned>> {
self.try_fetch(|s| s.get_1mn(timestamp, previous_timestamp))
}
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCents>> {
fn get_height(&mut self, height: Height) -> Option<Result<OHLCCentsUnsigned>> {
self.try_fetch(|s| s.get_height(height))
}