Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-30 20:28:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

bitview: reorg part 9

Browse Source
This commit is contained in:
nym21
2025-10-01 23:17:48 +02:00
parent 62d4b35c93
commit c4ce718bb2
102 changed files with 1654 additions and 1798 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_fetcher/src/binance.rs
+17 -16
View File
@@ -3,13 +3,12 @@ use std::{
fs::{self, File},
io::BufReader,
path::{Path, PathBuf},
str::FromStr,
};
use brk_error::{Error, Result};
use brk_structs::{Cents, OHLCCents, Timestamp};
use log::info;
use serde_json::Value;
use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry};
@@ -17,7 +16,7 @@ use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry};
pub struct Binance {
path: Option<PathBuf>,
_1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
pub _1d: Option<BTreeMap<Date, OHLCCents>>,
_1d: Option<BTreeMap<Date, OHLCCents>>,
har: Option<BTreeMap<Timestamp, OHLCCents>>,
}
@@ -69,11 +68,11 @@ impl Binance {
info!("Fetching 1mn prices from Binance...");
default_retry(|_| {
Self::json_to_timestamp_to_ohlc(
&minreq::get(Self::url("interval=1m&limit=1000"))
Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(
minreq::get(Self::url("interval=1m&limit=1000"))
.send()?
.json()?,
)
.as_str()?,
)?)
})
}
@@ -94,7 +93,9 @@ impl Binance {
info!("Fetching daily prices from Binance...");
default_retry(|_| {
Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?)
Self::json_to_date_to_ohlc(&sonic_rs::from_str(
minreq::get(Self::url("interval=1d")).send()?.as_str()?,
)?)
})
}
@@ -119,7 +120,7 @@ impl Binance {
let reader = BufReader::new(file);
let json: BTreeMap<String, Value> = if let Ok(json) = serde_json::from_reader(reader) {
let json: BTreeMap<String, Value> = if let Ok(json) = sonic_rs::from_reader(reader) {
json
} else {
return Ok(Default::default());
@@ -129,7 +130,7 @@ impl Binance {
.ok_or(Error::Str("Expect object to have log attribute"))?
.as_object()
.ok_or(Error::Str("Expect to be an object"))?
.get("entries")
.get(&"entries")
.ok_or(Error::Str("Expect object to have entries"))?
.as_array()
.ok_or(Error::Str("Expect to be an array"))?
@@ -138,11 +139,11 @@ impl Binance {
entry
.as_object()
.unwrap()
.get("request")
.get(&"request")
.unwrap()
.as_object()
.unwrap()
.get("url")
.get(&"url")
.unwrap()
.as_str()
.unwrap()
@@ -152,14 +153,14 @@ impl Binance {
let response = entry
.as_object()
.unwrap()
.get("response")
.get(&"response")
.unwrap()
.as_object()
.unwrap();
let content = response.get("content").unwrap().as_object().unwrap();
let content = response.get(&"content").unwrap().as_object().unwrap();
let text = content.get("text");
let text = content.get(&"text");
if text.is_none() {
return Ok(BTreeMap::new());
@@ -167,7 +168,7 @@ impl Binance {
let text = text.unwrap().as_str().unwrap();
Self::json_to_timestamp_to_ohlc(&serde_json::Value::from_str(text).unwrap())
Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(text).unwrap())
})
.try_fold(BTreeMap::default(), |mut all, res| {
all.append(&mut res?);
crates/brk_fetcher/src/brk.rs
+3 -3
View File
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use log::info;
use serde_json::Value;
use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
use crate::{Close, Dollars, High, Low, Open, default_retry};
@@ -51,7 +51,7 @@ impl BRK {
height + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
let body: Value = sonic_rs::from_str(minreq::get(url).send()?.as_str()?)?;
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
@@ -96,7 +96,7 @@ impl BRK {
dateindex + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
let body: Value = sonic_rs::from_str(minreq::get(url).send()?.json()?)?;
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
crates/brk_fetcher/src/kraken.rs
+11 -5
View File
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
use brk_error::{Error, Result};
use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
use log::info;
use serde_json::Value;
use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
use crate::{Fetcher, default_retry};
@@ -36,7 +36,9 @@ impl Kraken {
info!("Fetching 1mn prices from Kraken...");
default_retry(|_| {
Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?)
Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(
minreq::get(Self::url(1)).send()?.as_str()?,
)?)
})
}
@@ -55,7 +57,11 @@ impl Kraken {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
default_retry(|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?))
default_retry(|_| {
Self::json_to_date_to_ohlc(&sonic_rs::from_str(
minreq::get(Self::url(1440)).send()?.as_str()?,
)?)
})
}
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
@@ -73,11 +79,11 @@ impl Kraken {
{
json.as_object()
.ok_or(Error::Str("Expect to be an object"))?
.get("result")
.get(&"result")
.ok_or(Error::Str("Expect object to have result"))?
.as_object()
.ok_or(Error::Str("Expect to be an object"))?
.get("XXBTZUSD")
.get(&"XXBTZUSD")
.ok_or(Error::Str("Expect to have XXBTZUSD"))?
.as_array()
.ok_or(Error::Str("Expect to be an array"))?
crates/brk_fetcher/src/retry.rs
+1
View File
@@ -22,6 +22,7 @@ where
if i == retries || res.is_ok() {
return res;
} else {
let _ = dbg!(res);
info!("Failed, waiting {sleep_in_s} seconds...");
sleep(Duration::from_secs(sleep_in_s));
}