Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-28 00:29:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

general: snapshot

Browse Source
This commit is contained in:
k
2024-07-25 14:43:20 +02:00
parent 0f8d7d5fe2
commit d3d5e7f8d7
27 changed files with 254 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
parser/src/actions/iter_blocks.rs
View File

@@ -73,26 +73,6 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
let current_block_date = Date::from_timestamp(timestamp);
let current_block_height = height + blocks_loop_i;
if states.address_cohorts_durable_states.is_none()
&& datasets
.address
.needs_durable_states(current_block_height, current_block_date)
{
states.address_cohorts_durable_states =
Some(AddressCohortsDurableStates::init(
&mut databases.address_index_to_address_data,
));
}
if states.utxo_cohorts_durable_states.is_none()
&& datasets
.utxo
.needs_durable_states(current_block_height, current_block_date)
{
states.utxo_cohorts_durable_states =
Some(UTXOCohortsDurableStates::init(&states.date_data_vec));
}
let next_block_date = next_block_opt
.as_ref()
.map(|next_block| Date::from_timestamp(next_block.header.time));
@@ -137,6 +117,27 @@ pub fn iter_blocks(bitcoin_db: &BitcoinDB, block_count: usize) -> color_eyre::Re
blocks_loop_date,
);
if states.address_cohorts_durable_states.is_none()
&& (compute_addresses
|| datasets
.address
.needs_durable_states(current_block_height, current_block_date))
{
states.address_cohorts_durable_states =
Some(AddressCohortsDurableStates::init(
&mut databases.address_index_to_address_data,
));
}
if states.utxo_cohorts_durable_states.is_none()
&& datasets
.utxo
.needs_durable_states(current_block_height, current_block_date)
{
states.utxo_cohorts_durable_states =
Some(UTXOCohortsDurableStates::init(&states.date_data_vec));
}
parse(ParseData {
bitcoin_db,
block: current_block,

11
parser/src/datasets/mod.rs
View File

@@ -277,13 +277,16 @@ impl AllDatasets {
let datasets_len = path_to_type.len();
Json::export("../datasets/disk_path_to_type.json", &path_to_type)?;
let server_inputs_path = "../server/in";
let server_trigger_path = "../server/.trigger";
fs::create_dir_all(server_inputs_path)?;
fs::create_dir_all(server_trigger_path)?;
Json::export(
&format!("{server_inputs_path}/disk_path_to_type.json"),
&path_to_type,
)?;
let datasets_len_path = format!("{server_trigger_path}/datasets_len.txt");
let datasets_len_path = format!("{server_inputs_path}/datasets_len.txt");
if let Ok(len) = fs::read_to_string(&datasets_len_path) {
if let Ok(len) = len.parse::<usize>() {

89
parser/src/datasets/price/mod.rs
View File

@@ -26,6 +26,7 @@ pub struct PriceDatasets {
kraken_daily: Option<BTreeMap<Date, OHLC>>,
kraken_1mn: Option<BTreeMap<u32, OHLC>>,
binance_1mn: Option<BTreeMap<u32, OHLC>>,
binance_daily: Option<BTreeMap<Date, OHLC>>,
binance_har: Option<BTreeMap<u32, OHLC>>,
satonomics_by_height: BTreeMap<HeightMapChunkId, Vec<OHLC>>,
satonomics_by_date: BTreeMap<DateMapChunkId, BTreeMap<Date, OHLC>>,
@@ -89,6 +90,7 @@ impl PriceDatasets {
min_initial_states: MinInitialStates::default(),
binance_1mn: None,
binance_daily: None,
binance_har: None,
kraken_1mn: None,
kraken_daily: None,
@@ -310,8 +312,9 @@ impl PriceDatasets {
Ok(self.ohlcs.date.get(&date).unwrap().to_owned())
} else {
let ohlc = self
.get_from_date_satonomics(&date)
.or_else(|_| self.get_from_daily_kraken(&date))?;
.get_from_daily_kraken(&date)
.or_else(|_| self.get_from_daily_binance(&date))
.or_else(|_| self.get_from_date_satonomics(&date))?;
self.ohlcs.date.insert(date, ohlc);
@@ -323,7 +326,16 @@ impl PriceDatasets {
let chunk_id = date.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_date.contains_key(&chunk_id) {
if !self.satonomics_by_date.contains_key(&chunk_id)
|| self
.satonomics_by_date
.get(&chunk_id)
.unwrap()
.last_key_value()
.unwrap()
.0
<= date
{
self.satonomics_by_date
.insert(chunk_id, Satonomics::fetch_date_prices(chunk_id)?);
}
@@ -337,9 +349,17 @@ impl PriceDatasets {
}
fn get_from_daily_kraken(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.kraken_daily.is_none() {
self.kraken_daily
.replace(Kraken::fetch_daily_prices().or_else(|_| Binance::fetch_daily_prices())?);
if self.kraken_daily.is_none()
|| self
.kraken_daily
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= date
{
self.kraken_daily.replace(Kraken::fetch_daily_prices()?);
}
self.kraken_daily
@@ -350,6 +370,28 @@ impl PriceDatasets {
.ok_or(Error::msg("Couldn't find date"))
}
fn get_from_daily_binance(&mut self, date: &Date) -> color_eyre::Result<OHLC> {
if self.binance_daily.is_none()
|| self
.binance_daily
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= date
{
self.binance_daily.replace(Binance::fetch_daily_prices()?);
}
self.binance_daily
.as_ref()
.unwrap()
.get(date)
.cloned()
.ok_or(Error::msg("Couldn't find date"))
}
pub fn get_height_ohlc(
&mut self,
height: Height,
@@ -380,13 +422,13 @@ impl PriceDatasets {
let previous_timestamp = previous_timestamp.map(clean_timestamp);
let ohlc = self
.get_from_height_satonomics(&height)
.get_from_1mn_kraken(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_1mn_kraken(timestamp, previous_timestamp)
self.get_from_1mn_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_1mn_binance(timestamp, previous_timestamp)
self.get_from_har_binance(timestamp, previous_timestamp)
.unwrap_or_else(|_| {
self.get_from_har_binance(timestamp, previous_timestamp)
self.get_from_height_satonomics(&height)
.unwrap_or_else(|_| {
let date = Date::from_timestamp(timestamp);
@@ -419,7 +461,10 @@ How to fix this:
let chunk_id = height.to_chunk_id();
#[allow(clippy::map_entry)]
if !self.satonomics_by_height.contains_key(&chunk_id) {
if !self.satonomics_by_height.contains_key(&chunk_id)
|| ((chunk_id.to_usize() + self.satonomics_by_height.get(&chunk_id).unwrap().len())
<= height.to_usize())
{
self.satonomics_by_height
.insert(chunk_id, Satonomics::fetch_height_prices(chunk_id)?);
}
@@ -437,7 +482,16 @@ How to fix this:
timestamp: u32,
previous_timestamp: Option<u32>,
) -> color_eyre::Result<OHLC> {
if self.kraken_1mn.is_none() {
if self.kraken_1mn.is_none()
|| self
.kraken_1mn
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= &timestamp
{
self.kraken_1mn.replace(Kraken::fetch_1mn_prices()?);
}
@@ -449,7 +503,16 @@ How to fix this:
timestamp: u32,
previous_timestamp: Option<u32>,
) -> color_eyre::Result<OHLC> {
if self.binance_1mn.is_none() {
if self.binance_1mn.is_none()
|| self
.binance_1mn
.as_ref()
.unwrap()
.last_key_value()
.unwrap()
.0
<= &timestamp
{
self.binance_1mn.replace(Binance::fetch_1mn_prices()?);
}

2
parser/src/io/consts.rs
View File

@@ -1,2 +1,2 @@
pub const IMPORTS_FOLDER_PATH: &str = "./imports";
pub const INPUTS_FOLDER_PATH: &str = "./in";
pub const OUTPUTS_FOLDER_PATH: &str = "./target/outputs";

10
parser/src/price/binance.rs
View File

@@ -1,6 +1,6 @@
#![allow(dead_code)]
use std::{collections::BTreeMap, path::Path};
use std::{collections::BTreeMap, fs, path::Path};
use color_eyre::eyre::ContextCompat;
use itertools::Itertools;
@@ -8,7 +8,7 @@ use serde_json::Value;
use crate::{
datasets::OHLC,
io::{Json, IMPORTS_FOLDER_PATH},
io::{Json, INPUTS_FOLDER_PATH},
structs::Date,
utils::{log, retry},
};
@@ -19,7 +19,9 @@ impl Binance {
pub fn read_har_file() -> color_eyre::Result<BTreeMap<u32, OHLC>> {
log("binance: read har file");
let path_binance_har = Path::new(IMPORTS_FOLDER_PATH).join("binance.har");
fs::create_dir_all(INPUTS_FOLDER_PATH)?;
let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
let json: BTreeMap<String, Value> =
Json::import(path_binance_har.to_str().unwrap()).unwrap_or_default();
@@ -195,7 +197,7 @@ impl Binance {
.collect::<BTreeMap<_, _>>())
},
10,
5,
10,
)
}
}

4
parser/src/price/kraken.rs
View File

@@ -62,7 +62,7 @@ impl Kraken {
.collect::<BTreeMap<_, _>>())
},
10,
5,
10,
)
}
@@ -117,7 +117,7 @@ impl Kraken {
.collect::<BTreeMap<_, _>>())
},
10,
5,
10,
)
}
}

10
parser/src/price/satonomics.rs
View File

@@ -21,11 +21,11 @@ const RETRIES: usize = 10;
impl Satonomics {
fn get_base_url(try_index: usize) -> &'static str {
if try_index < RETRIES / 2 {
SATONOMICS_OFFICIAL_URL
} else {
SATONOMICS_OFFICIAL_BACKUP_URL
}
// if try_index < RETRIES / 2 {
SATONOMICS_OFFICIAL_URL
// } else {
// SATONOMICS_OFFICIAL_BACKUP_URL
// }
}
pub fn fetch_height_prices(chunk_id: HeightMapChunkId) -> color_eyre::Result<Vec<OHLC>> {