release: v0.0.43

Cargo.lock

@@ -388,7 +388,7 @@ dependencies = [
 
 [[package]]
 name = "brk"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "brk_cli",
  "brk_computer",
@@ -407,7 +407,7 @@ dependencies = [
 
 [[package]]
 name = "brk_cli"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bitcoincore-rpc",
  "brk_computer",
@@ -432,7 +432,7 @@ dependencies = [
 
 [[package]]
 name = "brk_computer"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -453,7 +453,7 @@ dependencies = [
 
 [[package]]
 name = "brk_core"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bincode",
  "bitcoin",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "brk_exit"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "brk_logger",
  "ctrlc",
@@ -483,7 +483,7 @@ dependencies = [
 
 [[package]]
 name = "brk_fetcher"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -496,7 +496,7 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -514,7 +514,7 @@ dependencies = [
 
 [[package]]
 name = "brk_logger"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -524,7 +524,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -539,7 +539,7 @@ dependencies = [
 
 [[package]]
 name = "brk_query"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "brk_computer",
  "brk_core",
@@ -557,7 +557,7 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "axum",
  "bitcoincore-rpc",
@@ -586,7 +586,7 @@ dependencies = [
 
 [[package]]
 name = "brk_state"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "bincode",
  "brk_core",
@@ -600,7 +600,7 @@ dependencies = [
 
 [[package]]
 name = "brk_store"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "arc-swap",
  "brk_core",
@@ -610,7 +610,7 @@ dependencies = [
 
 [[package]]
 name = "brk_vec"
-version = "0.0.42"
+version = "0.0.43"
 dependencies = [
  "arc-swap",
  "brk_core",

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.42"
+package.version = "0.0.43"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 

crates/brk_computer/src/vecs/stateful/cohort.rs

@@ -54,8 +54,8 @@ pub struct Vecs {
     pub height_to_satblocks_destroyed: EagerVec<Height, Sats>,
     pub height_to_satdays_destroyed: EagerVec<Height, Sats>,
 
-    pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<Bitcoin>,
-    pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<Bitcoin>,
+    pub indexes_to_coinblocks_destroyed: ComputedVecsFromHeight<StoredF64>,
+    pub indexes_to_coindays_destroyed: ComputedVecsFromHeight<StoredF64>,
     pub dateindex_to_adjusted_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
     pub dateindex_to_realized_cap_30d_change: Option<EagerVec<DateIndex, Dollars>>,
     pub dateindex_to_sell_side_risk_ratio: Option<EagerVec<DateIndex, StoredF32>>,
@@ -891,7 +891,7 @@ impl Vecs {
                 path,
                 &suffix("coinblocks_destroyed"),
                 true,
-                version + VERSION + Version::ONE,
+                version + VERSION + Version::TWO,
                 format,
                 StorableVecGeneatorOptions::default().add_sum(),
             )?,
@@ -899,7 +899,7 @@ impl Vecs {
                 path,
                 &suffix("coindays_destroyed"),
                 true,
-                version + VERSION + Version::ONE,
+                version + VERSION + Version::TWO,
                 format,
                 StorableVecGeneatorOptions::default().add_sum(),
             )?,
@@ -1839,9 +1839,10 @@ impl Vecs {
             starting_indexes,
             exit,
             |v, _, _, starting_indexes, exit| {
-                v.compute_from_sats(
+                v.compute_transform(
                     starting_indexes.height,
                     &self.height_to_satblocks_destroyed,
+                    |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
                     exit,
                 )
             },
@@ -1853,9 +1854,10 @@ impl Vecs {
             starting_indexes,
             exit,
             |v, _, _, starting_indexes, exit| {
-                v.compute_from_sats(
+                v.compute_transform(
                     starting_indexes.height,
                     &self.height_to_satdays_destroyed,
+                    |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
                     exit,
                 )
             },

crates/brk_core/src/structs/dateindex.rs

@@ -1,4 +1,7 @@
-use std::ops::Add;
+use std::{
+    fmt,
+    ops::{Add, Rem},
+};
 
 use serde::Serialize;
 // use color_eyre::eyre::eyre;
@@ -77,3 +80,16 @@ impl CheckedSub for DateIndex {
         self.0.checked_sub(rhs.0).map(Self)
     }
 }
+
+impl Rem<usize> for DateIndex {
+    type Output = Self;
+    fn rem(self, rhs: usize) -> Self::Output {
+        Self(self.0 % rhs as u16)
+    }
+}
+
+impl fmt::Display for DateIndex {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}

crates/brk_fetcher/examples/main.rs

@@ -1,11 +1,15 @@
-use brk_core::Date;
-use brk_fetcher::Fetcher;
+use brk_core::{Date, Height};
+use brk_fetcher::{BRK, Fetcher};
 
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
     brk_logger::init(None);
 
+    let mut brk = BRK::default();
+    dbg!(brk.get_from_height(Height::new(900_000))?);
+    dbg!(brk.get_from_date(Date::new(2025, 6, 7))?);
+
     let mut fetcher = Fetcher::import(None)?;
 
     dbg!(fetcher.get_date(Date::new(2025, 6, 5))?);

crates/brk_fetcher/src/fetchers/brk.rs

@@ -0,0 +1,143 @@
+use std::collections::BTreeMap;
+
+use brk_core::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
+use color_eyre::eyre::{ContextCompat, eyre};
+use log::info;
+use serde_json::Value;
+
+use crate::{Close, Dollars, High, Low, Open, fetchers::retry};
+
+#[derive(Default, Clone)]
+pub struct BRK {
+    height_to_ohlc: BTreeMap<Height, Vec<OHLCCents>>,
+    dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
+}
+
+const API_URL: &str = "https://bitcoinresearchkit.org/api";
+
+const RETRIES: usize = 10;
+
+const CHUNK_SIZE: usize = 10_000;
+
+impl BRK {
+    pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
+        let key = height.checked_sub(height % CHUNK_SIZE).unwrap();
+
+        #[allow(clippy::map_entry)]
+        if !self.height_to_ohlc.contains_key(&key)
+            || ((key + self.height_to_ohlc.get(&key).unwrap().len()) <= height)
+        {
+            self.height_to_ohlc.insert(
+                key,
+                Self::fetch_height_prices(key).inspect_err(|e| {
+                    dbg!(e);
+                })?,
+            );
+        }
+
+        self.height_to_ohlc
+            .get(&key)
+            .unwrap()
+            .get(usize::from(height.checked_sub(key).unwrap()))
+            .cloned()
+            .ok_or(eyre!("Couldn't find height in kibo"))
+    }
+
+    fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
+        info!("Fetching Kibo height {height} prices...");
+
+        retry(
+            |_| {
+                let url = format!(
+                    "{API_URL}/query?index=height&values=ohlc&from={}&to={}",
+                    height,
+                    height + CHUNK_SIZE
+                );
+
+                let body: Value = minreq::get(url).send()?.json()?;
+
+                body.as_array()
+                    .context("Expect to be an array")?
+                    .iter()
+                    .map(Self::value_to_ohlc)
+                    .collect::<Result<Vec<_>, _>>()
+            },
+            30,
+            RETRIES,
+        )
+    }
+
+    pub fn get_from_date(&mut self, date: Date) -> color_eyre::Result<OHLCCents> {
+        let dateindex = DateIndex::try_from(date)?;
+
+        let key = dateindex.checked_sub(dateindex % CHUNK_SIZE).unwrap();
+
+        #[allow(clippy::map_entry)]
+        if !self.dateindex_to_ohlc.contains_key(&key)
+            || ((key + self.dateindex_to_ohlc.get(&key).unwrap().len()) <= dateindex)
+        {
+            self.dateindex_to_ohlc.insert(
+                key,
+                Self::fetch_date_prices(key).inspect_err(|e| {
+                    dbg!(e);
+                })?,
+            );
+        }
+
+        self.dateindex_to_ohlc
+            .get(&key)
+            .unwrap()
+            .get(usize::from(dateindex.checked_sub(key).unwrap()))
+            .cloned()
+            .ok_or(eyre!("Couldn't find date in kibo"))
+    }
+
+    fn fetch_date_prices(dateindex: DateIndex) -> color_eyre::Result<Vec<OHLCCents>> {
+        info!("Fetching Kibo dateindex {dateindex} prices...");
+
+        retry(
+            |_| {
+                let url = format!(
+                    "{API_URL}/query?index=dateindex&values=ohlc&from={}&to={}",
+                    dateindex,
+                    dateindex + CHUNK_SIZE
+                );
+
+                let body: Value = minreq::get(url).send()?.json()?;
+
+                body.as_array()
+                    .context("Expect to be an array")?
+                    .iter()
+                    .map(Self::value_to_ohlc)
+                    .collect::<Result<Vec<_>, _>>()
+            },
+            30,
+            RETRIES,
+        )
+    }
+
+    fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
+        let ohlc = value.as_array().context("Expect as_array to work")?;
+
+        let get_value = |index: usize| -> color_eyre::Result<_> {
+            Ok(Cents::from(Dollars::from(
+                ohlc.get(index)
+                    .context("Expect index key to work")?
+                    .as_f64()
+                    .context("Expect as_f64 to work")?,
+            )))
+        };
+
+        Ok(OHLCCents::from((
+            Open::new(get_value(0)?),
+            High::new(get_value(1)?),
+            Low::new(get_value(2)?),
+            Close::new(get_value(3)?),
+        )))
+    }
+
+    pub fn clear(&mut self) {
+        self.height_to_ohlc.clear();
+        self.dateindex_to_ohlc.clear();
+    }
+}

crates/brk_fetcher/src/fetchers/kibo.rs

@@ -1,155 +0,0 @@
-use std::{collections::BTreeMap, str::FromStr};
-
-use brk_core::{CheckedSub, Date, Height, OHLCCents};
-use color_eyre::eyre::{ContextCompat, eyre};
-use log::info;
-use serde_json::Value;
-
-use crate::{Cents, Close, Dollars, High, Low, Open, fetchers::retry};
-
-#[derive(Default, Clone)]
-pub struct Kibo {
-    height_to_ohlc_vec: BTreeMap<Height, Vec<OHLCCents>>,
-    year_to_date_to_ohlc: BTreeMap<u16, BTreeMap<Date, OHLCCents>>,
-}
-
-const KIBO_OFFICIAL_URL: &str = "https://kibo.money/api";
-
-const RETRIES: usize = 10;
-
-impl Kibo {
-    pub fn get_from_height(&mut self, height: Height) -> color_eyre::Result<OHLCCents> {
-        let key = height.checked_sub(height % 10_000).unwrap_or_default();
-
-        #[allow(clippy::map_entry)]
-        if !self.height_to_ohlc_vec.contains_key(&key)
-            || ((key + self.height_to_ohlc_vec.get(&key).unwrap().len()) <= height)
-        {
-            self.height_to_ohlc_vec.insert(
-                key,
-                Self::fetch_height_prices(key).inspect_err(|e| {
-                    dbg!(e);
-                })?,
-            );
-        }
-
-        self.height_to_ohlc_vec
-            .get(&key)
-            .unwrap()
-            .get(usize::from(height.checked_sub(key).unwrap()))
-            .cloned()
-            .ok_or(eyre!("Couldn't find height in kibo"))
-    }
-
-    fn fetch_height_prices(height: Height) -> color_eyre::Result<Vec<OHLCCents>> {
-        info!("Fetching Kibo height {height} prices...");
-
-        retry(
-            |_| {
-                let url = format!("{KIBO_OFFICIAL_URL}/height-to-price?chunk={}", height);
-
-                let body: Value = minreq::get(url).send()?.json()?;
-
-                body.as_object()
-                    .context("Expect to be an object")?
-                    .get("dataset")
-                    .context("Expect object to have dataset")?
-                    .as_object()
-                    .context("Expect to be an object")?
-                    .get("map")
-                    .context("Expect to have map")?
-                    .as_array()
-                    .context("Expect to be an array")?
-                    .iter()
-                    .map(Self::value_to_ohlc)
-                    .collect::<Result<Vec<_>, _>>()
-            },
-            30,
-            RETRIES,
-        )
-    }
-
-    pub fn get_from_date(&mut self, date: &Date) -> color_eyre::Result<OHLCCents> {
-        let year = date.year();
-
-        #[allow(clippy::map_entry)]
-        if !self.year_to_date_to_ohlc.contains_key(&year)
-            || self
-                .year_to_date_to_ohlc
-                .get(&year)
-                .unwrap()
-                .last_key_value()
-                .unwrap()
-                .0
-                <= date
-        {
-            self.year_to_date_to_ohlc
-                .insert(year, Self::fetch_date_prices(year)?);
-        }
-
-        self.year_to_date_to_ohlc
-            .get(&year)
-            .unwrap()
-            .get(date)
-            .cloned()
-            .ok_or(eyre!("Couldn't find date in kibo"))
-    }
-
-    fn fetch_date_prices(year: u16) -> color_eyre::Result<BTreeMap<Date, OHLCCents>> {
-        info!("Fetching Kibo date {year} prices...");
-
-        retry(
-            |_| {
-                let body: Value =
-                    minreq::get(format!("{KIBO_OFFICIAL_URL}/date-to-price?chunk={}", year))
-                        .send()?
-                        .json()?;
-
-                body.as_object()
-                    .context("Expect to be an object")?
-                    .get("dataset")
-                    .context("Expect object to have dataset")?
-                    .as_object()
-                    .context("Expect to be an object")?
-                    .get("map")
-                    .context("Expect to have map")?
-                    .as_object()
-                    .context("Expect to be an object")?
-                    .iter()
-                    .map(|(serialized_date, value)| -> color_eyre::Result<_> {
-                        let date =
-                            Date::from(jiff::civil::Date::from_str(serialized_date).unwrap());
-                        Ok((date, Self::value_to_ohlc(value)?))
-                    })
-                    .collect::<Result<BTreeMap<_, _>, _>>()
-            },
-            30,
-            RETRIES,
-        )
-    }
-
-    fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLCCents> {
-        let ohlc = value.as_object().context("Expect as_object to work")?;
-
-        let get_value = |key: &str| -> color_eyre::Result<_> {
-            Ok(Cents::from(Dollars::from(
-                ohlc.get(key)
-                    .context("Expect get key to work")?
-                    .as_f64()
-                    .context("Expect as_f64 to work")?,
-            )))
-        };
-
-        Ok(OHLCCents::from((
-            Open::new(get_value("open")?),
-            High::new(get_value("high")?),
-            Low::new(get_value("low")?),
-            Close::new(get_value("close")?),
-        )))
-    }
-
-    pub fn clear(&mut self) {
-        self.height_to_ohlc_vec.clear();
-        self.year_to_date_to_ohlc.clear();
-    }
-}

crates/brk_fetcher/src/fetchers/mod.rs

@@ -1,9 +1,9 @@
 mod binance;
-// mod kibo;
+mod brk;
 mod kraken;
 mod retry;
 
 pub use binance::*;
-// pub use kibo::*;
+pub use brk::*;
 pub use kraken::*;
 use retry::*;

crates/brk_fetcher/src/lib.rs

@@ -10,7 +10,7 @@ use color_eyre::eyre::Error;
 
 mod fetchers;
 
-use fetchers::*;
+pub use fetchers::*;
 use log::info;
 
 const TRIES: usize = 12 * 60;
@@ -19,7 +19,7 @@ const TRIES: usize = 12 * 60;
 pub struct Fetcher {
     binance: Binance,
     kraken: Kraken,
-    // kibo: Kibo,
+    brk: BRK,
 }
 
 impl Fetcher {
@@ -31,7 +31,7 @@ impl Fetcher {
         Ok(Self {
             binance: Binance::init(hars_path),
             kraken: Kraken::default(),
-            // kibo: Kibo::default(),
+            brk: BRK::default(),
         })
     }
 
@@ -46,6 +46,10 @@ impl Fetcher {
                 // eprintln!("{e}");
                 self.kraken.get_from_1d(&date)
             })
+            .or_else(|_| {
+                // eprintln!("{e}");
+                self.brk.get_from_date(date)
+            })
             .or_else(|e| {
                 sleep(Duration::from_secs(60));
 
@@ -94,28 +98,28 @@ impl Fetcher {
                     .get_from_1mn(timestamp, previous_timestamp)
                     .unwrap_or_else(|_report| {
                         //         // eprintln!("{_report}");
-                        // self.kibo.get_from_height(height).unwrap_or_else(|_report| {
-                        // eprintln!("{_report}");
+                        self.brk.get_from_height(height).unwrap_or_else(|_report| {
+                            // eprintln!("{_report}");
 
-                        sleep(Duration::from_secs(60));
+                            sleep(Duration::from_secs(60));
 
-                        if tries < TRIES {
-                            self.clear();
+                            if tries < TRIES {
+                                self.clear();
 
-                            info!("Retrying to fetch height prices...");
-                            // dbg!((height, timestamp, previous_timestamp));
+                                info!("Retrying to fetch height prices...");
+                                // dbg!((height, timestamp, previous_timestamp));
 
-                            return self
-                                .get_height_(height, timestamp, previous_timestamp, tries + 1)
-                                .unwrap();
-                        }
+                                return self
+                                    .get_height_(height, timestamp, previous_timestamp, tries + 1)
+                                    .unwrap();
+                            }
 
-                        info!("Failed to fetch height prices");
+                            info!("Failed to fetch height prices");
 
-                        let date = Date::from(timestamp);
-                        // eprintln!("{e}");
-                        panic!(
-                            "
+                            let date = Date::from(timestamp);
+                            // eprintln!("{e}");
+                            panic!(
+                                "
 Can't find the price for: height: {height} - date: {date}
 1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
 How to fix this:
@@ -130,8 +134,8 @@ How to fix this:
 8. Export to a har file (if there is no explicit button, click on the cog button)
 9. Move the file to 'parser/imports/binance.har'
         "
-                        )
-                        // })
+                            )
+                        })
                     })
             });
 
@@ -182,7 +186,7 @@ How to fix this:
 
     pub fn clear(&mut self) {
         self.binance.clear();
-        // self.kibo.clear();
+        self.brk.clear();
         self.kraken.clear();
     }
 }