diff --git a/Cargo.lock b/Cargo.lock index 834bca84f..dbc010479 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,7 +371,6 @@ name = "brk_fetcher" version = "0.0.1" dependencies = [ "brk_core", - "brk_indexer", "brk_logger", "color-eyre", "jiff", diff --git a/README.md b/README.md index 7aecf84a9..6af995d39 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,82 @@ - - - - - kibō - - +# Bitcoin Research Kit + +> ⚠️ In the middle of a massive rewrite, use very carefully ⚠️ +> Everything is still a work in progress ## Description +The Bitcoin Research Kit is a suite of tools designed to help you extract, explore and analyze the data stored from your Bitcoin Core node. + +## Crates + +`brk` + +Wrapper around all other `brk-*` crates. + +> Status: ⚠️ + +`brk_cli` + +A command line interface to interact with the Bitcoin Research Kit. + +> Status: ❌ + +`brk_computer` + +A Bitcoin dataset computer, built on top of brk_indexer. + +> Status: ⚠️ + +`brk_core` + +The Core (Structs and Errors) of the Bitcoin Research Kit. + +> Status: ✅ + +`brk_fetcher` + +A Bitcoin price fetcher. + +> Status: ⚠️ + +`brk_indexer` + +A Bitcoin Core indexer built on top of brk_parser. + +> Status: ⚠️ + +`brk_logger` + +A clean logger used in the Bitcoin Research Kit. + +> Status: ✅ + +`brk_parser` + +A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust. + +> Status: ✅ + +`brk_server` + +A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer. + +> Status: ⚠️ + +`hodor` + +Hold the door, an exit blocker built on top of ctrlc. + +> Status: ✅ + +`storable_vec` + +A very small, fast, efficient and simple storable Vec. + +> Status: ✅ + +## Old README + [**kibō**](https://kibo.money) (_hope_ in japanese) is primarily an open source Bitcoin Core data extractor and visualizer (similar to [Glassnode](https://glassnode.com)) which goal is to empower anybody with data about Bitcoin for free. The project is split in 3 parts: @@ -25,13 +94,11 @@ If you are a user of [mempool.space](https://mempool.space), you'll find this to ## Instances -| URL | Type | Version | Status | Last Height | Up Time Ratio | +| URL | Front-end | Version | Status | Last Height | Up Time Ratio | | ------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| [kibo.money](https://kibo.money) | Main | ![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fkibo.money%2FCargo.toml&query=%24.package.version&style=for-the-badge&label=%20&color=%23db9e03) | ![Status](https://img.shields.io/uptimerobot/status/m797259009-043f6b92d4cc2deef7d13f50?style=for-the-badge&label=%20&up_color=%231cb454&down_color=%23e63636) | ![Height](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fkibo.money%2Fapi%2Flast-height.json&query=%24.value&style=for-the-badge&label=%20&color=%23f26610) | ![Ratio](https://img.shields.io/uptimerobot/ratio/m797259009-043f6b92d4cc2deef7d13f50?style=for-the-badge&label=%20&color=%232f73f1) | -| [backup.kibo.money](https://backup.kibo.money) | Backup | ![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fbackup.kibo.money%2FCargo.toml&query=%24.package.version&style=for-the-badge&label=%20&color=%23db9e03) | ![Status](https://img.shields.io/uptimerobot/status/m797259013-bb29a8264fab8786fb80c5ed?style=for-the-badge&label=%20&up_color=%231cb454&down_color=%23e63636) | ![Height](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fbackup.kibo.money%2Fapi%2Flast-height.json&query=%24.value&style=for-the-badge&label=%20&color=%23f26610) | ![Ratio](https://img.shields.io/uptimerobot/ratio/m797259013-bb29a8264fab8786fb80c5ed?style=for-the-badge&label=%20&color=%232f73f1) | -| [preview.kibo.money](https://preview.kibo.money) | Dev | ![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fpreview.kibo.money%2FCargo.toml&query=%24.package.version&style=for-the-badge&label=%20&color=%23db9e03) | ![Status](https://img.shields.io/uptimerobot/status/m797869753-d40fc161bcb34624857a8082?style=for-the-badge&label=%20&up_color=%231cb454&down_color=%23e63636) | ![Height](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpreview.kibo.money%2Fapi%2Flast-height.json&query=%24.value&style=for-the-badge&label=%20&color=%23f26610) | ![Ratio](https://img.shields.io/uptimerobot/ratio/m797869753-d40fc161bcb34624857a8082?style=for-the-badge&label=%20&color=%232f73f1) | +| [kibo.money](https://kibo.money) | kibo.money | ![Version](https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fkibo.money%2FCargo.toml&query=%24.package.version&style=for-the-badge&label=%20&color=%23db9e03) | ![Status](https://img.shields.io/uptimerobot/status/m797259009-043f6b92d4cc2deef7d13f50?style=for-the-badge&label=%20&up_color=%231cb454&down_color=%23e63636) | ![Height](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fkibo.money%2Fapi%2Flast-height.json&query=%24.value&style=for-the-badge&label=%20&color=%23f26610) | ![Ratio](https://img.shields.io/uptimerobot/ratio/m797259009-043f6b92d4cc2deef7d13f50?style=for-the-badge&label=%20&color=%232f73f1) | -Please open an issue if you want to add another instance +Feel free to open an issue if you want to add another instance ## Endpoints @@ -48,16 +115,6 @@ Please open an issue if you want to add another instance - [/api/date-to-close.csv?chunk=2009](https://kibo.money/api/date-to-close.csv?chunk=2009) - [/api/date-to-close.csv?all=true](https://kibo.money/api/date-to-close.csv?all=true) -## Roadmap - -- **More Datasets/Charts** -- **Simulations** -- **Nostr integration** -- **API Documentation** -- **Descriptions** -- **Docker support** -- **Start9 support** - ## Setup ### Requirements diff --git a/crates/brk_computer/src/storage/storable_vecs.rs b/crates/brk_computer/src/storage/storable_vecs.rs index dd0a83116..da879a04a 100644 --- a/crates/brk_computer/src/storage/storable_vecs.rs +++ b/crates/brk_computer/src/storage/storable_vecs.rs @@ -1,6 +1,9 @@ use std::{fs, path::Path}; -use brk_core::{Addressindex, Date, Dateindex, Feerate, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; +use brk_core::{ + Addressindex, Cents, Close, Date, Dateindex, Dollars, Feerate, Height, High, Low, Open, Sats, Timestamp, Txindex, + Txinindex, Txoutindex, +}; use storable_vec::{StorableVec, Version}; // mod base; @@ -11,6 +14,22 @@ pub struct StorableVecs { pub dateindex_to_first_height: StorableVec, // pub dateindex_to_last_height: StorableVec, // pub height_to_block_interval: StorableVec, + pub dateindex_to_close_in_cents: StorableVec, MODE>, + pub dateindex_to_close_in_dollars: StorableVec, MODE>, + pub dateindex_to_high_in_cents: StorableVec, MODE>, + pub dateindex_to_high_in_dollars: StorableVec, MODE>, + pub dateindex_to_low_in_cents: StorableVec, MODE>, + pub dateindex_to_low_in_dollars: StorableVec, MODE>, + pub dateindex_to_open_in_cents: StorableVec, MODE>, + pub dateindex_to_open_in_dollars: StorableVec, MODE>, + pub height_to_close_in_cents: StorableVec, MODE>, + pub height_to_close_in_dollars: StorableVec, MODE>, + pub height_to_high_in_cents: StorableVec, MODE>, + pub height_to_high_in_dollars: StorableVec, MODE>, + pub height_to_low_in_cents: StorableVec, MODE>, + pub height_to_low_in_dollars: StorableVec, MODE>, + pub height_to_open_in_cents: StorableVec, MODE>, + pub height_to_open_in_dollars: StorableVec, MODE>, pub height_to_date: StorableVec, pub height_to_dateindex: StorableVec, // pub height_to_fee: StorableVec, @@ -47,6 +66,46 @@ impl StorableVecs { Version::from(1), )?, // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, + dateindex_to_close_in_cents: StorableVec::import( + &path.join("dateindex_to_close_in_cents"), + Version::from(1), + )?, + dateindex_to_close_in_dollars: StorableVec::import( + &path.join("dateindex_to_close_in_dollars"), + Version::from(1), + )?, + dateindex_to_high_in_cents: StorableVec::import( + &path.join("dateindex_to_high_in_cents"), + Version::from(1), + )?, + dateindex_to_high_in_dollars: StorableVec::import( + &path.join("dateindex_to_high_in_dollars"), + Version::from(1), + )?, + dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?, + dateindex_to_low_in_dollars: StorableVec::import( + &path.join("dateindex_to_low_in_dollars"), + Version::from(1), + )?, + dateindex_to_open_in_cents: StorableVec::import( + &path.join("dateindex_to_open_in_cents"), + Version::from(1), + )?, + dateindex_to_open_in_dollars: StorableVec::import( + &path.join("dateindex_to_open_in_dollars"), + Version::from(1), + )?, + height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?, + height_to_close_in_dollars: StorableVec::import( + &path.join("height_to_close_in_dollars"), + Version::from(1), + )?, + height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?, + height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?, + height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?, + height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?, + height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?, + height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?, height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?, height_to_dateindex: StorableVec::forced_import(&path.join("height_to_dateindex"), Version::from(1))?, // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, @@ -90,10 +149,44 @@ impl StorableVecs { } // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] { - // [&self.dateindex_to_first_height] + // [ + // &self.dateindex_to_close_in_cents as &dyn AnyJsonStorableVec, + // &self.dateindex_to_close_in_dollars, + // &self.dateindex_to_high_in_cents, + // &self.dateindex_to_high_in_dollars, + // &self.dateindex_to_low_in_cents, + // &self.dateindex_to_low_in_dollars, + // &self.dateindex_to_open_in_cents, + // &self.dateindex_to_open_in_dollars, + // &self.height_to_close_in_cents, + // &self.height_to_close_in_dollars, + // &self.height_to_high_in_cents, + // &self.height_to_high_in_dollars, + // &self.height_to_low_in_cents, + // &self.height_to_low_in_dollars, + // &self.height_to_open_in_cents, + // &self.height_to_open_in_dollars, + // ] // } // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] { - // [&mut self.dateindex_to_first_height] + // [ + // &mut self.dateindex_to_close_in_cents as &mut dyn AnyStorableVec, + // &mut self.dateindex_to_close_in_dollars, + // &mut self.dateindex_to_high_in_cents, + // &mut self.dateindex_to_high_in_dollars, + // &mut self.dateindex_to_low_in_cents, + // &mut self.dateindex_to_low_in_dollars, + // &mut self.dateindex_to_open_in_cents, + // &mut self.dateindex_to_open_in_dollars, + // &mut self.height_to_close_in_cents, + // &mut self.height_to_close_in_dollars, + // &mut self.height_to_high_in_cents, + // &mut self.height_to_high_in_dollars, + // &mut self.height_to_low_in_cents, + // &mut self.height_to_low_in_dollars, + // &mut self.height_to_open_in_cents, + // &mut self.height_to_open_in_dollars, + // ] // } } diff --git a/crates/brk_core/src/structs/unit.rs b/crates/brk_core/src/structs/unit.rs index 177c6c568..035a1ff65 100644 --- a/crates/brk_core/src/structs/unit.rs +++ b/crates/brk_core/src/structs/unit.rs @@ -1,11 +1,11 @@ use byteview::ByteView; #[derive(Debug)] -pub struct Unit(); +pub struct Unit; impl From for Unit { fn from(_: ByteView) -> Self { - Self() + Self } } impl From for ByteView { diff --git a/crates/brk_fetcher/Cargo.toml b/crates/brk_fetcher/Cargo.toml index 64e846493..76d572085 100644 --- a/crates/brk_fetcher/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brk_fetcher" -description = "A Bitcoin price fetcher built on top of brk_indexer" +description = "A Bitcoin price fetcher" version.workspace = true edition.workspace = true license.workspace = true @@ -8,7 +8,6 @@ repository.workspace = true [dependencies] brk_core = { workspace = true } -brk_indexer = { workspace = true } brk_logger = { workspace = true } color-eyre = { workspace = true } jiff = { workspace = true } diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index 4be2da90e..df3e0c4a7 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -9,32 +9,14 @@ use color_eyre::eyre::Error; mod fetchers; -use brk_indexer::Indexer; +// use brk_indexer::Indexer; pub use fetchers::*; use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version}; pub struct Pricer { - path: PathBuf, binance: Binance, kraken: Kraken, kibo: Kibo, - - pub dateindex_to_close_in_cents: StorableVec, MODE>, - pub dateindex_to_close_in_dollars: StorableVec, MODE>, - pub dateindex_to_high_in_cents: StorableVec, MODE>, - pub dateindex_to_high_in_dollars: StorableVec, MODE>, - pub dateindex_to_low_in_cents: StorableVec, MODE>, - pub dateindex_to_low_in_dollars: StorableVec, MODE>, - pub dateindex_to_open_in_cents: StorableVec, MODE>, - pub dateindex_to_open_in_dollars: StorableVec, MODE>, - pub height_to_close_in_cents: StorableVec, MODE>, - pub height_to_close_in_dollars: StorableVec, MODE>, - pub height_to_high_in_cents: StorableVec, MODE>, - pub height_to_high_in_dollars: StorableVec, MODE>, - pub height_to_low_in_cents: StorableVec, MODE>, - pub height_to_low_in_dollars: StorableVec, MODE>, - pub height_to_open_in_cents: StorableVec, MODE>, - pub height_to_open_in_dollars: StorableVec, MODE>, } impl Pricer { @@ -42,61 +24,19 @@ impl Pricer { fs::create_dir_all(path)?; Ok(Self { - path: path.to_owned(), binance: Binance::init(path), kraken: Kraken::default(), kibo: Kibo::default(), - - dateindex_to_close_in_cents: StorableVec::import( - &path.join("dateindex_to_close_in_cents"), - Version::from(1), - )?, - dateindex_to_close_in_dollars: StorableVec::import( - &path.join("dateindex_to_close_in_dollars"), - Version::from(1), - )?, - dateindex_to_high_in_cents: StorableVec::import( - &path.join("dateindex_to_high_in_cents"), - Version::from(1), - )?, - dateindex_to_high_in_dollars: StorableVec::import( - &path.join("dateindex_to_high_in_dollars"), - Version::from(1), - )?, - dateindex_to_low_in_cents: StorableVec::import(&path.join("dateindex_to_low_in_cents"), Version::from(1))?, - dateindex_to_low_in_dollars: StorableVec::import( - &path.join("dateindex_to_low_in_dollars"), - Version::from(1), - )?, - dateindex_to_open_in_cents: StorableVec::import( - &path.join("dateindex_to_open_in_cents"), - Version::from(1), - )?, - dateindex_to_open_in_dollars: StorableVec::import( - &path.join("dateindex_to_open_in_dollars"), - Version::from(1), - )?, - height_to_close_in_cents: StorableVec::import(&path.join("height_to_close_in_cents"), Version::from(1))?, - height_to_close_in_dollars: StorableVec::import( - &path.join("height_to_close_in_dollars"), - Version::from(1), - )?, - height_to_high_in_cents: StorableVec::import(&path.join("height_to_high_in_cents"), Version::from(1))?, - height_to_high_in_dollars: StorableVec::import(&path.join("height_to_high_in_dollars"), Version::from(1))?, - height_to_low_in_cents: StorableVec::import(&path.join("height_to_low_in_cents"), Version::from(1))?, - height_to_low_in_dollars: StorableVec::import(&path.join("height_to_low_in_dollars"), Version::from(1))?, - height_to_open_in_cents: StorableVec::import(&path.join("height_to_open_in_cents"), Version::from(1))?, - height_to_open_in_dollars: StorableVec::import(&path.join("height_to_open_in_dollars"), Version::from(1))?, }) } - pub fn compute_if_needed(&mut self, indexer: &mut Indexer) { + pub fn compute_if_needed(&mut self) { // TODO: Remove all outdated - indexer - .vecs - .height_to_timestamp - .iter_from(Height::default(), |v| Ok(())); + // indexer + // .vecs + // .height_to_timestamp + // .iter_from(Height::default(), |v| Ok(())); // self.open // .multi_insert_simple_transform(heights, dates, &mut self.ohlc, &|ohlc| ohlc.open); @@ -227,46 +167,4 @@ impl Pricer { Ok(final_ohlc) } - - pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 16] { - [ - &self.dateindex_to_close_in_cents as &dyn AnyJsonStorableVec, - &self.dateindex_to_close_in_dollars, - &self.dateindex_to_high_in_cents, - &self.dateindex_to_high_in_dollars, - &self.dateindex_to_low_in_cents, - &self.dateindex_to_low_in_dollars, - &self.dateindex_to_open_in_cents, - &self.dateindex_to_open_in_dollars, - &self.height_to_close_in_cents, - &self.height_to_close_in_dollars, - &self.height_to_high_in_cents, - &self.height_to_high_in_dollars, - &self.height_to_low_in_cents, - &self.height_to_low_in_dollars, - &self.height_to_open_in_cents, - &self.height_to_open_in_dollars, - ] - } - - pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 16] { - [ - &mut self.dateindex_to_close_in_cents as &mut dyn AnyStorableVec, - &mut self.dateindex_to_close_in_dollars, - &mut self.dateindex_to_high_in_cents, - &mut self.dateindex_to_high_in_dollars, - &mut self.dateindex_to_low_in_cents, - &mut self.dateindex_to_low_in_dollars, - &mut self.dateindex_to_open_in_cents, - &mut self.dateindex_to_open_in_dollars, - &mut self.height_to_close_in_cents, - &mut self.height_to_close_in_dollars, - &mut self.height_to_high_in_cents, - &mut self.height_to_high_in_dollars, - &mut self.height_to_low_in_cents, - &mut self.height_to_low_in_dollars, - &mut self.height_to_open_in_cents, - &mut self.height_to_open_in_dollars, - ] - } } diff --git a/crates/hodor/Cargo.toml b/crates/hodor/Cargo.toml index 7689d6ba9..a069ca0ea 100644 --- a/crates/hodor/Cargo.toml +++ b/crates/hodor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hodor" -description = "Hold the door, an exit blocker that can wait until a task is completed" +description = "Hold the door, an exit blocker built on top of ctrlc" version = "0.1.1" edition.workspace = true license.workspace = true