general: snapshot

Cargo.toml

@@ -1,7 +1,8 @@
 [workspace]
 members = [
+    "cli",
     "computer",
-    "exit",
+    "hodor",
     "indexer",
     "iterator",
     "logger",
@@ -10,22 +11,27 @@ members = [
     "storable_vec",
 ]
 resolver = "2"
+package.license = "MIT"
+package.edition = "2021"
 
 [workspace.dependencies]
 bitcoin = { version = "0.32.5", features = ["serde"] }
 color-eyre = "0.6.3"
-computer = { path = "computer", package = "bomputer" }
+computer = { version = "0", path = "computer", package = "bomputer" }
 derive_deref = "1.1.1"
-exit = { path = "exit" }
 fjall = "2.6.3"
-indexer = { path = "indexer", package = "bindex" }
-iterator = { path = "iterator", package = "biter" }
+hodor = { version = "0", path = "hodor" }
+indexer = { version = "0", path = "indexer", package = "bindexer" }
+iterator = { version = "0", path = "iterator", package = "biter" }
 jiff = "0.2.1"
-logger = { path = "logger" }
+log = { version = "0.4.25" }
+logger = { version = "0", path = "logger", package = "cl0g" }
+minreq = { version = "2.13.2", features = ["https", "serde_json"] }
+pricer = { version = "0", path = "pricer", package = "bricer" }
 rayon = "1.10.0"
-pricer = { path = "pricer", package = "brice" }
 serde = { version = "1.0.217", features = ["derive"] }
+serde_bytes = "0.11.15"
 serde_json = { version = "1.0.138", features = ["float_roundtrip"] }
-server = { path = "server", package = "berver" }
-storable_vec = { path = "storable_vec", features = ["json"] }
+server = { version = "0", path = "server", package = "berver" }
+storable_vec = { version = "0", path = "storable_vec", features = ["json"] }
 zerocopy = { version = "0.8.18", features = ["derive"] }

cli/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "bli"
+description = "A command line interface to run berver"
+version = "0.1.0"
+edition = { workspace = true }
+license = { workspace = true }
+
+[dependencies]

cli/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

computer/Cargo.toml

@@ -1,16 +1,17 @@
 [package]
 name = "bomputer"
-description = "A Bitcoin dataset computer built on top of brice"
+description = "A Bitcoin dataset computer built on top of bindexer and bricer"
 version = "0.1.0"
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
-iterator = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
-exit = { workspace = true }
 fjall = { workspace = true }
+hodor = { workspace = true }
 indexer = { workspace = true }
+iterator = { workspace = true }
 pricer = { workspace = true }
 storable_vec = { workspace = true }
 zerocopy = { workspace = true }

computer/src/lib.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 
-use exit::Exit;
+use hodor::Exit;
 use indexer::Indexer;
 pub use iterator::rpc;
 

computer/src/main.rs

@@ -1,7 +1,7 @@
 use std::path::Path;
 
 use bomputer::Computer;
-use exit::Exit;
+use hodor::Exit;
 use indexer::Indexer;
 
 mod structs;

exit/Cargo.toml

@@ -1,8 +0,0 @@
-[package]
-name = "exit"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-ctrlc = "3.4.5"
-logger = { workspace = true }

hodor/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "hodor"
+description = "An exit blocker, can hold the door until a task is completed"
+version = "0.1.0"
+edition = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+ctrlc = "3.4.5"
+log = { workspace = true }

hodor/src/lib.rs

@@ -8,7 +8,7 @@ use std::{
     time::Duration,
 };
 
-use logger::info;
+use log::info;
 
 #[derive(Default, Clone)]
 pub struct Exit {

indexer/Cargo.toml

@@ -1,15 +1,16 @@
 [package]
-name = "bindex"
+name = "bindexer"
 description = "A bitcoin-core indexer built on top of biter"
 version = "0.1.0"
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 bitcoin = { workspace = true }
 color-eyre = { workspace = true }
 derive_deref = { workspace = true }
-exit = { workspace = true }
 fjall = { workspace = true }
+hodor = { workspace = true }
 iterator = { workspace = true }
 jiff = { workspace = true }
 logger = { workspace = true }
@@ -17,6 +18,6 @@ rapidhash = "1.3.0"
 rayon = { workspace = true }
 rlimit = { version = "0.10.2" }
 serde = { workspace = true }
-serde_bytes = "0.11.15"
+serde_bytes = { workspace = true }
 storable_vec = { workspace = true }
 zerocopy = { workspace = true }

indexer/README.md

@@ -12,7 +12,9 @@ Vecs are used sparingly instead of stores for multiple reasons:
 
 ## Usage
 
-Peaks at 11-12 GB of RAM
+Storage wise, the expected overhead should be around 30% of the chain itself.
+
+Peaks at 11-13 GB of RAM
 
 ## Outputs
 

indexer/src/lib.rs

@@ -10,7 +10,7 @@ pub use iterator::*;
 
 use bitcoin::{Transaction, TxIn, TxOut};
 use color_eyre::eyre::{eyre, ContextCompat};
-use exit::Exit;
+use hodor::Exit;
 use logger::info;
 use rayon::prelude::*;
 use storable_vec::CACHED_GETS;
@@ -31,14 +31,14 @@ pub struct Indexer<const MODE: u8> {
 
 impl<const MODE: u8> Indexer<MODE> {
     pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
-        info!("Importing indexes...");
-
+        // info!("Increasing limit of opened files to 210_000...");
         rlimit::setrlimit(
             rlimit::Resource::NOFILE,
             210_000,
             rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().1,
         )?;
 
+        info!("Importing indexes...");
         let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
         let trees = Fjalls::import(&indexes_dir.join("fjall"))?;
 
@@ -47,12 +47,12 @@ impl<const MODE: u8> Indexer<MODE> {
 }
 
 impl Indexer<CACHED_GETS> {
-    pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
+    pub fn index(&mut self, bitcoin_dir: &Path, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
         info!("Started indexing...");
 
         let check_collisions = true;
 
-        let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, &rpc)).unwrap_or_else(|_| {
+        let starting_indexes = Indexes::try_from((&mut self.vecs, &self.trees, rpc)).unwrap_or_else(|_| {
             let indexes = Indexes::default();
             indexes.push_if_needed(&mut self.vecs).unwrap();
             indexes

indexer/src/main.rs

@@ -1,8 +1,9 @@
-use std::path::Path;
+use std::{path::Path, thread::sleep, time::Duration};
 
-use bindex::Indexer;
-use exit::Exit;
+use bindexer::{rpc::RpcApi, Indexer};
+use hodor::Exit;
 use iterator::rpc;
+use logger::info;
 use storable_vec::CACHED_GETS;
 
 fn main() -> color_eyre::Result<()> {
@@ -11,19 +12,29 @@ fn main() -> color_eyre::Result<()> {
     logger::init_log(None);
 
     let data_dir = Path::new("../../bitcoin");
-    let rpc = rpc::Client::new(
+    let rpc = Box::leak(Box::new(rpc::Client::new(
         "http://localhost:8332",
         rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")),
-    )?;
+    )?));
     let exit = Exit::new();
 
-    let i = std::time::Instant::now();
+    loop {
+        let block_count = rpc.get_blockchain_info().unwrap().blocks as usize;
 
-    let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../_outputs/indexes"))?;
+        info!("{block_count} blocks found.");
 
-    indexer.index(data_dir, rpc, &exit)?;
+        let i = std::time::Instant::now();
 
-    dbg!(i.elapsed());
+        let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../_outputs/indexes"))?;
 
-    Ok(())
+        indexer.index(data_dir, rpc, &exit)?;
+
+        dbg!(i.elapsed());
+
+        info!("Waiting for a new block...");
+
+        while block_count == rpc.get_blockchain_info().unwrap().blocks as usize {
+            sleep(Duration::from_secs(1))
+        }
+    }
 }

iterator/Cargo.toml

@@ -1,18 +1,18 @@
 [package]
 name = "biter"
 description = "A very fast Bitcoin block iterator built on top of bitcoin-rust"
-version = "0.2.2"
-license = "MIT"
+version = "0.2.3"
 repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"
 keywords = ["bitcoin", "block", "iterator"]
 categories = ["cryptography::cryptocurrencies", "encoding"]
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 bitcoin = { workspace = true }
 rayon = { workspace = true }
 crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = "1.0.138"
+serde = { workspace = true }
+serde_json = { workspace = true }
 derive_deref = { workspace = true }
 bitcoincore-rpc = "0.19.0"

iterator/README.md

@@ -9,43 +9,7 @@ The element returned by the iterator is a tuple which includes the:
 
 ## Example
 
-```rust
-use std::path::Path;
-
-use bitcoincore_rpc::{Auth, Client};
-
-fn main() {
-    let i = std::time::Instant::now();
-
-    // Path to the Bitcoin data directory
-    let data_dir = "../../bitcoin";
-
-    // Inclusive starting height of the blocks received, `None` for 0
-    let start = Some(850_000);
-
-    // Inclusive ending height of the blocks received, `None` for the last one
-    let end = None;
-
-    // RPC client to filter out forks
-    let url = "http://localhost:8332";
-    let cookie = Path::new(data_dir).join(".cookie");
-    let auth = Auth::CookieFile(cookie);
-    let rpc = Client::new(url, auth).unwrap();
-
-    if cookie.is_file() {
-        Ok()
-
-    // Create channel receiver then iterate over the blocks
-    biter::new(data_dir, start, end, rpc)
-        .iter()
-        .for_each(|(height, _block, hash)| {
-            println!("{height}: {hash}");
-        });
-
-    dbg!(i.elapsed());
-}
-
-```
+`src/main.rs`
 
 ## Requirements
 

iterator/src/blk_index_to_blk_recap.rs

@@ -32,6 +32,8 @@ impl BlkIndexToBlkRecap {
             }
         };
 
+        // dbg!(&tree);
+
         let mut this = Self {
             path,
             tree,
@@ -43,7 +45,9 @@ impl BlkIndexToBlkRecap {
         this
     }
 
-    pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
+    fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
+        self.tree.pop_last();
+
         let mut unprocessed_keys = self.tree.keys().copied().collect::<BTreeSet<_>>();
 
         blocks_dir.iter().for_each(|(blk_index, blk_path)| {
@@ -59,22 +63,15 @@ impl BlkIndexToBlkRecap {
             self.tree.remove(&blk_index);
         });
 
-        while self.tree.last_entry().map(|last| *last.key()).is_some_and(|key| {
-            if key >= self.tree.len() {
-                self.tree.pop_last();
-                true
-            } else {
-                false
-            }
-        }) {}
-
         self.last_safe_height = self.tree.values().map(|recap| recap.height()).max();
     }
 
-    pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
+    pub fn get_start_recap(&mut self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
         if let Some(start) = start {
             let (last_key, last_value) = self.tree.last_key_value()?;
 
+            dbg!((last_key, last_value));
+
             if last_value.height() < start {
                 return Some((*last_key, *last_value));
             } else if let Some((blk_index, _)) =
@@ -82,7 +79,7 @@ impl BlkIndexToBlkRecap {
             {
                 if *blk_index != 0 {
                     // Temporary fix, need to rethink the whole thing
-                    let blk_index = (*blk_index).checked_sub(3).unwrap_or_default();
+                    let blk_index = (*blk_index).checked_sub(1).unwrap_or_default();
                     return Some((blk_index, *self.tree.get(&blk_index).unwrap()));
                 }
             }

iterator/src/blk_metadata.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use crate::path_to_modified_time;
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 pub struct BlkMetadata {
     pub index: usize,
     pub modified_time: u64,

iterator/src/blk_metadata_and_block.rs

@@ -2,6 +2,7 @@ use bitcoin::Block;
 
 use crate::BlkMetadata;
 
+#[derive(Debug)]
 pub struct BlkMetadataAndBlock {
     pub blk_metadata: BlkMetadata,
     pub block: Block,
@@ -9,9 +10,6 @@ pub struct BlkMetadataAndBlock {
 
 impl BlkMetadataAndBlock {
     pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
-        Self {
-            blk_metadata,
-            block,
-        }
+        Self { blk_metadata, block }
     }
 }

iterator/src/blk_recap.rs

@@ -30,6 +30,9 @@ impl BlkRecap {
     }
 
     pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
+        if self.modified_time != path_to_modified_time(blk_path) {
+            dbg!(self.modified_time, path_to_modified_time(blk_path));
+        }
         self.modified_time != path_to_modified_time(blk_path)
     }
 

iterator/src/lib.rs

@@ -36,11 +36,6 @@ pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 1000;
 const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
 const BOUND_CAP: usize = 210;
 
-enum BlockState {
-    Raw(Vec<u8>),
-    Decoded(Block),
-}
-
 ///
 /// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
 ///
@@ -82,7 +77,7 @@ pub fn new(
     data_dir: &Path,
     start: Option<usize>,
     end: Option<usize>,
-    rpc: bitcoincore_rpc::Client,
+    rpc: &'static bitcoincore_rpc::Client,
 ) -> Receiver<(usize, Block, BlockHash)> {
     let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
     let (send_block, recv_block) = bounded(BOUND_CAP);
@@ -225,6 +220,8 @@ pub fn new(
                               tuple: BlkMetadataAndBlock| {
             let mut tuple = Some(tuple);
 
+            println!("{} {} {}", recent_hashes.len(), recent_chain.len(), future_blocks.len(),);
+
             while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
                 let hash = tuple.block.block_hash();
 
@@ -306,3 +303,8 @@ pub fn new(
 
     recv_height_block_hash
 }
+
+enum BlockState {
+    Raw(Vec<u8>),
+    Decoded(Block),
+}

iterator/src/main.rs

@@ -6,12 +6,15 @@ fn main() {
     let i = std::time::Instant::now();
 
     let data_dir = Path::new("../../bitcoin");
-    let url = "http://localhost:8332";
-    let cookie = Path::new(data_dir).join(".cookie");
-    let auth = Auth::CookieFile(cookie);
-    let rpc = Client::new(url, auth).unwrap();
+    let rpc = Box::leak(Box::new(
+        Client::new(
+            "http://localhost:8332",
+            Auth::CookieFile(Path::new(data_dir).join(".cookie")),
+        )
+        .unwrap(),
+    ));
 
-    let start = None;
+    let start = Some(460_001);
     let end = None;
 
     biter::new(data_dir, start, end, rpc)

logger/Cargo.toml

@@ -1,10 +1,12 @@
 [package]
-name = "logger"
+name = "cl0g"
+description = "A clean logger"
 version = "0.1.0"
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 color-eyre = { workspace = true }
 env_logger = "0.11.6"
 jiff = { workspace = true }
-log = { version = "0.4.25" }
+log = { workspace = true }

logger/src/main.rs

@@ -1,5 +1,5 @@
+use cl0g::init_log;
 use log::info;
-use logger::init_log;
 
 fn main() {
     init_log(None);

pricer/Cargo.toml

@@ -1,8 +1,9 @@
 [package]
-name = "brice"
-description = "A bitcoin price fetcher built on top of bindex"
+name = "bricer"
+description = "A bitcoin price fetcher built on top of bindexer"
 version = "0.1.0"
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 color-eyre = { workspace = true }
@@ -10,7 +11,7 @@ derive_deref = { workspace = true }
 indexer = { workspace = true }
 jiff = { workspace = true }
 logger = { workspace = true }
-minreq = { version = "2.13.2", features = ["https", "serde_json"] }
+minreq = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 storable_vec = { workspace = true }

pricer/src/main.rs

@@ -1,4 +1,4 @@
-use brice::{Binance, Kibo, Kraken};
+use bricer::{Binance, Kibo, Kraken};
 use indexer::Height;
 use serde_json::Value;
 

server/Cargo.toml

@@ -1,7 +1,9 @@
 [package]
 name = "berver"
-version = "0.6.0"
-edition = "2021"
+description = "A Bitcoin data server built on top of bindexer, bricer and bomputer"
+version = "0.1.0"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 axum = "0.8.1"
@@ -12,7 +14,6 @@ indexer = { workspace = true }
 jiff = { workspace = true }
 logger = { workspace = true }
 oxc = { version = "0.51.0", features = ["codegen", "minifier"] }
-reqwest = { version = "0.12.12", features = ["blocking", "json"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
 storable_vec = { workspace = true }

server/src/api/vecs/mod.rs

@@ -2,12 +2,11 @@ use std::time::Instant;
 
 use axum::{
     extract::{Query, State},
-    http::{HeaderMap, Uri},
+    http::{HeaderMap, StatusCode, Uri},
     response::{IntoResponse, Response},
     Json,
 };
 use color_eyre::eyre::eyre;
-use reqwest::StatusCode;
 use serde_json::Value;
 
 use crate::{log_result, traits::HeaderMapExtended};

server/src/files/file.rs

@@ -7,11 +7,10 @@ use std::{
 use axum::{
     body::Body,
     extract,
-    http::HeaderMap,
+    http::{HeaderMap, StatusCode},
     response::{IntoResponse, Response},
 };
 use logger::{error, info};
-use reqwest::StatusCode;
 
 use crate::{
     log_result,

server/src/lib.rs

@@ -1,13 +1,12 @@
 use std::time::Instant;
 
 use api::{ApiRoutes, VecIdToIndexToVec};
-use axum::{routing::get, serve, Json, Router};
+use axum::{http::StatusCode, routing::get, serve, Json, Router};
 use color_eyre::owo_colors::OwoColorize;
 use computer::Computer;
 use files::FilesRoutes;
 use indexer::Indexer;
 use logger::{error, info};
-use reqwest::StatusCode;
 use storable_vec::STATELESS;
 use tokio::net::TcpListener;
 use tower_http::compression::CompressionLayer;
@@ -23,7 +22,7 @@ pub struct AppState {
     computer: &'static Computer<STATELESS>,
 }
 
-pub const WEBSITE_DEV_PATH: &str = "../website/";
+pub const WEBSITE_DEV_PATH: &str = "../websites/kibo.money/";
 
 pub async fn main(indexer: Indexer<STATELESS>, computer: Computer<STATELESS>) -> color_eyre::Result<()> {
     let indexer = Box::leak(Box::new(indexer));

server/src/traits/header_map.rs

@@ -1,9 +1,11 @@
 use std::{path::Path, time};
 
-use axum::http::{header, HeaderMap};
+use axum::http::{
+    header::{self, HOST, IF_MODIFIED_SINCE},
+    HeaderMap,
+};
 use jiff::{civil::DateTime, fmt::strtime, tz::TimeZone, Timestamp};
 use logger::info;
-use reqwest::header::{HOST, IF_MODIFIED_SINCE};
 
 const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish
 const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT";

server/src/traits/response.rs

@@ -1,5 +1,8 @@
-use axum::{body::Body, http::Response, response::IntoResponse};
-use reqwest::StatusCode;
+use axum::{
+    body::Body,
+    http::{Response, StatusCode},
+    response::IntoResponse,
+};
 
 use super::header_map::HeaderMapExtended;
 

storable_vec/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "storable_vec"
 description = "A very small, fast, efficient and simple storable Vec"
-version = "0.1.2"
-license = "MIT"
+version = "0.1.3"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [features]
 json = ["dep:serde", "dep:serde_json"]

storable_vec/src/traits/any.rs

@@ -43,6 +43,7 @@ pub trait AnyJsonStorableVec: AnyStorableVec {
     fn collect_range_values(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<serde_json::Value>>;
 }
 
+#[cfg(feature = "json")]
 impl<I, T, const MODE: u8> AnyJsonStorableVec for StorableVec<I, T, MODE>
 where
     I: StoredIndex,