general: snapshot

iterator/Cargo.toml

@@ -1,18 +1,18 @@
 [package]
 name = "biter"
 description = "A very fast Bitcoin block iterator built on top of bitcoin-rust"
-version = "0.2.2"
-license = "MIT"
+version = "0.2.3"
 repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"
 keywords = ["bitcoin", "block", "iterator"]
 categories = ["cryptography::cryptocurrencies", "encoding"]
-edition = "2021"
+edition = { workspace = true }
+license = { workspace = true }
 
 [dependencies]
 bitcoin = { workspace = true }
 rayon = { workspace = true }
 crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = "1.0.138"
+serde = { workspace = true }
+serde_json = { workspace = true }
 derive_deref = { workspace = true }
 bitcoincore-rpc = "0.19.0"

iterator/README.md

@@ -9,43 +9,7 @@ The element returned by the iterator is a tuple which includes the:
 
 ## Example
 
-```rust
-use std::path::Path;
-
-use bitcoincore_rpc::{Auth, Client};
-
-fn main() {
-    let i = std::time::Instant::now();
-
-    // Path to the Bitcoin data directory
-    let data_dir = "../../bitcoin";
-
-    // Inclusive starting height of the blocks received, `None` for 0
-    let start = Some(850_000);
-
-    // Inclusive ending height of the blocks received, `None` for the last one
-    let end = None;
-
-    // RPC client to filter out forks
-    let url = "http://localhost:8332";
-    let cookie = Path::new(data_dir).join(".cookie");
-    let auth = Auth::CookieFile(cookie);
-    let rpc = Client::new(url, auth).unwrap();
-
-    if cookie.is_file() {
-        Ok()
-
-    // Create channel receiver then iterate over the blocks
-    biter::new(data_dir, start, end, rpc)
-        .iter()
-        .for_each(|(height, _block, hash)| {
-            println!("{height}: {hash}");
-        });
-
-    dbg!(i.elapsed());
-}
-
-```
+`src/main.rs`
 
 ## Requirements
 

iterator/src/blk_index_to_blk_recap.rs

@@ -32,6 +32,8 @@ impl BlkIndexToBlkRecap {
             }
         };
 
+        // dbg!(&tree);
+
         let mut this = Self {
             path,
             tree,
@@ -43,7 +45,9 @@ impl BlkIndexToBlkRecap {
         this
     }
 
-    pub fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
+    fn clean_outdated(&mut self, blocks_dir: &BlkIndexToBlkPath) {
+        self.tree.pop_last();
+
         let mut unprocessed_keys = self.tree.keys().copied().collect::<BTreeSet<_>>();
 
         blocks_dir.iter().for_each(|(blk_index, blk_path)| {
@@ -59,22 +63,15 @@ impl BlkIndexToBlkRecap {
             self.tree.remove(&blk_index);
         });
 
-        while self.tree.last_entry().map(|last| *last.key()).is_some_and(|key| {
-            if key >= self.tree.len() {
-                self.tree.pop_last();
-                true
-            } else {
-                false
-            }
-        }) {}
-
         self.last_safe_height = self.tree.values().map(|recap| recap.height()).max();
     }
 
-    pub fn get_start_recap(&self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
+    pub fn get_start_recap(&mut self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
         if let Some(start) = start {
             let (last_key, last_value) = self.tree.last_key_value()?;
 
+            dbg!((last_key, last_value));
+
             if last_value.height() < start {
                 return Some((*last_key, *last_value));
             } else if let Some((blk_index, _)) =
@@ -82,7 +79,7 @@ impl BlkIndexToBlkRecap {
             {
                 if *blk_index != 0 {
                     // Temporary fix, need to rethink the whole thing
-                    let blk_index = (*blk_index).checked_sub(3).unwrap_or_default();
+                    let blk_index = (*blk_index).checked_sub(1).unwrap_or_default();
                     return Some((blk_index, *self.tree.get(&blk_index).unwrap()));
                 }
             }

iterator/src/blk_metadata.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use crate::path_to_modified_time;
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 pub struct BlkMetadata {
     pub index: usize,
     pub modified_time: u64,

iterator/src/blk_metadata_and_block.rs

@@ -2,6 +2,7 @@ use bitcoin::Block;
 
 use crate::BlkMetadata;
 
+#[derive(Debug)]
 pub struct BlkMetadataAndBlock {
     pub blk_metadata: BlkMetadata,
     pub block: Block,
@@ -9,9 +10,6 @@ pub struct BlkMetadataAndBlock {
 
 impl BlkMetadataAndBlock {
     pub fn new(blk_metadata: BlkMetadata, block: Block) -> Self {
-        Self {
-            blk_metadata,
-            block,
-        }
+        Self { blk_metadata, block }
     }
 }

iterator/src/blk_recap.rs

@@ -30,6 +30,9 @@ impl BlkRecap {
     }
 
     pub fn has_different_modified_time(&self, blk_path: &PathBuf) -> bool {
+        if self.modified_time != path_to_modified_time(blk_path) {
+            dbg!(self.modified_time, path_to_modified_time(blk_path));
+        }
         self.modified_time != path_to_modified_time(blk_path)
     }
 

iterator/src/lib.rs

@@ -36,11 +36,6 @@ pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 1000;
 const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
 const BOUND_CAP: usize = 210;
 
-enum BlockState {
-    Raw(Vec<u8>),
-    Decoded(Block),
-}
-
 ///
 /// Returns a crossbeam channel receiver that receives `(usize, Block, BlockHash)` tuples (with `usize` being the height) in sequential order.
 ///
@@ -82,7 +77,7 @@ pub fn new(
     data_dir: &Path,
     start: Option<usize>,
     end: Option<usize>,
-    rpc: bitcoincore_rpc::Client,
+    rpc: &'static bitcoincore_rpc::Client,
 ) -> Receiver<(usize, Block, BlockHash)> {
     let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
     let (send_block, recv_block) = bounded(BOUND_CAP);
@@ -225,6 +220,8 @@ pub fn new(
                               tuple: BlkMetadataAndBlock| {
             let mut tuple = Some(tuple);
 
+            println!("{} {} {}", recent_hashes.len(), recent_chain.len(), future_blocks.len(),);
+
             while let Some(tuple) = tuple.take().or_else(|| future_blocks.remove(prev_hash)) {
                 let hash = tuple.block.block_hash();
 
@@ -306,3 +303,8 @@ pub fn new(
 
     recv_height_block_hash
 }
+
+enum BlockState {
+    Raw(Vec<u8>),
+    Decoded(Block),
+}

iterator/src/main.rs

@@ -6,12 +6,15 @@ fn main() {
     let i = std::time::Instant::now();
 
     let data_dir = Path::new("../../bitcoin");
-    let url = "http://localhost:8332";
-    let cookie = Path::new(data_dir).join(".cookie");
-    let auth = Auth::CookieFile(cookie);
-    let rpc = Client::new(url, auth).unwrap();
+    let rpc = Box::leak(Box::new(
+        Client::new(
+            "http://localhost:8332",
+            Auth::CookieFile(Path::new(data_dir).join(".cookie")),
+        )
+        .unwrap(),
+    ));
 
-    let start = None;
+    let start = Some(460_001);
     let end = None;
 
     biter::new(data_dir, start, end, rpc)