indexer: moved height to iterator

Cargo.lock

@@ -262,7 +262,7 @@ name = "bindexer"
 version = "0.1.0"
 dependencies = [
  "bitcoin",
- "biter",
+ "biterator",
  "cl0g",
  "color-eyre",
  "derive_deref",
@@ -357,16 +357,18 @@ dependencies = [
 ]
 
 [[package]]
-name = "biter"
+name = "biterator"
 version = "0.2.3"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
  "crossbeam",
  "derive_deref",
+ "fjall",
  "rayon",
  "serde",
  "serde_json",
+ "zerocopy 0.8.18",
 ]
 
 [[package]]
@@ -384,7 +386,7 @@ name = "bomputer"
 version = "0.1.0"
 dependencies = [
  "bindexer",
- "biter",
+ "biterator",
  "bricer",
  "color-eyre",
  "derive_deref",

Cargo.toml

@@ -22,7 +22,10 @@ derive_deref = "1.1.1"
 fjall = "2.6.3"
 hodor = { version = "0", path = "hodor" }
 indexer = { version = "0", path = "indexer", package = "bindexer" }
-iterator = { version = "0", path = "iterator", package = "biter" }
+iterator = { version = "0", path = "iterator", package = "biterator", features = [
+    "fjall",
+    "zerocopy",
+] }
 jiff = "0.2.1"
 log = { version = "0.4.25" }
 logger = { version = "0", path = "logger", package = "cl0g" }

indexer/src/lib.rs

@@ -78,12 +78,11 @@ impl Indexer<CACHED_GETS> {
 
         let mut idxs = starting_indexes;
 
-        iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc)
+        iterator::new(bitcoin_dir, Some(idxs.height), None, rpc)
             .iter()
             .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> {
                 info!("Indexing block {height}...");
 
-                let height = Height::from(height);
                 idxs.height = height;
 
                 let blockhash = BlockHash::from(blockhash);

indexer/src/storage/fjalls/base.rs

@@ -8,11 +8,10 @@ use fjall::{
     PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
     TransactionalPartitionHandle,
 };
+use iterator::Height;
 use storable_vec::{Value, Version};
 use zerocopy::{Immutable, IntoBytes};
 
-use crate::structs::Height;
-
 use super::StoreMeta;
 
 pub struct Store<Key, Value> {

indexer/src/storage/storable_vecs/base.rs

@@ -39,7 +39,7 @@ where
         self.vec.truncate_if_needed(index)
     }
 
-    pub fn height(&self) -> storable_vec::Result<Height> {
+    pub fn height(&self) -> iterator::Result<Height> {
         Height::try_from(self.path_height().as_path())
     }
     fn path_height(&self) -> PathBuf {
@@ -71,7 +71,7 @@ impl<I, T, const MODE: u8> DerefMut for StorableVec<I, T, MODE> {
 }
 
 pub trait AnyStorableVec: Send + Sync {
-    fn height(&self) -> storable_vec::Result<Height>;
+    fn height(&self) -> iterator::Result<Height>;
     fn flush(&mut self, height: Height) -> io::Result<()>;
 }
 
@@ -80,7 +80,7 @@ where
     I: StoredIndex,
     T: StoredType,
 {
-    fn height(&self) -> storable_vec::Result<Height> {
+    fn height(&self) -> iterator::Result<Height> {
         self.height()
     }
 

indexer/src/storage/storable_vecs/mod.rs

@@ -1,15 +1,16 @@
 use std::{fs, io, path::Path};
 
+use iterator::Height;
 use rayon::prelude::*;
 use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS};
 
 use crate::{
     structs::{
-        Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime,
-        Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index,
-        P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes,
-        P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex,
-        Txinindex, Txoutindex, Unknownindex, Weight,
+        Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, LockTime, Multisigindex,
+        Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex,
+        P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes,
+        P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex,
+        Weight,
     },
     Indexes,
 };

indexer/src/structs/blockhash.rs

@@ -1,12 +1,13 @@
 use std::mem;
 
 use derive_deref::Deref;
-use iterator::rpc::{Client, RpcApi};
+use iterator::{
+    rpc::{Client, RpcApi},
+    Height,
+};
 use serde::Serialize;
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use super::Height;
-
 #[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
 pub struct BlockHash([u8; 32]);
 

indexer/src/structs/indexes.rs

@@ -1,13 +1,13 @@
 use color_eyre::eyre::ContextCompat;
-use iterator::rpc::Client;
 use iterator::NUMBER_OF_UNSAFE_BLOCKS;
+use iterator::{rpc::Client, Height};
 use storable_vec::CACHED_GETS;
 
 use crate::storage::{Fjalls, StorableVecs};
 
 use super::{
-    Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex,
-    P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
+    Addressindex, BlockHash, Emptyindex, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, P2SHindex,
+    P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
 };
 
 #[derive(Debug, Default)]

indexer/src/structs/locktime.rs

@@ -1,7 +1,8 @@
+use iterator::Height;
 use serde::Serialize;
 use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
 
-use super::{Height, Timestamp};
+use super::Timestamp;
 
 #[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)]
 #[repr(C)]

indexer/src/structs/mod.rs

@@ -4,7 +4,6 @@ mod addresstype;
 mod addresstypeindex;
 mod blockhash;
 mod compressed;
-mod height;
 mod indexes;
 mod locktime;
 mod sats;
@@ -24,7 +23,6 @@ pub use addresstype::*;
 pub use addresstypeindex::*;
 pub use blockhash::*;
 pub use compressed::*;
-pub use height::*;
 pub use indexes::*;
 pub use locktime::*;
 pub use sats::*;

indexer/src/structs/sats.rs

@@ -4,12 +4,10 @@ use std::{
 };
 
 use derive_deref::{Deref, DerefMut};
-use iterator::bitcoin::Amount;
+use iterator::{bitcoin::Amount, Height};
 use serde::Serialize;
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-use super::Height;
-
 #[derive(
     Debug,
     PartialEq,

iterator/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "biter"
+name = "biterator"
 description = "A very fast Bitcoin block iterator built on top of bitcoin-rust"
 version = "0.2.3"
 repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter"
@@ -8,11 +8,17 @@ categories = ["cryptography::cryptocurrencies", "encoding"]
 edition = { workspace = true }
 license = { workspace = true }
 
+[features]
+fjall = ["dep:fjall"]
+zerocopy = ["dep:zerocopy"]
+
 [dependencies]
 bitcoin = { workspace = true }
 rayon = { workspace = true }
 crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] }
+fjall = { workspace = true, optional = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 derive_deref = { workspace = true }
 bitcoincore-rpc = "0.19.0"
+zerocopy = { workspace = true, optional = true }

iterator/src/blk_index_to_blk_recap.rs

@@ -6,7 +6,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock};
+use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock, Height};
 
 const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30;
 
@@ -14,7 +14,7 @@ const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30;
 pub struct BlkIndexToBlkRecap {
     path: PathBuf,
     tree: BTreeMap<usize, BlkRecap>,
-    last_safe_height: Option<usize>,
+    last_safe_height: Option<Height>,
 }
 
 impl BlkIndexToBlkRecap {
@@ -66,7 +66,7 @@ impl BlkIndexToBlkRecap {
         self.last_safe_height = self.tree.values().map(|recap| recap.height()).max();
     }
 
-    pub fn get_start_recap(&mut self, start: Option<usize>) -> Option<(usize, BlkRecap)> {
+    pub fn get_start_recap(&mut self, start: Option<Height>) -> Option<(usize, BlkRecap)> {
         if let Some(start) = start {
             let (last_key, last_value) = self.tree.last_key_value()?;
 
@@ -88,7 +88,7 @@ impl BlkIndexToBlkRecap {
         None
     }
 
-    pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) {
+    pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: Height) {
         let blk_index = blk_metadata_and_block.blk_metadata.index;
 
         if let Some(last_entry) = self.tree.last_entry() {

iterator/src/blk_recap.rs

@@ -3,11 +3,11 @@ use std::path::PathBuf;
 use bitcoin::{hashes::Hash, BlockHash};
 use serde::{Deserialize, Serialize};
 
-use crate::{path_to_modified_time, BlkMetadataAndBlock};
+use crate::{path_to_modified_time, BlkMetadataAndBlock, Height};
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub struct BlkRecap {
-    min_continuous_height: usize,
+    min_continuous_height: Height,
     min_continuous_prev_hash: BlockHash,
     modified_time: u64,
 }
@@ -15,13 +15,13 @@ pub struct BlkRecap {
 impl BlkRecap {
     pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
         Self {
-            min_continuous_height: 0,
+            min_continuous_height: Height::default(),
             min_continuous_prev_hash: BlockHash::all_zeros(),
             modified_time: blk_metadata_and_block.blk_metadata.modified_time,
         }
     }
 
-    pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
+    pub fn from(height: Height, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self {
         Self {
             min_continuous_height: height,
             min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash,
@@ -36,11 +36,11 @@ impl BlkRecap {
         self.modified_time != path_to_modified_time(blk_path)
     }
 
-    pub fn is_younger_than(&self, height: usize) -> bool {
+    pub fn is_younger_than(&self, height: Height) -> bool {
         self.min_continuous_height > height
     }
 
-    pub fn height(&self) -> usize {
+    pub fn height(&self) -> Height {
         self.min_continuous_height
     }
 

iterator/src/error.rs

@@ -0,0 +1,35 @@
+use std::{
+    fmt::{self, Debug},
+    io,
+};
+
+pub type Result<T, E = Error> = std::result::Result<T, E>;
+
+#[derive(Debug)]
+pub enum Error {
+    IO(io::Error),
+    ZeroCopyError,
+}
+
+impl From<io::Error> for Error {
+    fn from(value: io::Error) -> Self {
+        Self::IO(value)
+    }
+}
+
+impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
+    fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
+        Self::ZeroCopyError
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Error::IO(error) => Debug::fmt(&error, f),
+            Error::ZeroCopyError => write!(f, "Zero copy convert error"),
+        }
+    }
+}
+
+impl std::error::Error for Error {}

iterator/src/height.rs

@@ -1,32 +1,21 @@
 use std::{
-    fmt, fs, io,
+    fmt::{self, Debug},
+    fs, io,
     ops::{Add, AddAssign, Rem, Sub},
     path::Path,
 };
 
 use derive_deref::{Deref, DerefMut};
-use fjall::Slice;
-use iterator::rpc::{self, RpcApi};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    Deref,
-    DerefMut,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Default,
-    FromBytes,
-    Immutable,
-    IntoBytes,
-    KnownLayout,
-    Serialize,
-)]
+use crate::{
+    rpc::{self, RpcApi},
+    Error,
+};
+
+#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)]
+#[cfg_attr(feature = "zerocopy", derive(FromBytes, Immutable, IntoBytes, KnownLayout))]
 pub struct Height(u32);
 
 impl Height {
@@ -157,13 +146,6 @@ impl From<Height> for u64 {
     }
 }
 
-impl TryFrom<&Path> for Height {
-    type Error = storable_vec::Error;
-    fn try_from(value: &Path) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned())
-    }
-}
-
 impl TryFrom<&rpc::Client> for Height {
     type Error = rpc::Error;
     fn try_from(value: &rpc::Client) -> Result<Self, Self::Error> {
@@ -171,18 +153,6 @@ impl TryFrom<&rpc::Client> for Height {
     }
 }
 
-impl TryFrom<Slice> for Height {
-    type Error = storable_vec::Error;
-    fn try_from(value: Slice) -> Result<Self, Self::Error> {
-        Ok(Self::read_from_bytes(&value)?)
-    }
-}
-impl From<Height> for Slice {
-    fn from(value: Height) -> Self {
-        Self::new(value.as_bytes())
-    }
-}
-
 impl From<bitcoin::locktime::absolute::Height> for Height {
     fn from(value: bitcoin::locktime::absolute::Height) -> Self {
         Self(value.to_consensus_u32())
@@ -194,3 +164,25 @@ impl From<Height> for bitcoin::locktime::absolute::Height {
         bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap()
     }
 }
+
+#[cfg(feature = "zerocopy")]
+impl TryFrom<&Path> for Height {
+    type Error = Error;
+    fn try_from(value: &Path) -> Result<Self, Self::Error> {
+        Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned())
+    }
+}
+
+#[cfg(feature = "fjall")]
+impl TryFrom<fjall::Slice> for Height {
+    type Error = Error;
+    fn try_from(value: fjall::Slice) -> Result<Self, Self::Error> {
+        Ok(Self::read_from_bytes(&value)?)
+    }
+}
+#[cfg(feature = "fjall")]
+impl From<Height> for fjall::Slice {
+    fn from(value: Height) -> Self {
+        Self::new(value.as_bytes())
+    }
+}

iterator/src/lib.rs

@@ -25,11 +25,15 @@ mod blk_index_to_blk_recap;
 mod blk_metadata;
 mod blk_metadata_and_block;
 mod blk_recap;
+mod error;
+mod height;
 mod utils;
 
 use blk_index_to_blk_recap::*;
 use blk_metadata::*;
 use blk_metadata_and_block::*;
+pub use error::*;
+pub use height::*;
 use utils::*;
 
 pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 1000;
@@ -75,10 +79,10 @@ const BOUND_CAP: usize = 210;
 ///
 pub fn new(
     data_dir: &Path,
-    start: Option<usize>,
-    end: Option<usize>,
+    start: Option<Height>,
+    end: Option<Height>,
     rpc: &'static bitcoincore_rpc::Client,
-) -> Receiver<(usize, Block, BlockHash)> {
+) -> Receiver<(Height, Block, BlockHash)> {
     let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP);
     let (send_block, recv_block) = bounded(BOUND_CAP);
     let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP);
@@ -189,7 +193,7 @@ pub fn new(
     });
 
     thread::spawn(move || {
-        let mut height = start_recap.map_or(0, |(_, recap)| recap.height());
+        let mut height = start_recap.map_or(Height::default(), |(_, recap)| recap.height());
 
         let mut future_blocks = BTreeMap::default();
         let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default();

iterator/src/main.rs

@@ -14,10 +14,10 @@ fn main() {
         .unwrap(),
     ));
 
-    let start = Some(460_001);
+    let start = Some(460_001_u32.into());
     let end = None;
 
-    biter::new(data_dir, start, end, rpc)
+    biterator::new(data_dir, start, end, rpc)
         .iter()
         .for_each(|(height, _block, hash)| {
             println!("{height}: {hash}");

storable_vec/src/enums/error.rs

@@ -13,7 +13,6 @@ pub enum Error {
     DifferentVersion { found: Version, expected: Version },
     MmapsVecIsTooSmall,
     IO(io::Error),
-    // UnsafeSliceSerde(zerocopy::error::),
     ZeroCopyError,
     IndexTooHigh,
     IndexTooLow,
@@ -51,7 +50,6 @@ impl fmt::Display for Error {
             }
             Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
             Error::IO(error) => Debug::fmt(&error, f),
-            // Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
             Error::IndexTooHigh => write!(f, "Index too high"),
             Error::IndexTooLow => write!(f, "Index too low"),
             Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),