diff --git a/Cargo.lock b/Cargo.lock index 29e6d4886..96a8b7669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,7 +262,7 @@ name = "bindexer" version = "0.1.0" dependencies = [ "bitcoin", - "biter", + "biterator", "cl0g", "color-eyre", "derive_deref", @@ -357,16 +357,18 @@ dependencies = [ ] [[package]] -name = "biter" +name = "biterator" version = "0.2.3" dependencies = [ "bitcoin", "bitcoincore-rpc", "crossbeam", "derive_deref", + "fjall", "rayon", "serde", "serde_json", + "zerocopy 0.8.18", ] [[package]] @@ -384,7 +386,7 @@ name = "bomputer" version = "0.1.0" dependencies = [ "bindexer", - "biter", + "biterator", "bricer", "color-eyre", "derive_deref", diff --git a/Cargo.toml b/Cargo.toml index 5271b3e7f..0cf5cc79f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,10 @@ derive_deref = "1.1.1" fjall = "2.6.3" hodor = { version = "0", path = "hodor" } indexer = { version = "0", path = "indexer", package = "bindexer" } -iterator = { version = "0", path = "iterator", package = "biter" } +iterator = { version = "0", path = "iterator", package = "biterator", features = [ + "fjall", + "zerocopy", +] } jiff = "0.2.1" log = { version = "0.4.25" } logger = { version = "0", path = "logger", package = "cl0g" } diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 6ecb1bb23..48219356a 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -78,12 +78,11 @@ impl Indexer { let mut idxs = starting_indexes; - iterator::new(bitcoin_dir, Some(idxs.height.into()), None, rpc) + iterator::new(bitcoin_dir, Some(idxs.height), None, rpc) .iter() .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); - let height = Height::from(height); idxs.height = height; let blockhash = BlockHash::from(blockhash); diff --git a/indexer/src/storage/fjalls/base.rs b/indexer/src/storage/fjalls/base.rs index 7f75d1af8..9b215b779 100644 --- a/indexer/src/storage/fjalls/base.rs +++ b/indexer/src/storage/fjalls/base.rs @@ -8,11 +8,10 @@ use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, TransactionalPartitionHandle, }; +use iterator::Height; use storable_vec::{Value, Version}; use zerocopy::{Immutable, IntoBytes}; -use crate::structs::Height; - use super::StoreMeta; pub struct Store { diff --git a/indexer/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs index adb76511f..c02b6cd7e 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/indexer/src/storage/storable_vecs/base.rs @@ -39,7 +39,7 @@ where self.vec.truncate_if_needed(index) } - pub fn height(&self) -> storable_vec::Result { + pub fn height(&self) -> iterator::Result { Height::try_from(self.path_height().as_path()) } fn path_height(&self) -> PathBuf { @@ -71,7 +71,7 @@ impl DerefMut for StorableVec { } pub trait AnyStorableVec: Send + Sync { - fn height(&self) -> storable_vec::Result; + fn height(&self) -> iterator::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } @@ -80,7 +80,7 @@ where I: StoredIndex, T: StoredType, { - fn height(&self) -> storable_vec::Result { + fn height(&self) -> iterator::Result { self.height() } diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index 170456860..572303bc6 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -1,15 +1,16 @@ use std::{fs, io, path::Path}; +use iterator::Height; use rayon::prelude::*; use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS}; use crate::{ structs::{ - Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height, LockTime, - Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, - P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, - P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, - Txinindex, Txoutindex, Unknownindex, Weight, + Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, LockTime, Multisigindex, + Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, + P2SHAddressBytes, P2SHindex, P2TRAddressBytes, P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, + P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, + Weight, }, Indexes, }; diff --git a/indexer/src/structs/blockhash.rs b/indexer/src/structs/blockhash.rs index 0bb44bd82..f8da62225 100644 --- a/indexer/src/structs/blockhash.rs +++ b/indexer/src/structs/blockhash.rs @@ -1,12 +1,13 @@ use std::mem; use derive_deref::Deref; -use iterator::rpc::{Client, RpcApi}; +use iterator::{ + rpc::{Client, RpcApi}, + Height, +}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Height; - #[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)] pub struct BlockHash([u8; 32]); diff --git a/indexer/src/structs/indexes.rs b/indexer/src/structs/indexes.rs index 2392ca4b5..02539dc06 100644 --- a/indexer/src/structs/indexes.rs +++ b/indexer/src/structs/indexes.rs @@ -1,13 +1,13 @@ use color_eyre::eyre::ContextCompat; -use iterator::rpc::Client; use iterator::NUMBER_OF_UNSAFE_BLOCKS; +use iterator::{rpc::Client, Height}; use storable_vec::CACHED_GETS; use crate::storage::{Fjalls, StorableVecs}; use super::{ - Addressindex, BlockHash, Emptyindex, Height, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, - P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, + Addressindex, BlockHash, Emptyindex, Multisigindex, Opreturnindex, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, + P2TRindex, P2WPKHindex, P2WSHindex, Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex, }; #[derive(Debug, Default)] diff --git a/indexer/src/structs/locktime.rs b/indexer/src/structs/locktime.rs index 58a2ea9f2..c0ba0826f 100644 --- a/indexer/src/structs/locktime.rs +++ b/indexer/src/structs/locktime.rs @@ -1,7 +1,8 @@ +use iterator::Height; use serde::Serialize; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; -use super::{Height, Timestamp}; +use super::Timestamp; #[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes, Serialize)] #[repr(C)] diff --git a/indexer/src/structs/mod.rs b/indexer/src/structs/mod.rs index e31c7bffc..9c64bd3f8 100644 --- a/indexer/src/structs/mod.rs +++ b/indexer/src/structs/mod.rs @@ -4,7 +4,6 @@ mod addresstype; mod addresstypeindex; mod blockhash; mod compressed; -mod height; mod indexes; mod locktime; mod sats; @@ -24,7 +23,6 @@ pub use addresstype::*; pub use addresstypeindex::*; pub use blockhash::*; pub use compressed::*; -pub use height::*; pub use indexes::*; pub use locktime::*; pub use sats::*; diff --git a/indexer/src/structs/sats.rs b/indexer/src/structs/sats.rs index eea28b04f..238e7cc3d 100644 --- a/indexer/src/structs/sats.rs +++ b/indexer/src/structs/sats.rs @@ -4,12 +4,10 @@ use std::{ }; use derive_deref::{Deref, DerefMut}; -use iterator::bitcoin::Amount; +use iterator::{bitcoin::Amount, Height}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::Height; - #[derive( Debug, PartialEq, diff --git a/iterator/Cargo.toml b/iterator/Cargo.toml index 81c42ef2a..1b95676f4 100644 --- a/iterator/Cargo.toml +++ b/iterator/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "biter" +name = "biterator" description = "A very fast Bitcoin block iterator built on top of bitcoin-rust" version = "0.2.3" repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter" @@ -8,11 +8,17 @@ categories = ["cryptography::cryptocurrencies", "encoding"] edition = { workspace = true } license = { workspace = true } +[features] +fjall = ["dep:fjall"] +zerocopy = ["dep:zerocopy"] + [dependencies] bitcoin = { workspace = true } rayon = { workspace = true } crossbeam = { version = "0.8.4", features = ["crossbeam-channel"] } +fjall = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } derive_deref = { workspace = true } bitcoincore-rpc = "0.19.0" +zerocopy = { workspace = true, optional = true } diff --git a/iterator/src/blk_index_to_blk_recap.rs b/iterator/src/blk_index_to_blk_recap.rs index 9aaf1b950..a03025a67 100644 --- a/iterator/src/blk_index_to_blk_recap.rs +++ b/iterator/src/blk_index_to_blk_recap.rs @@ -6,7 +6,7 @@ use std::{ path::{Path, PathBuf}, }; -use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock}; +use crate::{blk_recap::BlkRecap, BlkIndexToBlkPath, BlkMetadataAndBlock, Height}; const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30; @@ -14,7 +14,7 @@ const TARGET_BLOCKS_PER_MONTH: usize = 144 * 30; pub struct BlkIndexToBlkRecap { path: PathBuf, tree: BTreeMap, - last_safe_height: Option, + last_safe_height: Option, } impl BlkIndexToBlkRecap { @@ -66,7 +66,7 @@ impl BlkIndexToBlkRecap { self.last_safe_height = self.tree.values().map(|recap| recap.height()).max(); } - pub fn get_start_recap(&mut self, start: Option) -> Option<(usize, BlkRecap)> { + pub fn get_start_recap(&mut self, start: Option) -> Option<(usize, BlkRecap)> { if let Some(start) = start { let (last_key, last_value) = self.tree.last_key_value()?; @@ -88,7 +88,7 @@ impl BlkIndexToBlkRecap { None } - pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: usize) { + pub fn update(&mut self, blk_metadata_and_block: &BlkMetadataAndBlock, height: Height) { let blk_index = blk_metadata_and_block.blk_metadata.index; if let Some(last_entry) = self.tree.last_entry() { diff --git a/iterator/src/blk_recap.rs b/iterator/src/blk_recap.rs index f1a7f9cb9..bad813830 100644 --- a/iterator/src/blk_recap.rs +++ b/iterator/src/blk_recap.rs @@ -3,11 +3,11 @@ use std::path::PathBuf; use bitcoin::{hashes::Hash, BlockHash}; use serde::{Deserialize, Serialize}; -use crate::{path_to_modified_time, BlkMetadataAndBlock}; +use crate::{path_to_modified_time, BlkMetadataAndBlock, Height}; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct BlkRecap { - min_continuous_height: usize, + min_continuous_height: Height, min_continuous_prev_hash: BlockHash, modified_time: u64, } @@ -15,13 +15,13 @@ pub struct BlkRecap { impl BlkRecap { pub fn first(blk_metadata_and_block: &BlkMetadataAndBlock) -> Self { Self { - min_continuous_height: 0, + min_continuous_height: Height::default(), min_continuous_prev_hash: BlockHash::all_zeros(), modified_time: blk_metadata_and_block.blk_metadata.modified_time, } } - pub fn from(height: usize, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self { + pub fn from(height: Height, blk_metadata_and_block: &BlkMetadataAndBlock) -> Self { Self { min_continuous_height: height, min_continuous_prev_hash: blk_metadata_and_block.block.header.prev_blockhash, @@ -36,11 +36,11 @@ impl BlkRecap { self.modified_time != path_to_modified_time(blk_path) } - pub fn is_younger_than(&self, height: usize) -> bool { + pub fn is_younger_than(&self, height: Height) -> bool { self.min_continuous_height > height } - pub fn height(&self) -> usize { + pub fn height(&self) -> Height { self.min_continuous_height } diff --git a/iterator/src/error.rs b/iterator/src/error.rs new file mode 100644 index 000000000..0a7562c65 --- /dev/null +++ b/iterator/src/error.rs @@ -0,0 +1,35 @@ +use std::{ + fmt::{self, Debug}, + io, +}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub enum Error { + IO(io::Error), + ZeroCopyError, +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::IO(value) + } +} + +impl From> for Error { + fn from(_: zerocopy::error::SizeError) -> Self { + Self::ZeroCopyError + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::IO(error) => Debug::fmt(&error, f), + Error::ZeroCopyError => write!(f, "Zero copy convert error"), + } + } +} + +impl std::error::Error for Error {} diff --git a/indexer/src/structs/height.rs b/iterator/src/height.rs similarity index 85% rename from indexer/src/structs/height.rs rename to iterator/src/height.rs index e19631fd3..8104cb05b 100644 --- a/indexer/src/structs/height.rs +++ b/iterator/src/height.rs @@ -1,32 +1,21 @@ use std::{ - fmt, fs, io, + fmt::{self, Debug}, + fs, io, ops::{Add, AddAssign, Rem, Sub}, path::Path, }; use derive_deref::{Deref, DerefMut}; -use fjall::Slice; -use iterator::rpc::{self, RpcApi}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive( - Debug, - Clone, - Copy, - Deref, - DerefMut, - PartialEq, - Eq, - PartialOrd, - Ord, - Default, - FromBytes, - Immutable, - IntoBytes, - KnownLayout, - Serialize, -)] +use crate::{ + rpc::{self, RpcApi}, + Error, +}; + +#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "zerocopy", derive(FromBytes, Immutable, IntoBytes, KnownLayout))] pub struct Height(u32); impl Height { @@ -157,13 +146,6 @@ impl From for u64 { } } -impl TryFrom<&Path> for Height { - type Error = storable_vec::Error; - fn try_from(value: &Path) -> Result { - Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned()) - } -} - impl TryFrom<&rpc::Client> for Height { type Error = rpc::Error; fn try_from(value: &rpc::Client) -> Result { @@ -171,18 +153,6 @@ impl TryFrom<&rpc::Client> for Height { } } -impl TryFrom for Height { - type Error = storable_vec::Error; - fn try_from(value: Slice) -> Result { - Ok(Self::read_from_bytes(&value)?) - } -} -impl From for Slice { - fn from(value: Height) -> Self { - Self::new(value.as_bytes()) - } -} - impl From for Height { fn from(value: bitcoin::locktime::absolute::Height) -> Self { Self(value.to_consensus_u32()) @@ -194,3 +164,25 @@ impl From for bitcoin::locktime::absolute::Height { bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap() } } + +#[cfg(feature = "zerocopy")] +impl TryFrom<&Path> for Height { + type Error = Error; + fn try_from(value: &Path) -> Result { + Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned()) + } +} + +#[cfg(feature = "fjall")] +impl TryFrom for Height { + type Error = Error; + fn try_from(value: fjall::Slice) -> Result { + Ok(Self::read_from_bytes(&value)?) + } +} +#[cfg(feature = "fjall")] +impl From for fjall::Slice { + fn from(value: Height) -> Self { + Self::new(value.as_bytes()) + } +} diff --git a/iterator/src/lib.rs b/iterator/src/lib.rs index 01f0433e9..7ebcb5faf 100644 --- a/iterator/src/lib.rs +++ b/iterator/src/lib.rs @@ -25,11 +25,15 @@ mod blk_index_to_blk_recap; mod blk_metadata; mod blk_metadata_and_block; mod blk_recap; +mod error; +mod height; mod utils; use blk_index_to_blk_recap::*; use blk_metadata::*; use blk_metadata_and_block::*; +pub use error::*; +pub use height::*; use utils::*; pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 1000; @@ -75,10 +79,10 @@ const BOUND_CAP: usize = 210; /// pub fn new( data_dir: &Path, - start: Option, - end: Option, + start: Option, + end: Option, rpc: &'static bitcoincore_rpc::Client, -) -> Receiver<(usize, Block, BlockHash)> { +) -> Receiver<(Height, Block, BlockHash)> { let (send_block_reader, recv_block_reader) = bounded(BOUND_CAP); let (send_block, recv_block) = bounded(BOUND_CAP); let (send_height_block_hash, recv_height_block_hash) = bounded(BOUND_CAP); @@ -189,7 +193,7 @@ pub fn new( }); thread::spawn(move || { - let mut height = start_recap.map_or(0, |(_, recap)| recap.height()); + let mut height = start_recap.map_or(Height::default(), |(_, recap)| recap.height()); let mut future_blocks = BTreeMap::default(); let mut recent_chain: VecDeque<(BlockHash, BlkMetadataAndBlock)> = VecDeque::default(); diff --git a/iterator/src/main.rs b/iterator/src/main.rs index 1a7ff466c..09792ef49 100644 --- a/iterator/src/main.rs +++ b/iterator/src/main.rs @@ -14,10 +14,10 @@ fn main() { .unwrap(), )); - let start = Some(460_001); + let start = Some(460_001_u32.into()); let end = None; - biter::new(data_dir, start, end, rpc) + biterator::new(data_dir, start, end, rpc) .iter() .for_each(|(height, _block, hash)| { println!("{height}: {hash}"); diff --git a/storable_vec/src/enums/error.rs b/storable_vec/src/enums/error.rs index 8db892c0d..b066a4ece 100644 --- a/storable_vec/src/enums/error.rs +++ b/storable_vec/src/enums/error.rs @@ -13,7 +13,6 @@ pub enum Error { DifferentVersion { found: Version, expected: Version }, MmapsVecIsTooSmall, IO(io::Error), - // UnsafeSliceSerde(zerocopy::error::), ZeroCopyError, IndexTooHigh, IndexTooLow, @@ -51,7 +50,6 @@ impl fmt::Display for Error { } Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), Error::IO(error) => Debug::fmt(&error, f), - // Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), Error::IndexTooHigh => write!(f, "Index too high"), Error::IndexTooLow => write!(f, "Index too low"), Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),