diff --git a/Cargo.lock b/Cargo.lock index 918eb85d9..eda8ab27a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ dependencies = [ "rapidhash", "rayon", "storable_vec", - "unsafe_slice_serde", + "zerocopy 0.8.16", ] [[package]] @@ -192,7 +192,7 @@ dependencies = [ "fjall", "jiff", "storable_vec", - "unsafe_slice_serde", + "zerocopy 0.8.16", ] [[package]] @@ -721,7 +721,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -945,7 +945,7 @@ name = "storable_vec" version = "0.1.2" dependencies = [ "memmap2", - "unsafe_slice_serde", + "zerocopy 0.8.16", ] [[package]] @@ -1061,10 +1061,6 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" -[[package]] -name = "unsafe_slice_serde" -version = "0.1.0" - [[package]] name = "valuable" version = "0.1.1" @@ -1185,7 +1181,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b8c07a70861ce02bad1607b5753ecb2501f67847b9f9ada7c160fff0ec6300c" +dependencies = [ + "zerocopy-derive 0.8.16", ] [[package]] @@ -1198,3 +1203,14 @@ dependencies = [ "quote", "syn 2.0.98", ] + +[[package]] +name = "zerocopy-derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] diff --git a/Cargo.toml b/Cargo.toml index 1d166cb17..c069e0bee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ members = [ "iterator", "storable_vec", "struct_iterable", - "unsafe_slice_serde", ] resolver = "2" @@ -23,5 +22,4 @@ jiff = "0.1.29" rayon = "1.10.0" storable_vec = { path = "storable_vec" } struct_iterable = { path = "struct_iterable" } -unsafe_slice_serde = { path = "unsafe_slice_serde" } -zerocopy = "0.8.15" +zerocopy = { version = "0.8.15", features = ["derive"] } diff --git a/computer/Cargo.toml b/computer/Cargo.toml index b1ac43a13..5ff3bb781 100644 --- a/computer/Cargo.toml +++ b/computer/Cargo.toml @@ -12,4 +12,4 @@ exit = { workspace = true } fjall = { workspace = true } jiff = { workspace = true } storable_vec = { workspace = true } -unsafe_slice_serde = { workspace = true } +zerocopy = { workspace = true } diff --git a/computer/src/lib.rs b/computer/src/lib.rs index 0d4a788d8..ff656d32c 100644 --- a/computer/src/lib.rs +++ b/computer/src/lib.rs @@ -55,7 +55,7 @@ impl Computer { .txindex_to_last_txinindex .compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?; - self.vecs.txindex_to_inputcount.compute_count_from_indexes( + self.vecs.txindex_to_inputs_count.compute_count_from_indexes( &mut indexer.vecs.txindex_to_first_txinindex, &mut self.vecs.txindex_to_last_txinindex, )?; @@ -64,14 +64,16 @@ impl Computer { .txindex_to_last_txoutindex .compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?; - self.vecs.txindex_to_outputcount.compute_count_from_indexes( + self.vecs.txindex_to_outputs_count.compute_count_from_indexes( &mut indexer.vecs.txindex_to_first_txoutindex, &mut self.vecs.txindex_to_last_txoutindex, )?; self.vecs .height_to_date - .compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| Date::from(timestamp))?; + .compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| { + Date::from(*timestamp) + })?; self.vecs .height_to_last_txindex @@ -82,6 +84,16 @@ impl Computer { &mut self.vecs.height_to_last_txindex, )?; + self.vecs.txindex_to_is_coinbase.compute_is_first_ordered( + &mut self.vecs.txindex_to_height, + &mut indexer.vecs.height_to_first_txindex, + )?; + + self.vecs.txindex_to_fee.compute_transform( + &mut self.vecs.txindex_to_height, + &mut indexer.vecs.height_to_first_txindex, + )?; + let date_count = self.vecs.height_to_date.len(); self.vecs diff --git a/computer/src/storage/fjalls.rs b/computer/src/storage/fjalls.rs index aba352bc6..9ba82cd4a 100644 --- a/computer/src/storage/fjalls.rs +++ b/computer/src/storage/fjalls.rs @@ -1,6 +1,7 @@ use std::path::Path; -use bindex::{Store, Version}; +use bindex::Store; +use storable_vec::Version; use crate::structs::{AddressindexTxoutindex, Unit}; diff --git a/computer/src/storage/storable_vecs.rs b/computer/src/storage/storable_vecs.rs new file mode 100644 index 000000000..0a78610fd --- /dev/null +++ b/computer/src/storage/storable_vecs.rs @@ -0,0 +1,95 @@ +use std::{fs, path::Path}; + +use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex}; +use storable_vec::{StorableVec, Version}; + +use crate::structs::{Date, Feerate}; + +// mod base; + +// use base::*; + +pub struct StorableVecs { + pub date_to_first_height: StorableVec, + // pub height_to_block_interval: StorableVec, + pub height_to_date: StorableVec, + // pub height_to_fee: StorableVec, + // pub height_to_inputcount: StorableVec, + // pub height_to_last_addressindex: StorableVec, + pub height_to_last_txindex: StorableVec, + // pub height_to_last_txoutindex: StorableVec, + // pub height_to_maxfeerate: StorableVec, + // pub height_to_medianfeerate: StorableVec, + // pub height_to_minfeerate: StorableVec, + // pub height_to_outputcount: StorableVec, + // pub height_to_subsidy: StorableVec, + // pub height_to_totalfees: StorableVec, + // pub height_to_txcount: StorableVec, + pub txindex_to_fee: StorableVec, + pub txindex_to_height: StorableVec, + pub txindex_to_is_coinbase: StorableVec, + // pub txindex_to_feerate: StorableVec, + pub txindex_to_inputs_count: StorableVec, + pub txindex_to_inputs_sum: StorableVec, + pub txindex_to_last_txinindex: StorableVec, + pub txindex_to_last_txoutindex: StorableVec, + pub txindex_to_outputs_count: StorableVec, + pub txindex_to_outputs_sum: StorableVec, +} + +impl StorableVecs { + pub fn import(path: &Path) -> color_eyre::Result { + fs::create_dir_all(path)?; + + Ok(Self { + date_to_first_height: StorableVec::forced_import(&path.join("date_to_first_height"), Version::from(1))?, + // height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?, + height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?, + // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?, + // height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?, + // height_to_last_addressindex: StorableVec::forced_import( + // &path.join("height_to_last_addressindex"), + // Version::from(1), + // )?, + height_to_last_txindex: StorableVec::forced_import(&path.join("height_to_last_txindex"), Version::from(1))?, + // height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?, + // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?, + // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?, + // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?, + // height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?, + // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?, + // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?, + // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?, + txindex_to_fee: StorableVec::forced_import(&path.join("txindex_to_fee"), Version::from(1))?, + txindex_to_height: StorableVec::forced_import(&path.join("txindex_to_height"), Version::from(1))?, + txindex_to_is_coinbase: StorableVec::forced_import(&path.join("txindex_to_is_coinbase"), Version::from(1))?, + // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?, + txindex_to_inputs_count: StorableVec::forced_import( + &path.join("txindex_to_inputs_count"), + Version::from(1), + )?, + txindex_to_inputs_sum: StorableVec::forced_import(&path.join("txindex_to_inputs_sum"), Version::from(1))?, + txindex_to_last_txinindex: StorableVec::forced_import( + &path.join("txindex_to_last_txinindex"), + Version::from(1), + )?, + txindex_to_last_txoutindex: StorableVec::forced_import( + &path.join("txindex_to_last_txoutindex"), + Version::from(1), + )?, + txindex_to_outputs_count: StorableVec::forced_import( + &path.join("txindex_to_outputs_count"), + Version::from(1), + )?, + txindex_to_outputs_sum: StorableVec::forced_import(&path.join("txindex_to_outputs_sum"), Version::from(1))?, + }) + } + + // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] { + // [&self.date_to_first_height] + // } + + // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] { + // [&mut self.date_to_first_height] + // } +} diff --git a/computer/src/storage/storable_vecs/base.rs b/computer/src/storage/storable_vecs/base.rs deleted file mode 100644 index 11cbbcee3..000000000 --- a/computer/src/storage/storable_vecs/base.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::{fmt::Debug, path::Path}; - -use derive_deref::{Deref, DerefMut}; -use storable_vec::{StorableVecIndex, StorableVecType, Version}; - -#[derive(Debug, Deref, DerefMut)] -pub struct StorableVec(storable_vec::StorableVec); - -impl StorableVec -where - I: StorableVecIndex, - T: StorableVecType, -{ - pub fn import(path: &Path, version: Version) -> storable_vec::Result { - Ok(Self(storable_vec::StorableVec::forced_import(path, version)?)) - } -} diff --git a/computer/src/storage/storable_vecs/mod.rs b/computer/src/storage/storable_vecs/mod.rs deleted file mode 100644 index a63fdbc2c..000000000 --- a/computer/src/storage/storable_vecs/mod.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::{fs, path::Path}; - -use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex}; -use storable_vec::Version; - -use crate::structs::{Date, Feerate}; - -mod base; - -use base::*; - -pub struct StorableVecs { - pub date_to_first_height: StorableVec, - // pub height_to_block_interval: StorableVec, - pub height_to_date: StorableVec, - // pub height_to_fee: StorableVec, - // pub height_to_inputcount: StorableVec, - // pub height_to_last_addressindex: StorableVec, - pub height_to_last_txindex: StorableVec, - // pub height_to_last_txoutindex: StorableVec, - // pub height_to_maxfeerate: StorableVec, - // pub height_to_medianfeerate: StorableVec, - // pub height_to_minfeerate: StorableVec, - // pub height_to_outputcount: StorableVec, - // pub height_to_subsidy: StorableVec, - // pub height_to_totalfees: StorableVec, - // pub height_to_txcount: StorableVec, - pub txindex_to_fee: StorableVec, - pub txindex_to_height: StorableVec, - pub txindex_to_is_coinbase: StorableVec, - // pub txindex_to_feerate: StorableVec, - pub txindex_to_inputcount: StorableVec, - pub txindex_to_last_txinindex: StorableVec, - pub txindex_to_last_txoutindex: StorableVec, - pub txindex_to_outputcount: StorableVec, -} - -impl StorableVecs { - pub fn import(path: &Path) -> color_eyre::Result { - fs::create_dir_all(path)?; - - Ok(Self { - date_to_first_height: StorableVec::import(&path.join("date_to_first_height"), Version::from(1))?, - // height_to_block_interval: StorableVec::import(&path.join("height_to_block_interval"), Version::from(1))?, - height_to_date: StorableVec::import(&path.join("height_to_date"), Version::from(1))?, - // height_to_fee: StorableVec::import(&path.join("height_to_fee"), Version::from(1))?, - // height_to_inputcount: StorableVec::import(&path.join("height_to_inputcount"), Version::from(1))?, - // height_to_last_addressindex: StorableVec::import( - // &path.join("height_to_last_addressindex"), - // Version::from(1), - // )?, - height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?, - // height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?, - // height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?, - // height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?, - // height_to_minfeerate: StorableVec::import(&path.join("height_to_minfeerate"), Version::from(1))?, - // height_to_outputcount: StorableVec::import(&path.join("height_to_outputcount"), Version::from(1))?, - // height_to_subsidy: StorableVec::import(&path.join("height_to_subsidy"), Version::from(1))?, - // height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?, - // height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?, - txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?, - txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?, - txindex_to_is_coinbase: StorableVec::import(&path.join("txindex_to_is_coinbase"), Version::from(1))?, - // txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?, - txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?, - txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?, - txindex_to_last_txoutindex: StorableVec::import( - &path.join("txindex_to_last_txoutindex"), - Version::from(1), - )?, - txindex_to_outputcount: StorableVec::import(&path.join("txindex_to_outputcount"), Version::from(1))?, - }) - } - - // pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] { - // [&self.date_to_first_height] - // } - - // pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] { - // [&mut self.date_to_first_height] - // } -} diff --git a/computer/src/structs/addressindextxoutindex.rs b/computer/src/structs/addressindextxoutindex.rs index bc5ef32a2..ebaab0bae 100644 --- a/computer/src/structs/addressindextxoutindex.rs +++ b/computer/src/structs/addressindextxoutindex.rs @@ -1,21 +1,22 @@ use bindex::{Addressindex, Txoutindex}; use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct AddressindexTxoutindex { addressindex: Addressindex, + _padding: u32, txoutindex: Txoutindex, } impl TryFrom for AddressindexTxoutindex { - type Error = unsafe_slice_serde::Error; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From for Slice { fn from(value: AddressindexTxoutindex) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } diff --git a/computer/src/structs/date.rs b/computer/src/structs/date.rs index 9d965662b..3ae799946 100644 --- a/computer/src/structs/date.rs +++ b/computer/src/structs/date.rs @@ -2,11 +2,30 @@ use std::ops::Add; use bindex::Timestamp; use color_eyre::eyre::eyre; -use derive_deref::Deref; -use jiff::{civil::Date as _Date, tz::TimeZone, Span}; +use jiff::{civil::Date as Date_, tz::TimeZone, Span}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)] -pub struct Date(_Date); +#[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)] +pub struct Date { + year: u16, + month: u8, + day: u8, +} + +impl Date { + const INDEX_ZERO: Self = Self { + year: 2009, + month: 1, + day: 3, + }; + const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3); + const INDEX_ONE: Self = Self { + year: 2009, + month: 1, + day: 9, + }; + const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9); +} impl Default for Date { fn default() -> Self { @@ -14,49 +33,67 @@ impl Default for Date { } } -impl Date { - const INDEX_ZERO: Self = Self(_Date::constant(2009, 1, 3)); - const INDEX_ONE: Self = Self(_Date::constant(2009, 1, 9)); -} - -impl From<_Date> for Date { - fn from(value: _Date) -> Self { - Self(value) +impl PartialOrd for Date { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } } -impl From<&Timestamp> for Date { - fn from(value: &Timestamp) -> Self { - Self(_Date::from(value.to_zoned(TimeZone::UTC))) +impl Ord for Date { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + Date_::from(*self).cmp(&Date_::from(*other)) + } +} + +impl From for Date { + fn from(value: Date_) -> Self { + Self { + year: value.year() as u16, + month: value.month() as u8, + day: value.day() as u8, + } + } +} + +impl From for Date_ { + fn from(value: Date) -> Self { + Self::new(value.year as i16, value.month as i8, value.day as i8).unwrap() + } +} + +impl From for Date { + fn from(value: Timestamp) -> Self { + Self::from(Date_::from(jiff::Timestamp::from(value).to_zoned(TimeZone::UTC))) } } impl TryFrom for usize { type Error = color_eyre::Report; fn try_from(value: Date) -> Result { - if value < Date::INDEX_ZERO { + let value_ = Date_::from(value); + if value_ < Date::INDEX_ZERO_ { Err(eyre!("Date is too early")) } else if value == Date::INDEX_ZERO { Ok(0) - } else if value < Date::INDEX_ONE { + } else if value_ < Date::INDEX_ONE_ { Err(eyre!("Date is between first and second")) } else if value == Date::INDEX_ONE { Ok(1) } else { - Ok(Date::INDEX_ONE.until(*value)?.get_days() as usize + 1) + Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1) } } } impl From for Date { fn from(value: usize) -> Self { - Self(Self::INDEX_ZERO.checked_add(Span::new().days(value as i64)).unwrap()) + Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap()) } } impl Add for Date { type Output = Self; fn add(self, rhs: usize) -> Self::Output { - Self(self.0.checked_add(Span::new().days(rhs as i64)).unwrap()) + Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap()) } } diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 2ef5fe3a3..64d91747c 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -13,6 +13,5 @@ fjall = { workspace = true } jiff = { workspace = true } rapidhash = "1.3.0" rayon = { workspace = true } -# snkrj = { workspace = true } storable_vec = { workspace = true } -unsafe_slice_serde = { workspace = true } +zerocopy = { workspace = true } diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs index 511feade3..3447e5d17 100644 --- a/indexer/src/lib.rs +++ b/indexer/src/lib.rs @@ -9,7 +9,7 @@ use std::{ pub use biter::*; -use bitcoin::{Transaction, TxIn, TxOut, Txid}; +use bitcoin::{Transaction, TxIn, TxOut}; use color_eyre::eyre::{eyre, ContextCompat}; use exit::Exit; use rayon::prelude::*; @@ -19,7 +19,7 @@ mod storage; mod structs; pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta}; -pub use structs::Version; +use structs::{BlockHash, Txid}; use storage::{Fjalls, StorableVecs}; pub use structs::{ @@ -98,6 +98,7 @@ impl Indexer { .try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> { println!("Processing block {_height}..."); + let blockhash = BlockHash::from(blockhash); height = Height::from(_height); if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? { @@ -124,9 +125,9 @@ impl Indexer { vecs.height_to_blockhash.push_if_needed(height, blockhash)?; vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?; - vecs.height_to_timestamp.push_if_needed(height, Timestamp::try_from(block.header.time)?)?; + vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?; vecs.height_to_size.push_if_needed(height, block.total_size())?; - vecs.height_to_weight.push_if_needed(height, block.weight())?; + vecs.height_to_weight.push_if_needed(height, block.weight().into())?; vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?; vecs.height_to_first_txinindex .push_if_needed(height, txinindex_global)?; @@ -192,7 +193,7 @@ impl Indexer { .par_iter() .enumerate() .map(|(index, tx)| -> color_eyre::Result<_> { - let txid = tx.compute_txid(); + let txid = Txid::from(tx.compute_txid()); let txid_prefix = TxidPrefix::try_from(&txid)?; @@ -233,6 +234,7 @@ impl Indexer { // dbg!((txindex, txinindex, vin)); let outpoint = txin.previous_output; + let txid = Txid::from(outpoint.txid); if tx.is_coinbase() { return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin)))); @@ -240,7 +242,7 @@ impl Indexer { let prev_txindex = if let Some(txindex) = trees .txid_prefix_to_txindex - .get(&TxidPrefix::try_from(&outpoint.txid)?)? + .get(&TxidPrefix::try_from(&txid)?)? .map(|v| *v) .and_then(|txindex| { // Checking if not finding txindex from the future @@ -537,7 +539,7 @@ impl Indexer { } let outpoint = txin.previous_output; - let txid = outpoint.txid; + let txid = Txid::from(outpoint.txid); let vout = Vout::from(outpoint.vout); let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex @@ -604,7 +606,7 @@ impl Indexer { .get(prev_txindex)? .context("To have txid for txindex") .inspect_err(|_| { - dbg!(txindex, txid, len); + dbg!(txindex, len); })?; // #[allow(clippy::redundant_locals)] @@ -614,12 +616,12 @@ impl Indexer { // If another Txid needs to be added to the list // We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated let only_known_dup_txids = [ - Txid::from_str( + bitcoin::Txid::from_str( "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599", - )?, - Txid::from_str( + )?.into(), + bitcoin::Txid::from_str( "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468", - )?, + )?.into(), ]; let is_dup = only_known_dup_txids.contains(prev_txid); @@ -627,7 +629,7 @@ impl Indexer { if !is_dup { let prev_height = vecs.txindex_to_height.get(prev_txindex)?.expect("To have height"); - dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex); + dbg!(height, txindex, prev_height, prev_txid, prev_txindex); return Err(eyre!("Expect none")); } } @@ -640,10 +642,10 @@ impl Indexer { txindex_to_tx_and_txid .into_iter() .try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> { - vecs.txindex_to_txversion.push_if_needed(txindex, tx.version)?; + vecs.txindex_to_txversion.push_if_needed(txindex, tx.version.into())?; vecs.txindex_to_txid.push_if_needed(txindex, txid)?; vecs.txindex_to_height.push_if_needed(txindex, height)?; - vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time)?; + vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time.into())?; Ok(()) })?; diff --git a/indexer/src/storage/fjalls/base.rs b/indexer/src/storage/fjalls/base.rs index 8f2767da7..6d6030f2e 100644 --- a/indexer/src/storage/fjalls/base.rs +++ b/indexer/src/storage/fjalls/base.rs @@ -4,10 +4,10 @@ use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, TransactionalPartitionHandle, }; -use storable_vec::Value; -use unsafe_slice_serde::UnsafeSliceSerde; +use storable_vec::{Value, Version}; +use zerocopy::{Immutable, IntoBytes}; -use crate::structs::{Height, Version}; +use crate::structs::Height; use super::StoreMeta; @@ -21,7 +21,7 @@ pub struct Store { impl Store where - K: Into + Ord, + K: Into + Ord + Immutable + IntoBytes, V: Into + TryFrom, >::Error: error::Error + Send + Sync + 'static, { @@ -54,7 +54,7 @@ where pub fn get(&self, key: &K) -> color_eyre::Result>> { if let Some(v) = self.puts.get(key) { Ok(Some(Value::Ref(v))) - } else if let Some(slice) = self.rtx.get(&self.part, key.unsafe_as_slice())? { + } else if let Some(slice) = self.rtx.get(&self.part, key.as_bytes())? { Ok(Some(Value::Owned(V::try_from(slice)?))) } else { Ok(None) diff --git a/indexer/src/storage/fjalls/meta.rs b/indexer/src/storage/fjalls/meta.rs index 1372f1d34..35f7bf24d 100644 --- a/indexer/src/storage/fjalls/meta.rs +++ b/indexer/src/storage/fjalls/meta.rs @@ -3,9 +3,10 @@ use std::{ path::{Path, PathBuf}, }; -use unsafe_slice_serde::UnsafeSliceSerde; +use storable_vec::Version; +use zerocopy::{FromBytes, IntoBytes}; -use super::{Height, Version}; +use super::Height; pub struct StoreMeta { pathbuf: PathBuf, @@ -87,14 +88,14 @@ impl StoreMeta { // } fn read_length_(path: &Path) -> color_eyre::Result { Ok(fs::read(Self::path_length(path)) - .map(|v| usize::unsafe_try_from_slice(v.as_slice()).cloned().unwrap_or_default()) + .map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default()) .unwrap_or_default()) } fn write_length(&self) -> io::Result<()> { Self::write_length_(&self.pathbuf, self.len) } fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> { - fs::write(Self::path_length(path), len.to_le_bytes()) + fs::write(Self::path_length(path), len.as_bytes()) } fn path_length(path: &Path) -> PathBuf { path.join("length") diff --git a/indexer/src/storage/fjalls/mod.rs b/indexer/src/storage/fjalls/mod.rs index be75266ea..4c6724929 100644 --- a/indexer/src/storage/fjalls/mod.rs +++ b/indexer/src/storage/fjalls/mod.rs @@ -1,9 +1,12 @@ use std::{path::Path, thread}; -use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex}; +use storable_vec::Version; + +use crate::{AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex}; mod base; mod meta; +// mod version; pub use base::*; pub use meta::*; diff --git a/indexer/src/storage/fjalls/version.rs b/indexer/src/storage/fjalls/version.rs new file mode 100644 index 000000000..1fb2028ba --- /dev/null +++ b/indexer/src/storage/fjalls/version.rs @@ -0,0 +1,51 @@ +use std::{fs, io, path::Path}; + +use derive_deref::Deref; +use fjall::Slice; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)] +pub struct Version(u32); + +impl Version { + pub fn write(&self, path: &Path) -> Result<(), io::Error> { + fs::write(path, self.to_ne_bytes()) + } +} + +impl From for Version { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl TryFrom<&Path> for Version { + type Error = io::Error; + fn try_from(value: &Path) -> Result { + Self::try_from(&fs::read(value)?) + } +} +impl TryFrom for Version { + type Error = fjall::Error; + fn try_from(value: Slice) -> Result { + Self::from(&value) + } +} +impl TryFrom<&[u8]> for Version { + type Error = storable_vec::Error; + fn try_from(value: &[u8]) -> Result { + let mut buf: [u8; 4] = [0; 4]; + let buf_len = buf.len(); + if value.len() != buf_len { + panic!(); + } + value.iter().enumerate().for_each(|(i, r)| { + buf[i] = *r; + }); + Ok(Self(u32::from_ne_bytes(buf))) + } +} +impl From for Slice { + fn from(value: Version) -> Self { + Self::new(&value.to_ne_bytes()) + } +} diff --git a/indexer/src/storage/storable_vecs/base.rs b/indexer/src/storage/storable_vecs/base.rs index 5f32bdc26..e165431ce 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/indexer/src/storage/storable_vecs/base.rs @@ -34,7 +34,7 @@ where self.vec.flush() } - pub fn height(&self) -> color_eyre::Result { + pub fn height(&self) -> storable_vec::Result { Height::try_from(self.path_height().as_path()) } fn path_height(&self) -> PathBuf { @@ -66,7 +66,7 @@ impl DerefMut for StorableVec { } pub trait AnyStorableVec { - fn height(&self) -> color_eyre::Result; + fn height(&self) -> storable_vec::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } @@ -75,7 +75,7 @@ where I: StorableVecIndex, T: StorableVecType, { - fn height(&self) -> color_eyre::Result { + fn height(&self) -> storable_vec::Result { self.height() } diff --git a/indexer/src/storage/storable_vecs/mod.rs b/indexer/src/storage/storable_vecs/mod.rs index 4c43f3e8f..09654f836 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/indexer/src/storage/storable_vecs/mod.rs @@ -1,14 +1,13 @@ use std::{fs, io, path::Path}; -use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight}; use exit::Exit; use rayon::prelude::*; use storable_vec::{Version, CACHED_GETS}; use crate::structs::{ - Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes, - P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex, - Txinindex, Txoutindex, + Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Height, LockTime, P2PK33AddressBytes, + P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, + Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Weight, }; mod base; @@ -50,9 +49,9 @@ pub struct StorableVecs { pub txindex_to_first_txinindex: StorableVec, pub txindex_to_first_txoutindex: StorableVec, pub txindex_to_height: StorableVec, - pub txindex_to_locktime: StorableVec, + pub txindex_to_locktime: StorableVec, pub txindex_to_txid: StorableVec, - pub txindex_to_txversion: StorableVec, + pub txindex_to_txversion: StorableVec, pub txinindex_to_txoutindex: StorableVec, pub txoutindex_to_addressindex: StorableVec, pub txoutindex_to_amount: StorableVec, diff --git a/indexer/src/structs/addressbytes.rs b/indexer/src/structs/addressbytes.rs index 463e5d542..34ce0663d 100644 --- a/indexer/src/structs/addressbytes.rs +++ b/indexer/src/structs/addressbytes.rs @@ -1,6 +1,7 @@ use biter::bitcoin::ScriptBuf; use color_eyre::eyre::eyre; use derive_deref::{Deref, DerefMut}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Addresstype; @@ -122,28 +123,28 @@ impl From for Addressbytes { } } -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2PK65AddressBytes(U8x65); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2PK33AddressBytes(U8x33); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2PKHAddressBytes(U8x20); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2SHAddressBytes(U8x20); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2WPKHAddressBytes(U8x20); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2WSHAddressBytes(U8x32); -#[derive(Debug, Clone, Deref, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct P2TRAddressBytes(U8x32); -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct U8x20([u8; 20]); impl From<&[u8]> for U8x20 { fn from(slice: &[u8]) -> Self { @@ -153,7 +154,7 @@ impl From<&[u8]> for U8x20 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct U8x32([u8; 32]); impl From<&[u8]> for U8x32 { fn from(slice: &[u8]) -> Self { @@ -163,7 +164,7 @@ impl From<&[u8]> for U8x32 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct U8x33([u8; 33]); impl From<&[u8]> for U8x33 { fn from(slice: &[u8]) -> Self { @@ -173,7 +174,7 @@ impl From<&[u8]> for U8x33 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct U8x64([u8; 64]); impl From<&[u8]> for U8x64 { fn from(slice: &[u8]) -> Self { @@ -183,7 +184,7 @@ impl From<&[u8]> for U8x64 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)] +#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] pub struct U8x65([u8; 65]); impl From<&[u8]> for U8x65 { fn from(slice: &[u8]) -> Self { diff --git a/indexer/src/structs/addressindex.rs b/indexer/src/structs/addressindex.rs index 0a5ea91c9..72fcf06a0 100644 --- a/indexer/src/structs/addressindex.rs +++ b/indexer/src/structs/addressindex.rs @@ -2,9 +2,24 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Addressindex(u32); impl Addressindex { @@ -52,14 +67,14 @@ impl From for usize { } impl TryFrom for Addressindex { - type Error = unsafe_slice_serde::Error; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From for Slice { fn from(value: Addressindex) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } diff --git a/indexer/src/structs/addresstype.rs b/indexer/src/structs/addresstype.rs index abc37a296..efb5a2a71 100644 --- a/indexer/src/structs/addresstype.rs +++ b/indexer/src/structs/addresstype.rs @@ -1,6 +1,8 @@ use biter::bitcoin::ScriptBuf; +use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromBytes, Immutable, IntoBytes, KnownLayout)] +#[repr(u8)] pub enum Addresstype { P2PK65, P2PK33, diff --git a/indexer/src/structs/addresstypeindex.rs b/indexer/src/structs/addresstypeindex.rs index d94938e97..042a8d352 100644 --- a/indexer/src/structs/addresstypeindex.rs +++ b/indexer/src/structs/addresstypeindex.rs @@ -1,8 +1,24 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Addresstypeindex(u32); impl Addresstypeindex { diff --git a/indexer/src/structs/amount.rs b/indexer/src/structs/amount.rs index 6c66fc725..bfe620a25 100644 --- a/indexer/src/structs/amount.rs +++ b/indexer/src/structs/amount.rs @@ -5,14 +5,30 @@ use std::{ use biter::bitcoin; use derive_deref::{Deref, DerefMut}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Height; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] -pub struct Amount(bitcoin::Amount); +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] +pub struct Amount(u64); impl Amount { - pub const ZERO: Self = Self(bitcoin::Amount::ZERO); + pub const ZERO: Self = Self(0); pub const ONE_BTC_F32: f32 = 100_000_000.0; pub const ONE_BTC_F64: f64 = 100_000_000.0; @@ -24,65 +40,70 @@ impl Amount { impl Add for Amount { type Output = Amount; fn add(self, rhs: Amount) -> Self::Output { - Amount::from(self.to_sat() + rhs.to_sat()) + Amount::from(*self + *rhs) } } impl AddAssign for Amount { fn add_assign(&mut self, rhs: Self) { - *self = Amount::from(self.to_sat() + rhs.to_sat()); + *self = *self + rhs; } } impl Sub for Amount { type Output = Amount; fn sub(self, rhs: Amount) -> Self::Output { - Amount::from(self.to_sat() - rhs.to_sat()) + Amount::from(*self - *rhs) } } impl SubAssign for Amount { fn sub_assign(&mut self, rhs: Self) { - *self = Amount::from(self.to_sat() - rhs.to_sat()); + *self = *self - rhs; } } impl Mul for Amount { type Output = Amount; fn mul(self, rhs: Amount) -> Self::Output { - Amount::from(self.to_sat() * rhs.to_sat()) + Amount::from(*self * *rhs) } } impl Mul for Amount { type Output = Amount; fn mul(self, rhs: u64) -> Self::Output { - Amount::from(self.to_sat() * rhs) + Amount::from(*self * rhs) } } impl Mul for Amount { type Output = Amount; fn mul(self, rhs: Height) -> Self::Output { - Amount::from(self.to_sat() * *rhs as u64) + Amount::from(*self * *rhs as u64) } } impl Sum for Amount { fn sum>(iter: I) -> Self { - let sats: u64 = iter.map(|amt| amt.to_sat()).sum(); + let sats: u64 = iter.map(|amt| *amt).sum(); Amount::from(sats) } } impl From for Amount { fn from(value: u64) -> Self { - Self(bitcoin::Amount::from_sat(value)) + Self(value) } } impl From for Amount { fn from(value: bitcoin::Amount) -> Self { - Self(value) + Self(value.to_sat()) + } +} +impl From for bitcoin::Amount { + fn from(value: Amount) -> Self { + Self::from_sat(value.0) } } diff --git a/indexer/src/structs/blockhash.rs b/indexer/src/structs/blockhash.rs new file mode 100644 index 000000000..df24104b8 --- /dev/null +++ b/indexer/src/structs/blockhash.rs @@ -0,0 +1,19 @@ +use std::mem; + +use derive_deref::Deref; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] +pub struct BlockHash([u8; 32]); + +impl From for BlockHash { + fn from(value: bitcoin::BlockHash) -> Self { + unsafe { mem::transmute(value) } + } +} + +impl From for bitcoin::BlockHash { + fn from(value: BlockHash) -> Self { + unsafe { mem::transmute(value) } + } +} diff --git a/indexer/src/structs/compressed.rs b/indexer/src/structs/compressed.rs index 59acbaa97..92301d14b 100644 --- a/indexer/src/structs/compressed.rs +++ b/indexer/src/structs/compressed.rs @@ -1,13 +1,12 @@ use std::hash::Hasher; -use biter::bitcoin::{BlockHash, Txid}; use derive_deref::Deref; use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use super::{Addressbytes, Addresstype, SliceExtended}; +use super::{Addressbytes, Addresstype, BlockHash, Txid}; -#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct AddressHash([u8; 8]); impl From<(&Addressbytes, Addresstype)> for AddressHash { fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self { @@ -24,14 +23,14 @@ impl From<[u8; 8]> for AddressHash { } } impl TryFrom for AddressHash { - type Error = color_eyre::Report; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From<&AddressHash> for Slice { fn from(value: &AddressHash) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } impl From for Slice { @@ -40,23 +39,23 @@ impl From for Slice { } } -#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct BlockHashPrefix([u8; 8]); impl TryFrom<&BlockHash> for BlockHashPrefix { type Error = color_eyre::Report; fn try_from(value: &BlockHash) -> Result { - Ok(Self((&value[..]).read_8x_u8()?)) + Ok(Self(copy_first_8bytes(&value[..]))) } } impl TryFrom for BlockHashPrefix { - type Error = color_eyre::Report; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From<&BlockHashPrefix> for Slice { fn from(value: &BlockHashPrefix) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } impl From for Slice { @@ -65,23 +64,23 @@ impl From for Slice { } } -#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] pub struct TxidPrefix([u8; 8]); impl TryFrom<&Txid> for TxidPrefix { type Error = color_eyre::Report; fn try_from(value: &Txid) -> Result { - Ok(Self((&value[..]).read_8x_u8()?)) + Ok(Self(copy_first_8bytes(&value[..]))) } } impl TryFrom for TxidPrefix { - type Error = color_eyre::Report; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From<&TxidPrefix> for Slice { fn from(value: &TxidPrefix) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } impl From for Slice { @@ -89,3 +88,15 @@ impl From for Slice { Self::from(&value) } } + +fn copy_first_8bytes(slice: &[u8]) -> [u8; 8] { + let mut buf: [u8; 8] = [0; 8]; + let buf_len = buf.len(); + if slice.len() < buf_len { + panic!("bad len"); + } + slice.iter().take(buf_len).enumerate().for_each(|(i, r)| { + buf[i] = *r; + }); + buf +} diff --git a/indexer/src/structs/height.rs b/indexer/src/structs/height.rs index b9ff4b92f..c8b999597 100644 --- a/indexer/src/structs/height.rs +++ b/indexer/src/structs/height.rs @@ -7,14 +7,29 @@ use std::{ use biter::rpc::{self, RpcApi}; use derive_deref::{Deref, DerefMut}; use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)] +#[derive( + Debug, + Clone, + Copy, + Deref, + DerefMut, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Height(u32); impl Height { pub fn write(&self, path: &Path) -> Result<(), io::Error> { - fs::write(path, self.unsafe_as_slice()) + fs::write(path, self.as_bytes()) } } @@ -113,9 +128,9 @@ impl From for usize { } impl TryFrom<&Path> for Height { - type Error = color_eyre::Report; + type Error = storable_vec::Error; fn try_from(value: &Path) -> Result { - Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned()) + Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned()) } } @@ -127,13 +142,25 @@ impl TryFrom<&rpc::Client> for Height { } impl TryFrom for Height { - type Error = unsafe_slice_serde::Error; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From for Slice { fn from(value: Height) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) + } +} + +impl From for Height { + fn from(value: bitcoin::locktime::absolute::Height) -> Self { + Self(value.to_consensus_u32()) + } +} + +impl From for bitcoin::locktime::absolute::Height { + fn from(value: Height) -> Self { + bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap() } } diff --git a/indexer/src/structs/locktime.rs b/indexer/src/structs/locktime.rs new file mode 100644 index 000000000..3d1963945 --- /dev/null +++ b/indexer/src/structs/locktime.rs @@ -0,0 +1,28 @@ +use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; + +use super::{Height, Timestamp}; + +#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes)] +#[repr(C)] +pub enum LockTime { + Height(Height), + Timestamp(Timestamp), +} + +impl From for LockTime { + fn from(value: bitcoin::absolute::LockTime) -> Self { + match value { + bitcoin::absolute::LockTime::Blocks(h) => LockTime::Height(h.into()), + bitcoin::absolute::LockTime::Seconds(t) => LockTime::Timestamp(t.into()), + } + } +} + +impl From for bitcoin::absolute::LockTime { + fn from(value: LockTime) -> Self { + match value { + LockTime::Height(h) => bitcoin::absolute::LockTime::Blocks(h.into()), + LockTime::Timestamp(t) => bitcoin::absolute::LockTime::Seconds(t.into()), + } + } +} diff --git a/indexer/src/structs/mod.rs b/indexer/src/structs/mod.rs index bb92346ed..04b682955 100644 --- a/indexer/src/structs/mod.rs +++ b/indexer/src/structs/mod.rs @@ -3,29 +3,35 @@ mod addressindex; mod addresstype; mod addresstypeindex; mod amount; +mod blockhash; mod compressed; mod height; -mod slice; +mod locktime; mod timestamp; +mod txid; mod txindex; mod txinindex; mod txoutindex; -mod version; +mod txversion; mod vin; mod vout; +mod weight; pub use addressbytes::*; pub use addressindex::*; pub use addresstype::*; pub use addresstypeindex::*; pub use amount::*; +pub use blockhash::*; pub use compressed::*; pub use height::*; -pub use slice::*; +pub use locktime::*; pub use timestamp::*; +pub use txid::*; pub use txindex::*; pub use txinindex::*; pub use txoutindex::*; -pub use version::*; +pub use txversion::*; pub use vin::*; pub use vout::*; +pub use weight::*; diff --git a/indexer/src/structs/slice.rs b/indexer/src/structs/slice.rs deleted file mode 100644 index d09b7890f..000000000 --- a/indexer/src/structs/slice.rs +++ /dev/null @@ -1,55 +0,0 @@ -use color_eyre::eyre::eyre; - -#[allow(unused)] -pub trait SliceExtended { - fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]>; - fn read_be_u8(&self) -> color_eyre::Result; - fn read_be_u16(&self) -> color_eyre::Result; - fn read_be_u32(&self) -> color_eyre::Result; - fn read_be_u64(&self) -> color_eyre::Result; - fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>; -} - -impl SliceExtended for &[u8] { - fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]> { - let mut buf: [u8; 8] = [0; 8]; - (&self[..8]).read_exact(&mut buf)?; - Ok(buf) - } - - fn read_be_u8(&self) -> color_eyre::Result { - let mut buf: [u8; 1] = [0; 1]; - self.read_exact(&mut buf)?; - Ok(u8::from_be_bytes(buf)) - } - - fn read_be_u16(&self) -> color_eyre::Result { - let mut buf: [u8; 2] = [0; 2]; - self.read_exact(&mut buf)?; - Ok(u16::from_be_bytes(buf)) - } - - fn read_be_u32(&self) -> color_eyre::Result { - let mut buf: [u8; 4] = [0; 4]; - self.read_exact(&mut buf)?; - Ok(u32::from_be_bytes(buf)) - } - - fn read_be_u64(&self) -> color_eyre::Result { - let mut buf: [u8; 8] = [0; 8]; - self.read_exact(&mut buf)?; - Ok(u64::from_be_bytes(buf)) - } - - fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> { - let buf_len = buf.len(); - if self.len() != buf_len { - dbg!(self.len(), buf_len); - return Err(eyre!("Not exact len")); - } - self.iter().take(buf_len).enumerate().for_each(|(i, r)| { - buf[i] = *r; - }); - Ok(()) - } -} diff --git a/indexer/src/structs/timestamp.rs b/indexer/src/structs/timestamp.rs index 4b9fc285e..b7c73106d 100644 --- a/indexer/src/structs/timestamp.rs +++ b/indexer/src/structs/timestamp.rs @@ -1,11 +1,29 @@ use derive_deref::Deref; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct Timestamp(jiff::Timestamp); +#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)] +pub struct Timestamp(u32); -impl TryFrom for Timestamp { - type Error = jiff::Error; - fn try_from(value: u32) -> Result { - Ok(Self(jiff::Timestamp::from_second(value as i64)?)) +impl From for Timestamp { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for jiff::Timestamp { + fn from(value: Timestamp) -> Self { + jiff::Timestamp::from_second(*value as i64).unwrap() + } +} + +impl From for Timestamp { + fn from(value: bitcoin::locktime::absolute::Time) -> Self { + Self(value.to_consensus_u32()) + } +} + +impl From for bitcoin::locktime::absolute::Time { + fn from(value: Timestamp) -> Self { + bitcoin::locktime::absolute::Time::from_consensus(*value).unwrap() } } diff --git a/indexer/src/structs/txid.rs b/indexer/src/structs/txid.rs new file mode 100644 index 000000000..fee437fa0 --- /dev/null +++ b/indexer/src/structs/txid.rs @@ -0,0 +1,19 @@ +use std::mem; + +use derive_deref::Deref; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)] +pub struct Txid([u8; 32]); + +impl From for Txid { + fn from(value: bitcoin::Txid) -> Self { + unsafe { mem::transmute(value) } + } +} + +impl From for bitcoin::Txid { + fn from(value: Txid) -> Self { + unsafe { mem::transmute(value) } + } +} diff --git a/indexer/src/structs/txindex.rs b/indexer/src/structs/txindex.rs index 906e605d8..479380958 100644 --- a/indexer/src/structs/txindex.rs +++ b/indexer/src/structs/txindex.rs @@ -2,9 +2,24 @@ use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Txindex(u32); impl Txindex { @@ -73,13 +88,13 @@ impl From for usize { } impl TryFrom for Txindex { - type Error = unsafe_slice_serde::Error; + type Error = storable_vec::Error; fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) + Ok(Self::read_from_bytes(&value)?) } } impl From for Slice { fn from(value: Txindex) -> Self { - Self::new(value.unsafe_as_slice()) + Self::new(value.as_bytes()) } } diff --git a/indexer/src/structs/txinindex.rs b/indexer/src/structs/txinindex.rs index 9c217961c..78727456e 100644 --- a/indexer/src/structs/txinindex.rs +++ b/indexer/src/structs/txinindex.rs @@ -1,10 +1,26 @@ use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Vin; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Txinindex(u64); impl Txinindex { diff --git a/indexer/src/structs/txoutindex.rs b/indexer/src/structs/txoutindex.rs index 541ae40ab..fbc85dea2 100644 --- a/indexer/src/structs/txoutindex.rs +++ b/indexer/src/structs/txoutindex.rs @@ -1,10 +1,26 @@ use std::ops::{Add, AddAssign, Sub}; use derive_deref::{Deref, DerefMut}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Vout; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)] +#[derive( + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + FromBytes, + Immutable, + IntoBytes, + KnownLayout, +)] pub struct Txoutindex(u64); impl Txoutindex { diff --git a/indexer/src/structs/txversion.rs b/indexer/src/structs/txversion.rs new file mode 100644 index 000000000..345e5f0e8 --- /dev/null +++ b/indexer/src/structs/txversion.rs @@ -0,0 +1,17 @@ +use derive_deref::Deref; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)] +pub struct TxVersion(i32); + +impl From for TxVersion { + fn from(value: bitcoin::transaction::Version) -> Self { + Self(value.0) + } +} + +impl From for bitcoin::transaction::Version { + fn from(value: TxVersion) -> Self { + Self(value.0) + } +} diff --git a/indexer/src/structs/version.rs b/indexer/src/structs/version.rs deleted file mode 100644 index 4d8257e2c..000000000 --- a/indexer/src/structs/version.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::{fs, io, path::Path}; - -use fjall::Slice; -use unsafe_slice_serde::UnsafeSliceSerde; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct Version(u32); - -impl Version { - pub fn write(&self, path: &Path) -> Result<(), io::Error> { - fs::write(path, self.unsafe_as_slice()) - } -} - -impl From for Version { - fn from(value: u32) -> Self { - Self(value) - } -} - -impl TryFrom<&Path> for Version { - type Error = color_eyre::Report; - fn try_from(value: &Path) -> Result { - Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned()) - } -} - -impl TryFrom for Version { - type Error = color_eyre::Report; - fn try_from(value: Slice) -> Result { - Ok(*Self::unsafe_try_from_slice(&value)?) - } -} -impl From for Slice { - fn from(value: Version) -> Self { - Self::new(value.unsafe_as_slice()) - } -} diff --git a/indexer/src/structs/weight.rs b/indexer/src/structs/weight.rs new file mode 100644 index 000000000..c930ee867 --- /dev/null +++ b/indexer/src/structs/weight.rs @@ -0,0 +1,17 @@ +use derive_deref::Deref; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)] +pub struct Weight(u64); + +impl From for Weight { + fn from(value: bitcoin::Weight) -> Self { + Self(value.to_wu()) + } +} + +impl From for bitcoin::Weight { + fn from(value: Weight) -> Self { + Self::from_wu(*value) + } +} diff --git a/storable_vec/Cargo.toml b/storable_vec/Cargo.toml index 61cb279ee..5604fb80b 100644 --- a/storable_vec/Cargo.toml +++ b/storable_vec/Cargo.toml @@ -9,4 +9,4 @@ edition = "2021" [dependencies] memmap2 = "0.9.5" -unsafe_slice_serde = { workspace = true } +zerocopy = { workspace = true } diff --git a/storable_vec/src/enums/error.rs b/storable_vec/src/enums/error.rs index 0905fd97c..37908bbae 100644 --- a/storable_vec/src/enums/error.rs +++ b/storable_vec/src/enums/error.rs @@ -13,7 +13,8 @@ pub enum Error { DifferentVersion { found: Version, expected: Version }, MmapsVecIsTooSmall, IO(io::Error), - UnsafeSliceSerde(unsafe_slice_serde::Error), + // UnsafeSliceSerde(zerocopy::error::), + ZeroCopyError, IndexTooHigh, IndexTooLow, ExpectFileToHaveIndex, @@ -28,9 +29,15 @@ impl From for Error { } } -impl From for Error { - fn from(value: unsafe_slice_serde::Error) -> Self { - Self::UnsafeSliceSerde(value) +impl From> for Error { + fn from(_: zerocopy::error::ConvertError) -> Self { + Self::ZeroCopyError + } +} + +impl From> for Error { + fn from(_: zerocopy::error::SizeError) -> Self { + Self::ZeroCopyError } } @@ -43,7 +50,7 @@ impl fmt::Display for Error { } Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"), Error::IO(error) => Debug::fmt(&error, f), - Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), + // Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f), Error::IndexTooHigh => write!(f, "Index too high"), Error::IndexTooLow => write!(f, "Index too low"), Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"), @@ -52,6 +59,7 @@ impl fmt::Display for Error { Error::UnsupportedUnflushedState => { write!(f, "Unsupported unflush state, please flush before using this function") } + Error::ZeroCopyError => write!(f, "Zero copy convert error"), } } } diff --git a/storable_vec/src/lib.rs b/storable_vec/src/lib.rs index cbaf540ae..3a916394d 100644 --- a/storable_vec/src/lib.rs +++ b/storable_vec/src/lib.rs @@ -11,8 +11,8 @@ use std::{ sync::OnceLock, }; -use memmap2::{Mmap, MmapOptions}; -use unsafe_slice_serde::UnsafeSliceSerde; +pub use memmap2; +pub use zerocopy; mod enums; mod structs; @@ -60,7 +60,7 @@ pub struct StorableVec { file_position: u64, buf: Buffer, /// Only for CACHED_GETS - cache: Vec>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16) + cache: Vec>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16) pushed: Vec, // updated: BTreeMap, // inserted: BTreeMap, @@ -207,7 +207,7 @@ where fn read_exact<'a>(file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> { file.read_exact(buf)?; - let v = T::unsafe_try_from_slice(&buf[..])?; + let v = T::try_ref_from_bytes(&buf[..])?; Ok(v) } @@ -277,7 +277,7 @@ where mem::take(&mut self.pushed) .into_iter() - .for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice())); + .for_each(|v| bytes.extend_from_slice(v.as_bytes())); self.file.write_all(&bytes)?; @@ -383,7 +383,7 @@ where .ok_or(Error::MmapsVecIsTooSmall)? .get_or_init(|| { Box::new(unsafe { - MmapOptions::new() + memmap2::MmapOptions::new() .len(Self::PAGE_SIZE) .offset((page_index * Self::PAGE_SIZE) as u64) .map(&self.file) @@ -393,7 +393,7 @@ where let range = Self::index_to_byte_range(index); let slice = &mmap[range]; - return Ok(Some(Value::Ref(T::unsafe_try_from_slice(slice)?))); + return Ok(Some(Value::Ref(T::try_ref_from_bytes(slice)?))); } Ok(Some(Value::Owned(self.open_file_at_then_read(index)?.to_owned()))) @@ -581,8 +581,9 @@ where other_to_self: &mut StorableVec, ) -> Result<()> where - A: StorableVecIndex + StorableVecType, + I: StorableVecType, T: From, + A: StorableVecIndex + StorableVecType, { self_to_other.iter_from(I::from(self.len()), |(i, other)| { self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i)) diff --git a/storable_vec/src/structs/version.rs b/storable_vec/src/structs/version.rs index 67af8da9f..565f868b1 100644 --- a/storable_vec/src/structs/version.rs +++ b/storable_vec/src/structs/version.rs @@ -4,16 +4,16 @@ use std::{ path::Path, }; -use unsafe_slice_serde::UnsafeSliceSerde; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::Error; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout)] pub struct Version(u32); impl Version { pub fn write(&self, path: &Path) -> Result<(), io::Error> { - fs::write(path, self.0.unsafe_as_slice()) + fs::write(path, self.as_bytes()) } pub fn swap_bytes(self) -> Self { @@ -32,6 +32,6 @@ impl TryFrom<&Path> for Version { fn try_from(value: &Path) -> Result { let mut buf = [0; 4]; fs::read(value)?.as_slice().read_exact(&mut buf)?; - Ok(*(Self::unsafe_try_from_slice(&buf)?)) + Ok(*(Self::ref_from_bytes(&buf)?)) } } diff --git a/storable_vec/src/traits/index.rs b/storable_vec/src/traits/index.rs index 6ff29dcca..c99b64320 100644 --- a/storable_vec/src/traits/index.rs +++ b/storable_vec/src/traits/index.rs @@ -2,10 +2,30 @@ use std::{fmt::Debug, ops::Add}; pub trait StorableVecIndex where - Self: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto + From + Add, + Self: Debug + + Default + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + TryInto + + From + + Add, { } impl StorableVecIndex for I where - I: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto + From + Add + I: Debug + + Default + + Copy + + Clone + + PartialEq + + Eq + + PartialOrd + + Ord + + TryInto + + From + + Add { } diff --git a/storable_vec/src/traits/type_.rs b/storable_vec/src/traits/type_.rs index 5fdeeeeef..06e94efc0 100644 --- a/storable_vec/src/traits/type_.rs +++ b/storable_vec/src/traits/type_.rs @@ -1,8 +1,10 @@ use std::fmt::Debug; +use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; + pub trait StorableVecType where - Self: Sized + Debug + Clone, + Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout, { } -impl StorableVecType for T where T: Sized + Debug + Clone {} +impl StorableVecType for T where T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout {}