Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-09 14:41:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: from custom unsafe_slice to zerocopy

Browse Source
This commit is contained in:
nym21
2025-02-05 16:42:53 +01:00
parent 138ca80c10
commit d86d614520
43 changed files with 684 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+25 -9
View File
@@ -80,7 +80,7 @@ dependencies = [
"rapidhash",
"rayon",
"storable_vec",
"unsafe_slice_serde",
"zerocopy 0.8.16",
]
[[package]]
@@ -192,7 +192,7 @@ dependencies = [
"fjall",
"jiff",
"storable_vec",
"unsafe_slice_serde",
"zerocopy 0.8.16",
]
[[package]]
@@ -721,7 +721,7 @@ version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
"zerocopy",
"zerocopy 0.7.35",
]
[[package]]
@@ -945,7 +945,7 @@ name = "storable_vec"
version = "0.1.2"
dependencies = [
"memmap2",
"unsafe_slice_serde",
"zerocopy 0.8.16",
]
[[package]]
@@ -1061,10 +1061,6 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unsafe_slice_serde"
version = "0.1.0"
[[package]]
name = "valuable"
version = "0.1.1"
@@ -1185,7 +1181,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive",
"zerocopy-derive 0.7.35",
]
[[package]]
name = "zerocopy"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b8c07a70861ce02bad1607b5753ecb2501f67847b9f9ada7c160fff0ec6300c"
dependencies = [
"zerocopy-derive 0.8.16",
]
[[package]]
@@ -1198,3 +1203,14 @@ dependencies = [
"quote",
"syn 2.0.98",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.98",
]
Cargo.toml
+1 -3
View File
@@ -6,7 +6,6 @@ members = [
"iterator",
"storable_vec",
"struct_iterable",
"unsafe_slice_serde",
]
resolver = "2"
@@ -23,5 +22,4 @@ jiff = "0.1.29"
rayon = "1.10.0"
storable_vec = { path = "storable_vec" }
struct_iterable = { path = "struct_iterable" }
unsafe_slice_serde = { path = "unsafe_slice_serde" }
zerocopy = "0.8.15"
zerocopy = { version = "0.8.15", features = ["derive"] }
computer/Cargo.toml
+1 -1
View File
@@ -12,4 +12,4 @@ exit = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
storable_vec = { workspace = true }
unsafe_slice_serde = { workspace = true }
zerocopy = { workspace = true }
computer/src/lib.rs
+15 -3
View File
@@ -55,7 +55,7 @@ impl Computer<SINGLE_THREAD> {
.txindex_to_last_txinindex
.compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?;
self.vecs.txindex_to_inputcount.compute_count_from_indexes(
self.vecs.txindex_to_inputs_count.compute_count_from_indexes(
&mut indexer.vecs.txindex_to_first_txinindex,
&mut self.vecs.txindex_to_last_txinindex,
)?;
@@ -64,14 +64,16 @@ impl Computer<SINGLE_THREAD> {
.txindex_to_last_txoutindex
.compute_last_index_from_first(&mut indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?;
self.vecs.txindex_to_outputcount.compute_count_from_indexes(
self.vecs.txindex_to_outputs_count.compute_count_from_indexes(
&mut indexer.vecs.txindex_to_first_txoutindex,
&mut self.vecs.txindex_to_last_txoutindex,
)?;
self.vecs
.height_to_date
.compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| Date::from(timestamp))?;
.compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| {
Date::from(*timestamp)
})?;
self.vecs
.height_to_last_txindex
@@ -82,6 +84,16 @@ impl Computer<SINGLE_THREAD> {
&mut self.vecs.height_to_last_txindex,
)?;
self.vecs.txindex_to_is_coinbase.compute_is_first_ordered(
&mut self.vecs.txindex_to_height,
&mut indexer.vecs.height_to_first_txindex,
)?;
self.vecs.txindex_to_fee.compute_transform(
&mut self.vecs.txindex_to_height,
&mut indexer.vecs.height_to_first_txindex,
)?;
let date_count = self.vecs.height_to_date.len();
self.vecs
computer/src/storage/fjalls.rs
+2 -1
View File
@@ -1,6 +1,7 @@
use std::path::Path;
use bindex::{Store, Version};
use bindex::Store;
use storable_vec::Version;
use crate::structs::{AddressindexTxoutindex, Unit};
computer/src/storage/storable_vecs.rs
+95
View File
@@ -0,0 +1,95 @@
use std::{fs, path::Path};
use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::{StorableVec, Version};
use crate::structs::{Date, Feerate};
// mod base;
// use base::*;
pub struct StorableVecs<const MODE: u8> {
pub date_to_first_height: StorableVec<Date, Height, MODE>,
// pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
pub height_to_date: StorableVec<Height, Date, MODE>,
// pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
// pub height_to_inputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_last_txindex: StorableVec<Height, Txindex, MODE>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex, MODE>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_outputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_subsidy: StorableVec<Txindex, u32, MODE>,
// pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
// pub height_to_txcount: StorableVec<Txindex, u32, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate, MODE>,
pub txindex_to_inputs_count: StorableVec<Txindex, u32, MODE>,
pub txindex_to_inputs_sum: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_outputs_count: StorableVec<Txindex, u32, MODE>,
pub txindex_to_outputs_sum: StorableVec<Txindex, Amount, MODE>,
}
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
date_to_first_height: StorableVec::forced_import(&path.join("date_to_first_height"), Version::from(1))?,
// height_to_block_interval: StorableVec::forced_import(&path.join("height_to_block_interval"), Version::from(1))?,
height_to_date: StorableVec::forced_import(&path.join("height_to_date"), Version::from(1))?,
// height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::forced_import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::forced_import(
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
height_to_last_txindex: StorableVec::forced_import(&path.join("height_to_last_txindex"), Version::from(1))?,
// height_to_last_txoutindex: StorableVec::forced_import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_outputcount: StorableVec::forced_import(&path.join("height_to_outputcount"), Version::from(1))?,
// height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::forced_import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::forced_import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_is_coinbase: StorableVec::forced_import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
// txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputs_count: StorableVec::forced_import(
&path.join("txindex_to_inputs_count"),
Version::from(1),
)?,
txindex_to_inputs_sum: StorableVec::forced_import(&path.join("txindex_to_inputs_sum"), Version::from(1))?,
txindex_to_last_txinindex: StorableVec::forced_import(
&path.join("txindex_to_last_txinindex"),
Version::from(1),
)?,
txindex_to_last_txoutindex: StorableVec::forced_import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
)?,
txindex_to_outputs_count: StorableVec::forced_import(
&path.join("txindex_to_outputs_count"),
Version::from(1),
)?,
txindex_to_outputs_sum: StorableVec::forced_import(&path.join("txindex_to_outputs_sum"), Version::from(1))?,
})
}
// pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] {
// [&self.date_to_first_height]
// }
// pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] {
// [&mut self.date_to_first_height]
// }
}
computer/src/storage/storable_vecs/base.rs
-17
View File
@@ -1,17 +0,0 @@
use std::{fmt::Debug, path::Path};
use derive_deref::{Deref, DerefMut};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
#[derive(Debug, Deref, DerefMut)]
pub struct StorableVec<I, T, const MODE: u8>(storable_vec::StorableVec<I, T, MODE>);
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import(path: &Path, version: Version) -> storable_vec::Result<Self> {
Ok(Self(storable_vec::StorableVec::forced_import(path, version)?))
}
}
computer/src/storage/storable_vecs/mod.rs
-82
View File
@@ -1,82 +0,0 @@
use std::{fs, path::Path};
use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::Version;
use crate::structs::{Date, Feerate};
mod base;
use base::*;
pub struct StorableVecs<const MODE: u8> {
pub date_to_first_height: StorableVec<Date, Height, MODE>,
// pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
pub height_to_date: StorableVec<Height, Date, MODE>,
// pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
// pub height_to_inputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_last_txindex: StorableVec<Height, Txindex, MODE>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex, MODE>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_outputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_subsidy: StorableVec<Txindex, u32, MODE>,
// pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
// pub height_to_txcount: StorableVec<Txindex, u32, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate, MODE>,
pub txindex_to_inputcount: StorableVec<Txindex, u32, MODE>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_outputcount: StorableVec<Txindex, u32, MODE>,
}
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Ok(Self {
date_to_first_height: StorableVec::import(&path.join("date_to_first_height"), Version::from(1))?,
// height_to_block_interval: StorableVec::import(&path.join("height_to_block_interval"), Version::from(1))?,
height_to_date: StorableVec::import(&path.join("height_to_date"), Version::from(1))?,
// height_to_fee: StorableVec::import(&path.join("height_to_fee"), Version::from(1))?,
// height_to_inputcount: StorableVec::import(&path.join("height_to_inputcount"), Version::from(1))?,
// height_to_last_addressindex: StorableVec::import(
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
// height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?,
// height_to_minfeerate: StorableVec::import(&path.join("height_to_minfeerate"), Version::from(1))?,
// height_to_outputcount: StorableVec::import(&path.join("height_to_outputcount"), Version::from(1))?,
// height_to_subsidy: StorableVec::import(&path.join("height_to_subsidy"), Version::from(1))?,
// height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_is_coinbase: StorableVec::import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
// txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?,
txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?,
txindex_to_last_txoutindex: StorableVec::import(
&path.join("txindex_to_last_txoutindex"),
Version::from(1),
)?,
txindex_to_outputcount: StorableVec::import(&path.join("txindex_to_outputcount"), Version::from(1))?,
})
}
// pub fn as_slice(&self) -> [&dyn AnyComputedStorableVec; 1] {
// [&self.date_to_first_height]
// }
// pub fn as_mut_slice(&mut self) -> [&mut dyn AnyComputedStorableVec; 1] {
// [&mut self.date_to_first_height]
// }
}
computer/src/structs/addressindextxoutindex.rs
+6 -5
View File
@@ -1,21 +1,22 @@
use bindex::{Addressindex, Txoutindex};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct AddressindexTxoutindex {
addressindex: Addressindex,
_padding: u32,
txoutindex: Txoutindex,
}
impl TryFrom<Slice> for AddressindexTxoutindex {
type Error = unsafe_slice_serde::Error;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<AddressindexTxoutindex> for Slice {
fn from(value: AddressindexTxoutindex) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
computer/src/structs/date.rs
+57 -20
View File
@@ -2,11 +2,30 @@ use std::ops::Add;
use bindex::Timestamp;
use color_eyre::eyre::eyre;
use derive_deref::Deref;
use jiff::{civil::Date as _Date, tz::TimeZone, Span};
use jiff::{civil::Date as Date_, tz::TimeZone, Span};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
pub struct Date(_Date);
#[derive(Debug, Clone, Copy, PartialEq, Eq, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Date {
year: u16,
month: u8,
day: u8,
}
impl Date {
const INDEX_ZERO: Self = Self {
year: 2009,
month: 1,
day: 3,
};
const INDEX_ZERO_: Date_ = Date_::constant(2009, 1, 3);
const INDEX_ONE: Self = Self {
year: 2009,
month: 1,
day: 9,
};
const INDEX_ONE_: Date_ = Date_::constant(2009, 1, 9);
}
impl Default for Date {
fn default() -> Self {
@@ -14,49 +33,67 @@ impl Default for Date {
}
}
impl Date {
const INDEX_ZERO: Self = Self(_Date::constant(2009, 1, 3));
const INDEX_ONE: Self = Self(_Date::constant(2009, 1, 9));
}
impl From<_Date> for Date {
fn from(value: _Date) -> Self {
Self(value)
impl PartialOrd for Date {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl From<&Timestamp> for Date {
fn from(value: &Timestamp) -> Self {
Self(_Date::from(value.to_zoned(TimeZone::UTC)))
impl Ord for Date {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
Date_::from(*self).cmp(&Date_::from(*other))
}
}
impl From<Date_> for Date {
fn from(value: Date_) -> Self {
Self {
year: value.year() as u16,
month: value.month() as u8,
day: value.day() as u8,
}
}
}
impl From<Date> for Date_ {
fn from(value: Date) -> Self {
Self::new(value.year as i16, value.month as i8, value.day as i8).unwrap()
}
}
impl From<Timestamp> for Date {
fn from(value: Timestamp) -> Self {
Self::from(Date_::from(jiff::Timestamp::from(value).to_zoned(TimeZone::UTC)))
}
}
impl TryFrom<Date> for usize {
type Error = color_eyre::Report;
fn try_from(value: Date) -> Result<Self, Self::Error> {
if value < Date::INDEX_ZERO {
let value_ = Date_::from(value);
if value_ < Date::INDEX_ZERO_ {
Err(eyre!("Date is too early"))
} else if value == Date::INDEX_ZERO {
Ok(0)
} else if value < Date::INDEX_ONE {
} else if value_ < Date::INDEX_ONE_ {
Err(eyre!("Date is between first and second"))
} else if value == Date::INDEX_ONE {
Ok(1)
} else {
Ok(Date::INDEX_ONE.until(*value)?.get_days() as usize + 1)
Ok(Date_::from(Date::INDEX_ONE).until(value_)?.get_days() as usize + 1)
}
}
}
impl From<usize> for Date {
fn from(value: usize) -> Self {
Self(Self::INDEX_ZERO.checked_add(Span::new().days(value as i64)).unwrap())
Self::from(Self::INDEX_ZERO_.checked_add(Span::new().days(value as i64)).unwrap())
}
}
impl Add<usize> for Date {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0.checked_add(Span::new().days(rhs as i64)).unwrap())
Self::from(Date_::from(self).checked_add(Span::new().days(rhs as i64)).unwrap())
}
}
indexer/Cargo.toml
+1 -2
View File
@@ -13,6 +13,5 @@ fjall = { workspace = true }
jiff = { workspace = true }
rapidhash = "1.3.0"
rayon = { workspace = true }
# snkrj = { workspace = true }
storable_vec = { workspace = true }
unsafe_slice_serde = { workspace = true }
zerocopy = { workspace = true }
indexer/src/lib.rs
+17 -15
View File
@@ -9,7 +9,7 @@ use std::{
pub use biter::*;
use bitcoin::{Transaction, TxIn, TxOut, Txid};
use bitcoin::{Transaction, TxIn, TxOut};
use color_eyre::eyre::{eyre, ContextCompat};
use exit::Exit;
use rayon::prelude::*;
@@ -19,7 +19,7 @@ mod storage;
mod structs;
pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
pub use structs::Version;
use structs::{BlockHash, Txid};
use storage::{Fjalls, StorableVecs};
pub use structs::{
@@ -98,6 +98,7 @@ impl Indexer<CACHED_GETS> {
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
println!("Processing block {_height}...");
let blockhash = BlockHash::from(blockhash);
height = Height::from(_height);
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
@@ -124,9 +125,9 @@ impl Indexer<CACHED_GETS> {
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
vecs.height_to_difficulty.push_if_needed(height, block.header.difficulty_float())?;
vecs.height_to_timestamp.push_if_needed(height, Timestamp::try_from(block.header.time)?)?;
vecs.height_to_timestamp.push_if_needed(height, Timestamp::from(block.header.time))?;
vecs.height_to_size.push_if_needed(height, block.total_size())?;
vecs.height_to_weight.push_if_needed(height, block.weight())?;
vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
vecs.height_to_first_txindex.push_if_needed(height, txindex_global)?;
vecs.height_to_first_txinindex
.push_if_needed(height, txinindex_global)?;
@@ -192,7 +193,7 @@ impl Indexer<CACHED_GETS> {
.par_iter()
.enumerate()
.map(|(index, tx)| -> color_eyre::Result<_> {
let txid = tx.compute_txid();
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::try_from(&txid)?;
@@ -233,6 +234,7 @@ impl Indexer<CACHED_GETS> {
// dbg!((txindex, txinindex, vin));
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
if tx.is_coinbase() {
return Ok((txinindex, InputSource::SameBlock((tx, txindex, txin, vin))));
@@ -240,7 +242,7 @@ impl Indexer<CACHED_GETS> {
let prev_txindex = if let Some(txindex) = trees
.txid_prefix_to_txindex
.get(&TxidPrefix::try_from(&outpoint.txid)?)?
.get(&TxidPrefix::try_from(&txid)?)?
.map(|v| *v)
.and_then(|txindex| {
// Checking if not finding txindex from the future
@@ -537,7 +539,7 @@ impl Indexer<CACHED_GETS> {
}
let outpoint = txin.previous_output;
let txid = outpoint.txid;
let txid = Txid::from(outpoint.txid);
let vout = Vout::from(outpoint.vout);
let block_txindex = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
@@ -604,7 +606,7 @@ impl Indexer<CACHED_GETS> {
.get(prev_txindex)?
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, txid, len);
dbg!(txindex, len);
})?;
// #[allow(clippy::redundant_locals)]
@@ -614,12 +616,12 @@ impl Indexer<CACHED_GETS> {
// If another Txid needs to be added to the list
// We need to check that it's also a coinbase tx otherwise par_iter inputs needs to be updated
let only_known_dup_txids = [
Txid::from_str(
bitcoin::Txid::from_str(
"d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599",
)?,
Txid::from_str(
)?.into(),
bitcoin::Txid::from_str(
"e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468",
)?,
)?.into(),
];
let is_dup = only_known_dup_txids.contains(prev_txid);
@@ -627,7 +629,7 @@ impl Indexer<CACHED_GETS> {
if !is_dup {
let prev_height =
vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex);
dbg!(height, txindex, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}
}
@@ -640,10 +642,10 @@ impl Indexer<CACHED_GETS> {
txindex_to_tx_and_txid
.into_iter()
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version)?;
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version.into())?;
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
vecs.txindex_to_height.push_if_needed(txindex, height)?;
vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time)?;
vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time.into())?;
Ok(())
})?;
indexer/src/storage/fjalls/base.rs
+5 -5
View File
@@ -4,10 +4,10 @@ use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use storable_vec::Value;
use unsafe_slice_serde::UnsafeSliceSerde;
use storable_vec::{Value, Version};
use zerocopy::{Immutable, IntoBytes};
use crate::structs::{Height, Version};
use crate::structs::Height;
use super::StoreMeta;
@@ -21,7 +21,7 @@ pub struct Store<Key, Value> {
impl<K, V> Store<K, V>
where
K: Into<Slice> + Ord,
K: Into<Slice> + Ord + Immutable + IntoBytes,
V: Into<Slice> + TryFrom<Slice>,
<V as TryFrom<Slice>>::Error: error::Error + Send + Sync + 'static,
{
@@ -54,7 +54,7 @@ where
pub fn get(&self, key: &K) -> color_eyre::Result<Option<Value<V>>> {
if let Some(v) = self.puts.get(key) {
Ok(Some(Value::Ref(v)))
} else if let Some(slice) = self.rtx.get(&self.part, key.unsafe_as_slice())? {
} else if let Some(slice) = self.rtx.get(&self.part, key.as_bytes())? {
Ok(Some(Value::Owned(V::try_from(slice)?)))
} else {
Ok(None)
indexer/src/storage/fjalls/meta.rs
+5 -4
View File
@@ -3,9 +3,10 @@ use std::{
path::{Path, PathBuf},
};
use unsafe_slice_serde::UnsafeSliceSerde;
use storable_vec::Version;
use zerocopy::{FromBytes, IntoBytes};
use super::{Height, Version};
use super::Height;
pub struct StoreMeta {
pathbuf: PathBuf,
@@ -87,14 +88,14 @@ impl StoreMeta {
// }
fn read_length_(path: &Path) -> color_eyre::Result<usize> {
Ok(fs::read(Self::path_length(path))
.map(|v| usize::unsafe_try_from_slice(v.as_slice()).cloned().unwrap_or_default())
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())
.unwrap_or_default())
}
fn write_length(&self) -> io::Result<()> {
Self::write_length_(&self.pathbuf, self.len)
}
fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
fs::write(Self::path_length(path), len.to_le_bytes())
fs::write(Self::path_length(path), len.as_bytes())
}
fn path_length(path: &Path) -> PathBuf {
path.join("length")
indexer/src/storage/fjalls/mod.rs
+4 -1
View File
@@ -1,9 +1,12 @@
use std::{path::Path, thread};
use crate::{structs::Version, AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
use storable_vec::Version;
use crate::{AddressHash, Addressindex, BlockHashPrefix, Height, TxidPrefix, Txindex};
mod base;
mod meta;
// mod version;
pub use base::*;
pub use meta::*;
indexer/src/storage/fjalls/version.rs
+51
View File
@@ -0,0 +1,51 @@
use std::{fs, io, path::Path};
use derive_deref::Deref;
use fjall::Slice;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.to_ne_bytes())
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = io::Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Self::try_from(&fs::read(value)?)
}
}
impl TryFrom<Slice> for Version {
type Error = fjall::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Self::from(&value)
}
}
impl TryFrom<&[u8]> for Version {
type Error = storable_vec::Error;
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
let mut buf: [u8; 4] = [0; 4];
let buf_len = buf.len();
if value.len() != buf_len {
panic!();
}
value.iter().enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(Self(u32::from_ne_bytes(buf)))
}
}
impl From<Version> for Slice {
fn from(value: Version) -> Self {
Self::new(&value.to_ne_bytes())
}
}
indexer/src/storage/storable_vecs/base.rs
+3 -3
View File
@@ -34,7 +34,7 @@ where
self.vec.flush()
}
pub fn height(&self) -> color_eyre::Result<Height> {
pub fn height(&self) -> storable_vec::Result<Height> {
Height::try_from(self.path_height().as_path())
}
fn path_height(&self) -> PathBuf {
@@ -66,7 +66,7 @@ impl<I, T, const MODE: u8> DerefMut for StorableVec<I, T, MODE> {
}
pub trait AnyStorableVec {
fn height(&self) -> color_eyre::Result<Height>;
fn height(&self) -> storable_vec::Result<Height>;
fn flush(&mut self, height: Height) -> io::Result<()>;
}
@@ -75,7 +75,7 @@ where
I: StorableVecIndex,
T: StorableVecType,
{
fn height(&self) -> color_eyre::Result<Height> {
fn height(&self) -> storable_vec::Result<Height> {
self.height()
}
indexer/src/storage/storable_vecs/mod.rs
+5 -6
View File
@@ -1,14 +1,13 @@
use std::{fs, io, path::Path};
use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::{Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes,
P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex,
Txinindex, Txoutindex,
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, BlockHash, Height, LockTime, P2PK33AddressBytes,
P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes,
Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Weight,
};
mod base;
@@ -50,9 +49,9 @@ pub struct StorableVecs<const MODE: u8> {
pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_locktime: StorableVec<Txindex, bitcoin::absolute::LockTime, MODE>,
pub txindex_to_locktime: StorableVec<Txindex, LockTime, MODE>,
pub txindex_to_txid: StorableVec<Txindex, Txid, MODE>,
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version, MODE>,
pub txindex_to_txversion: StorableVec<Txindex, TxVersion, MODE>,
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex, MODE>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex, MODE>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount, MODE>,
indexer/src/structs/addressbytes.rs
+13 -12
View File
@@ -1,6 +1,7 @@
use biter::bitcoin::ScriptBuf;
use color_eyre::eyre::eyre;
use derive_deref::{Deref, DerefMut};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Addresstype;
@@ -122,28 +123,28 @@ impl From<P2TRAddressBytes> for Addressbytes {
}
}
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2PK65AddressBytes(U8x65);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2PK33AddressBytes(U8x33);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2PKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2SHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2WPKHAddressBytes(U8x20);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2WSHAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct P2TRAddressBytes(U8x32);
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x20([u8; 20]);
impl From<&[u8]> for U8x20 {
fn from(slice: &[u8]) -> Self {
@@ -153,7 +154,7 @@ impl From<&[u8]> for U8x20 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x32([u8; 32]);
impl From<&[u8]> for U8x32 {
fn from(slice: &[u8]) -> Self {
@@ -163,7 +164,7 @@ impl From<&[u8]> for U8x32 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x33([u8; 33]);
impl From<&[u8]> for U8x33 {
fn from(slice: &[u8]) -> Self {
@@ -173,7 +174,7 @@ impl From<&[u8]> for U8x33 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x64([u8; 64]);
impl From<&[u8]> for U8x64 {
fn from(slice: &[u8]) -> Self {
@@ -183,7 +184,7 @@ impl From<&[u8]> for U8x64 {
}
}
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct U8x65([u8; 65]);
impl From<&[u8]> for U8x65 {
fn from(slice: &[u8]) -> Self {
indexer/src/structs/addressindex.rs
+20 -5
View File
@@ -2,9 +2,24 @@ use std::ops::Add;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Addressindex(u32);
impl Addressindex {
@@ -52,14 +67,14 @@ impl From<Addressindex> for usize {
}
impl TryFrom<Slice> for Addressindex {
type Error = unsafe_slice_serde::Error;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<Addressindex> for Slice {
fn from(value: Addressindex) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
indexer/src/structs/addresstype.rs
+3 -1
View File
@@ -1,6 +1,8 @@
use biter::bitcoin::ScriptBuf;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, TryFromBytes, Immutable, IntoBytes, KnownLayout)]
#[repr(u8)]
pub enum Addresstype {
P2PK65,
P2PK33,
indexer/src/structs/addresstypeindex.rs
+17 -1
View File
@@ -1,8 +1,24 @@
use std::ops::Add;
use derive_deref::{Deref, DerefMut};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Addresstypeindex(u32);
impl Addresstypeindex {
indexer/src/structs/amount.rs
+34 -13
View File
@@ -5,14 +5,30 @@ use std::{
use biter::bitcoin;
use derive_deref::{Deref, DerefMut};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Amount(bitcoin::Amount);
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Amount(u64);
impl Amount {
pub const ZERO: Self = Self(bitcoin::Amount::ZERO);
pub const ZERO: Self = Self(0);
pub const ONE_BTC_F32: f32 = 100_000_000.0;
pub const ONE_BTC_F64: f64 = 100_000_000.0;
@@ -24,65 +40,70 @@ impl Amount {
impl Add for Amount {
type Output = Amount;
fn add(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() + rhs.to_sat())
Amount::from(*self + *rhs)
}
}
impl AddAssign for Amount {
fn add_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() + rhs.to_sat());
*self = *self + rhs;
}
}
impl Sub for Amount {
type Output = Amount;
fn sub(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() - rhs.to_sat())
Amount::from(*self - *rhs)
}
}
impl SubAssign for Amount {
fn sub_assign(&mut self, rhs: Self) {
*self = Amount::from(self.to_sat() - rhs.to_sat());
*self = *self - rhs;
}
}
impl Mul<Amount> for Amount {
type Output = Amount;
fn mul(self, rhs: Amount) -> Self::Output {
Amount::from(self.to_sat() * rhs.to_sat())
Amount::from(*self * *rhs)
}
}
impl Mul<u64> for Amount {
type Output = Amount;
fn mul(self, rhs: u64) -> Self::Output {
Amount::from(self.to_sat() * rhs)
Amount::from(*self * rhs)
}
}
impl Mul<Height> for Amount {
type Output = Amount;
fn mul(self, rhs: Height) -> Self::Output {
Amount::from(self.to_sat() * *rhs as u64)
Amount::from(*self * *rhs as u64)
}
}
impl Sum for Amount {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
let sats: u64 = iter.map(|amt| amt.to_sat()).sum();
let sats: u64 = iter.map(|amt| *amt).sum();
Amount::from(sats)
}
}
impl From<u64> for Amount {
fn from(value: u64) -> Self {
Self(bitcoin::Amount::from_sat(value))
Self(value)
}
}
impl From<bitcoin::Amount> for Amount {
fn from(value: bitcoin::Amount) -> Self {
Self(value)
Self(value.to_sat())
}
}
impl From<Amount> for bitcoin::Amount {
fn from(value: Amount) -> Self {
Self::from_sat(value.0)
}
}
indexer/src/structs/blockhash.rs
+19
View File
@@ -0,0 +1,19 @@
use std::mem;
use derive_deref::Deref;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct BlockHash([u8; 32]);
impl From<bitcoin::BlockHash> for BlockHash {
fn from(value: bitcoin::BlockHash) -> Self {
unsafe { mem::transmute(value) }
}
}
impl From<BlockHash> for bitcoin::BlockHash {
fn from(value: BlockHash) -> Self {
unsafe { mem::transmute(value) }
}
}
indexer/src/structs/compressed.rs
+28 -17
View File
@@ -1,13 +1,12 @@
use std::hash::Hasher;
use biter::bitcoin::{BlockHash, Txid};
use derive_deref::Deref;
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Addressbytes, Addresstype, SliceExtended};
use super::{Addressbytes, Addresstype, BlockHash, Txid};
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct AddressHash([u8; 8]);
impl From<(&Addressbytes, Addresstype)> for AddressHash {
fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self {
@@ -24,14 +23,14 @@ impl From<[u8; 8]> for AddressHash {
}
}
impl TryFrom<Slice> for AddressHash {
type Error = color_eyre::Report;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<&AddressHash> for Slice {
fn from(value: &AddressHash) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
impl From<AddressHash> for Slice {
@@ -40,23 +39,23 @@ impl From<AddressHash> for Slice {
}
}
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct BlockHashPrefix([u8; 8]);
impl TryFrom<&BlockHash> for BlockHashPrefix {
type Error = color_eyre::Report;
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
Ok(Self(copy_first_8bytes(&value[..])))
}
}
impl TryFrom<Slice> for BlockHashPrefix {
type Error = color_eyre::Report;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<&BlockHashPrefix> for Slice {
fn from(value: &BlockHashPrefix) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
impl From<BlockHashPrefix> for Slice {
@@ -65,23 +64,23 @@ impl From<BlockHashPrefix> for Slice {
}
}
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct TxidPrefix([u8; 8]);
impl TryFrom<&Txid> for TxidPrefix {
type Error = color_eyre::Report;
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
Ok(Self((&value[..]).read_8x_u8()?))
Ok(Self(copy_first_8bytes(&value[..])))
}
}
impl TryFrom<Slice> for TxidPrefix {
type Error = color_eyre::Report;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<&TxidPrefix> for Slice {
fn from(value: &TxidPrefix) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
impl From<TxidPrefix> for Slice {
@@ -89,3 +88,15 @@ impl From<TxidPrefix> for Slice {
Self::from(&value)
}
}
fn copy_first_8bytes(slice: &[u8]) -> [u8; 8] {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
panic!("bad len");
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
buf
}
indexer/src/structs/height.rs
+35 -8
View File
@@ -7,14 +7,29 @@ use std::{
use biter::rpc::{self, RpcApi};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
#[derive(
Debug,
Clone,
Copy,
Deref,
DerefMut,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Height(u32);
impl Height {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
fs::write(path, self.as_bytes())
}
}
@@ -113,9 +128,9 @@ impl From<Height> for usize {
}
impl TryFrom<&Path> for Height {
type Error = color_eyre::Report;
type Error = storable_vec::Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
Ok(Self::read_from_bytes(fs::read(value)?.as_slice())?.to_owned())
}
}
@@ -127,13 +142,25 @@ impl TryFrom<&rpc::Client> for Height {
}
impl TryFrom<Slice> for Height {
type Error = unsafe_slice_serde::Error;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<Height> for Slice {
fn from(value: Height) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
impl From<bitcoin::locktime::absolute::Height> for Height {
fn from(value: bitcoin::locktime::absolute::Height) -> Self {
Self(value.to_consensus_u32())
}
}
impl From<Height> for bitcoin::locktime::absolute::Height {
fn from(value: Height) -> Self {
bitcoin::locktime::absolute::Height::from_consensus(*value).unwrap()
}
}
indexer/src/structs/locktime.rs
+28
View File
@@ -0,0 +1,28 @@
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
use super::{Height, Timestamp};
#[derive(Debug, Immutable, Clone, Copy, IntoBytes, KnownLayout, TryFromBytes)]
#[repr(C)]
pub enum LockTime {
Height(Height),
Timestamp(Timestamp),
}
impl From<bitcoin::absolute::LockTime> for LockTime {
fn from(value: bitcoin::absolute::LockTime) -> Self {
match value {
bitcoin::absolute::LockTime::Blocks(h) => LockTime::Height(h.into()),
bitcoin::absolute::LockTime::Seconds(t) => LockTime::Timestamp(t.into()),
}
}
}
impl From<LockTime> for bitcoin::absolute::LockTime {
fn from(value: LockTime) -> Self {
match value {
LockTime::Height(h) => bitcoin::absolute::LockTime::Blocks(h.into()),
LockTime::Timestamp(t) => bitcoin::absolute::LockTime::Seconds(t.into()),
}
}
}
indexer/src/structs/mod.rs
+10 -4
View File
@@ -3,29 +3,35 @@ mod addressindex;
mod addresstype;
mod addresstypeindex;
mod amount;
mod blockhash;
mod compressed;
mod height;
mod slice;
mod locktime;
mod timestamp;
mod txid;
mod txindex;
mod txinindex;
mod txoutindex;
mod version;
mod txversion;
mod vin;
mod vout;
mod weight;
pub use addressbytes::*;
pub use addressindex::*;
pub use addresstype::*;
pub use addresstypeindex::*;
pub use amount::*;
pub use blockhash::*;
pub use compressed::*;
pub use height::*;
pub use slice::*;
pub use locktime::*;
pub use timestamp::*;
pub use txid::*;
pub use txindex::*;
pub use txinindex::*;
pub use txoutindex::*;
pub use version::*;
pub use txversion::*;
pub use vin::*;
pub use vout::*;
pub use weight::*;
indexer/src/structs/slice.rs
-55
View File
@@ -1,55 +0,0 @@
use color_eyre::eyre::eyre;
#[allow(unused)]
pub trait SliceExtended {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]>;
fn read_be_u8(&self) -> color_eyre::Result<u8>;
fn read_be_u16(&self) -> color_eyre::Result<u16>;
fn read_be_u32(&self) -> color_eyre::Result<u32>;
fn read_be_u64(&self) -> color_eyre::Result<u64>;
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()>;
}
impl SliceExtended for &[u8] {
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
(&self[..8]).read_exact(&mut buf)?;
Ok(buf)
}
fn read_be_u8(&self) -> color_eyre::Result<u8> {
let mut buf: [u8; 1] = [0; 1];
self.read_exact(&mut buf)?;
Ok(u8::from_be_bytes(buf))
}
fn read_be_u16(&self) -> color_eyre::Result<u16> {
let mut buf: [u8; 2] = [0; 2];
self.read_exact(&mut buf)?;
Ok(u16::from_be_bytes(buf))
}
fn read_be_u32(&self) -> color_eyre::Result<u32> {
let mut buf: [u8; 4] = [0; 4];
self.read_exact(&mut buf)?;
Ok(u32::from_be_bytes(buf))
}
fn read_be_u64(&self) -> color_eyre::Result<u64> {
let mut buf: [u8; 8] = [0; 8];
self.read_exact(&mut buf)?;
Ok(u64::from_be_bytes(buf))
}
fn read_exact(&self, buf: &mut [u8]) -> color_eyre::Result<()> {
let buf_len = buf.len();
if self.len() != buf_len {
dbg!(self.len(), buf_len);
return Err(eyre!("Not exact len"));
}
self.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
});
Ok(())
}
}
indexer/src/structs/timestamp.rs
+24 -6
View File
@@ -1,11 +1,29 @@
use derive_deref::Deref;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Timestamp(jiff::Timestamp);
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, Immutable, IntoBytes, KnownLayout)]
pub struct Timestamp(u32);
impl TryFrom<u32> for Timestamp {
type Error = jiff::Error;
fn try_from(value: u32) -> Result<Self, Self::Error> {
Ok(Self(jiff::Timestamp::from_second(value as i64)?))
impl From<u32> for Timestamp {
fn from(value: u32) -> Self {
Self(value)
}
}
impl From<Timestamp> for jiff::Timestamp {
fn from(value: Timestamp) -> Self {
jiff::Timestamp::from_second(*value as i64).unwrap()
}
}
impl From<bitcoin::locktime::absolute::Time> for Timestamp {
fn from(value: bitcoin::locktime::absolute::Time) -> Self {
Self(value.to_consensus_u32())
}
}
impl From<Timestamp> for bitcoin::locktime::absolute::Time {
fn from(value: Timestamp) -> Self {
bitcoin::locktime::absolute::Time::from_consensus(*value).unwrap()
}
}
indexer/src/structs/txid.rs
+19
View File
@@ -0,0 +1,19 @@
use std::mem;
use derive_deref::Deref;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, PartialEq, Eq, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct Txid([u8; 32]);
impl From<bitcoin::Txid> for Txid {
fn from(value: bitcoin::Txid) -> Self {
unsafe { mem::transmute(value) }
}
}
impl From<Txid> for bitcoin::Txid {
fn from(value: Txid) -> Self {
unsafe { mem::transmute(value) }
}
}
indexer/src/structs/txindex.rs
+20 -5
View File
@@ -2,9 +2,24 @@ use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Txindex(u32);
impl Txindex {
@@ -73,13 +88,13 @@ impl From<Txindex> for usize {
}
impl TryFrom<Slice> for Txindex {
type Error = unsafe_slice_serde::Error;
type Error = storable_vec::Error;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
Ok(Self::read_from_bytes(&value)?)
}
}
impl From<Txindex> for Slice {
fn from(value: Txindex) -> Self {
Self::new(value.unsafe_as_slice())
Self::new(value.as_bytes())
}
}
indexer/src/structs/txinindex.rs
+17 -1
View File
@@ -1,10 +1,26 @@
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Vin;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Txinindex(u64);
impl Txinindex {
indexer/src/structs/txoutindex.rs
+17 -1
View File
@@ -1,10 +1,26 @@
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Vout;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Deref,
DerefMut,
Default,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct Txoutindex(u64);
impl Txoutindex {
indexer/src/structs/txversion.rs
+17
View File
@@ -0,0 +1,17 @@
use derive_deref::Deref;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct TxVersion(i32);
impl From<bitcoin::transaction::Version> for TxVersion {
fn from(value: bitcoin::transaction::Version) -> Self {
Self(value.0)
}
}
impl From<TxVersion> for bitcoin::transaction::Version {
fn from(value: TxVersion) -> Self {
Self(value.0)
}
}
indexer/src/structs/version.rs
-38
View File
@@ -1,38 +0,0 @@
use std::{fs, io, path::Path};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.unsafe_as_slice())
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = color_eyre::Report;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
}
}
impl TryFrom<Slice> for Version {
type Error = color_eyre::Report;
fn try_from(value: Slice) -> Result<Self, Self::Error> {
Ok(*Self::unsafe_try_from_slice(&value)?)
}
}
impl From<Version> for Slice {
fn from(value: Version) -> Self {
Self::new(value.unsafe_as_slice())
}
}
indexer/src/structs/weight.rs
+17
View File
@@ -0,0 +1,17 @@
use derive_deref::Deref;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes)]
pub struct Weight(u64);
impl From<bitcoin::Weight> for Weight {
fn from(value: bitcoin::Weight) -> Self {
Self(value.to_wu())
}
}
impl From<Weight> for bitcoin::Weight {
fn from(value: Weight) -> Self {
Self::from_wu(*value)
}
}
storable_vec/Cargo.toml
+1 -1
View File
@@ -9,4 +9,4 @@ edition = "2021"
[dependencies]
memmap2 = "0.9.5"
unsafe_slice_serde = { workspace = true }
zerocopy = { workspace = true }
storable_vec/src/enums/error.rs
+13 -5
View File
@@ -13,7 +13,8 @@ pub enum Error {
DifferentVersion { found: Version, expected: Version },
MmapsVecIsTooSmall,
IO(io::Error),
UnsafeSliceSerde(unsafe_slice_serde::Error),
// UnsafeSliceSerde(zerocopy::error::),
ZeroCopyError,
IndexTooHigh,
IndexTooLow,
ExpectFileToHaveIndex,
@@ -28,9 +29,15 @@ impl From<io::Error> for Error {
}
}
impl From<unsafe_slice_serde::Error> for Error {
fn from(value: unsafe_slice_serde::Error) -> Self {
Self::UnsafeSliceSerde(value)
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
@@ -43,7 +50,7 @@ impl fmt::Display for Error {
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
// Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -52,6 +59,7 @@ impl fmt::Display for Error {
Error::UnsupportedUnflushedState => {
write!(f, "Unsupported unflush state, please flush before using this function")
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
}
}
}
storable_vec/src/lib.rs
+9 -8
View File
@@ -11,8 +11,8 @@ use std::{
sync::OnceLock,
};
use memmap2::{Mmap, MmapOptions};
use unsafe_slice_serde::UnsafeSliceSerde;
pub use memmap2;
pub use zerocopy;
mod enums;
mod structs;
@@ -60,7 +60,7 @@ pub struct StorableVec<I, T, const MODE: u8> {
file_position: u64,
buf: Buffer,
/// Only for CACHED_GETS
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
cache: Vec<OnceLock<Box<memmap2::Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
pushed: Vec<T>,
// updated: BTreeMap<usize, T>,
// inserted: BTreeMap<usize, T>,
@@ -207,7 +207,7 @@ where
fn read_exact<'a>(file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> {
file.read_exact(buf)?;
let v = T::unsafe_try_from_slice(&buf[..])?;
let v = T::try_ref_from_bytes(&buf[..])?;
Ok(v)
}
@@ -277,7 +277,7 @@ where
mem::take(&mut self.pushed)
.into_iter()
.for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
.for_each(|v| bytes.extend_from_slice(v.as_bytes()));
self.file.write_all(&bytes)?;
@@ -383,7 +383,7 @@ where
.ok_or(Error::MmapsVecIsTooSmall)?
.get_or_init(|| {
Box::new(unsafe {
MmapOptions::new()
memmap2::MmapOptions::new()
.len(Self::PAGE_SIZE)
.offset((page_index * Self::PAGE_SIZE) as u64)
.map(&self.file)
@@ -393,7 +393,7 @@ where
let range = Self::index_to_byte_range(index);
let slice = &mmap[range];
return Ok(Some(Value::Ref(T::unsafe_try_from_slice(slice)?)));
return Ok(Some(Value::Ref(T::try_ref_from_bytes(slice)?)));
}
Ok(Some(Value::Owned(self.open_file_at_then_read(index)?.to_owned())))
@@ -581,8 +581,9 @@ where
other_to_self: &mut StorableVec<A, I, SINGLE_THREAD>,
) -> Result<()>
where
A: StorableVecIndex + StorableVecType,
I: StorableVecType,
T: From<bool>,
A: StorableVecIndex + StorableVecType,
{
self_to_other.iter_from(I::from(self.len()), |(i, other)| {
self.push_if_needed(i, T::from(other_to_self.get(*other)? == &i))
storable_vec/src/structs/version.rs
+4 -4
View File
@@ -4,16 +4,16 @@ use std::{
path::Path,
};
use unsafe_slice_serde::UnsafeSliceSerde;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::Error;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromBytes, IntoBytes, Immutable, KnownLayout)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.0.unsafe_as_slice())
fs::write(path, self.as_bytes())
}
pub fn swap_bytes(self) -> Self {
@@ -32,6 +32,6 @@ impl TryFrom<&Path> for Version {
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 4];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(*(Self::unsafe_try_from_slice(&buf)?))
Ok(*(Self::ref_from_bytes(&buf)?))
}
}
storable_vec/src/traits/index.rs
+22 -2
View File
@@ -2,10 +2,30 @@ use std::{fmt::Debug, ops::Add};
pub trait StorableVecIndex
where
Self: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto<usize> + From<usize> + Add<usize, Output = Self>,
Self: Debug
+ Default
+ Copy
+ Clone
+ PartialEq
+ Eq
+ PartialOrd
+ Ord
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>,
{
}
impl<I> StorableVecIndex for I where
I: Debug + Default + Copy + Clone + PartialOrd + Ord + TryInto<usize> + From<usize> + Add<usize, Output = Self>
I: Debug
+ Default
+ Copy
+ Clone
+ PartialEq
+ Eq
+ PartialOrd
+ Ord
+ TryInto<usize>
+ From<usize>
+ Add<usize, Output = Self>
{
}
storable_vec/src/traits/type_.rs
+4 -2
View File
@@ -1,8 +1,10 @@
use std::fmt::Debug;
use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
pub trait StorableVecType
where
Self: Sized + Debug + Clone,
Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout,
{
}
impl<T> StorableVecType for T where T: Sized + Debug + Clone {}
impl<T> StorableVecType for T where T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout {}