Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-12 16:03:31 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-02 23:28:03 +01:00
parent ad34d9d402
commit 1e37d75e49
28 changed files with 660 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+10 -48
View File
@@ -50,8 +50,8 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
dependencies = [
"bitcoin-internals 0.3.0",
"bitcoin_hashes 0.14.0",
"bitcoin-internals",
"bitcoin_hashes",
]
[[package]]
@@ -71,7 +71,6 @@ name = "bindex"
version = "0.1.0"
dependencies = [
"bitcoin",
"bitcoin_hashes 0.16.0",
"biter",
"color-eyre",
"derive_deref",
@@ -92,11 +91,11 @@ checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
dependencies = [
"base58ck",
"bech32",
"bitcoin-internals 0.3.0",
"bitcoin-io 0.1.3",
"bitcoin-internals",
"bitcoin-io",
"bitcoin-units",
"bitcoin_hashes 0.14.0",
"hex-conservative 0.2.1",
"bitcoin_hashes",
"hex-conservative",
"hex_lit",
"secp256k1",
"serde",
@@ -111,34 +110,19 @@ dependencies = [
"serde",
]
[[package]]
name = "bitcoin-internals"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b854212e29b96c8f0fe04cab11d57586c8f3257de0d146c76cb3b42b3eb9118"
[[package]]
name = "bitcoin-io"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf"
[[package]]
name = "bitcoin-io"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26792cd2bf245069a1c5acb06aa7ad7abe1de69b507c90b490bca81e0665d0ee"
dependencies = [
"bitcoin-internals 0.4.0",
]
[[package]]
name = "bitcoin-units"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
dependencies = [
"bitcoin-internals 0.3.0",
"bitcoin-internals",
"serde",
]
@@ -148,21 +132,11 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
dependencies = [
"bitcoin-io 0.1.3",
"hex-conservative 0.2.1",
"bitcoin-io",
"hex-conservative",
"serde",
]
[[package]]
name = "bitcoin_hashes"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e5d09f16329cd545d7e6008b2c6b2af3a90bc678cf41ac3d2f6755943301b16"
dependencies = [
"bitcoin-io 0.2.0",
"hex-conservative 0.3.0",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
@@ -217,7 +191,6 @@ dependencies = [
"exit",
"fjall",
"jiff",
"rayon",
"storable_vec",
"unsafe_slice_serde",
]
@@ -494,15 +467,6 @@ dependencies = [
"arrayvec",
]
[[package]]
name = "hex-conservative"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4afe881d0527571892c4034822e59bb10c6c991cce6abe8199b6f5cf10766f55"
dependencies = [
"arrayvec",
]
[[package]]
name = "hex_lit"
version = "0.1.1"
@@ -896,7 +860,7 @@ version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
dependencies = [
"bitcoin_hashes 0.14.0",
"bitcoin_hashes",
"rand",
"secp256k1-sys",
"serde",
@@ -996,9 +960,7 @@ dependencies = [
name = "struct_iterable_derive"
version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"struct_iterable_internal",
"syn 2.0.96",
]
computer/Cargo.toml
-1
View File
@@ -11,6 +11,5 @@ derive_deref = { workspace = true }
exit = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
rayon = { workspace = true }
storable_vec = { workspace = true }
unsafe_slice_serde = { workspace = true }
computer/src/lib.rs
+48 -19
View File
@@ -1,6 +1,6 @@
use std::path::{Path, PathBuf};
use bindex::{Height, Indexer};
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
@@ -36,34 +36,63 @@ impl Computer {
indexer.index(bitcoin_dir, rpc, exit)?;
}
let height_count = indexer.vecs().height_to_size.len();
let txindexes_count = indexer.vecs().txindex_to_txid.len();
let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// TODO: Remove all outdated
// Compute txindex to X
self.vecs
.txindex_to_last_txinindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txinindex, txinindexes_count)?;
self.vecs.txindex_to_inputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txinindex,
&self.vecs.txindex_to_last_txinindex,
)?;
self.vecs
.txindex_to_last_txoutindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txoutindex, txoutindexes_count)?;
self.vecs.txindex_to_outputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txoutindex,
&self.vecs.txindex_to_last_txoutindex,
)?;
// Compute height to X
indexer
.vecs()
.height_to_timestamp
.read_from_(self.vecs.height_to_date.len(), |(_height, timestamp)| {
self.vecs
.height_to_date
.push_if_needed(Height::from(_height), Date::from(timestamp))
})?;
self.vecs
.height_to_date
.read_from_(self.vecs.date_to_first_height.len(), |(_height, date)| {
self.vecs
.date_to_first_height
.push_if_needed(*date, Height::from(_height))
})?;
.compute_transform(&indexer.vecs().height_to_timestamp, |timestamp| Date::from(timestamp))?;
// Compute date to X
self.vecs
.height_to_last_txindex
.compute_last_index_from_first(&indexer.vecs().height_to_first_txindex, height_count)?;
self.vecs.txindex_to_height.compute_inverse_less_to_more(
&indexer.vecs().height_to_first_txindex,
&self.vecs.height_to_last_txindex,
)?;
let date_count = self.vecs.height_to_date.len();
self.vecs
.date_to_first_height
.compute_inverse_more_to_less(&self.vecs.height_to_date)?;
// ---
// Date to X
// ---
// ...
// Compute month to X
// ---
// Month to X
// ---
// ...
// Compute year to X
// ---
// Year to X
// ---
// ...
Ok(())
computer/src/storage/fjalls.rs
+6 -6
View File
@@ -5,18 +5,18 @@ use bindex::{Store, Version};
use crate::structs::{AddressindexTxoutindex, Unit};
pub struct Fjalls {
pub address_txoutindex_in: Store<AddressindexTxoutindex, Unit>,
pub address_txoutindex_out: Store<AddressindexTxoutindex, Unit>,
pub address_to_utxos_received: Store<AddressindexTxoutindex, Unit>,
pub address_to_utxos_spent: Store<AddressindexTxoutindex, Unit>,
}
impl Fjalls {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
let address_txoutindex_in = Store::import(&path.join("address_txoutindex_in"), Version::from(1))?;
let address_txoutindex_out = Store::import(&path.join("address_txoutindex_out"), Version::from(1))?;
let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?;
let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?;
Ok(Self {
address_txoutindex_in,
address_txoutindex_out,
address_to_utxos_received,
address_to_utxos_spent,
})
}
}
computer/src/storage/storable_vecs/base.rs
+123 -53
View File
@@ -1,74 +1,144 @@
use std::{
error,
fmt::Debug,
io,
ops::{Deref, DerefMut},
ops::{Add, Sub},
path::Path,
};
use bindex::{Indexer, Version};
use derive_deref::{Deref, DerefMut};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use crate::Computer;
#[derive(Debug, Deref, DerefMut)]
pub struct StorableVec<I, T>(storable_vec::StorableVec<I, T>);
pub struct StorableVec<I, T> {
vec: bindex::StorableVec<I, T>,
f: Box<dyn Fn(&Indexer, &Computer) -> storable_vec::Result<Vec<(I, T)>>>,
}
const FLUSH_EVERY: usize = 10_000;
impl<I, T> StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import<F>(path: &Path, version: Version, f: F) -> io::Result<Self>
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
Ok(Self(storable_vec::StorableVec::import(path, version)?))
}
fn flush_vec_if_needed(&mut self) -> io::Result<()> {
if self.pushed_len() == FLUSH_EVERY {
self.flush()
} else {
Ok(())
}
}
pub fn compute_inverse_more_to_less(&mut self, other: &storable_vec::StorableVec<T, I>) -> storable_vec::Result<()>
where
F: Fn(&Indexer, &Computer) -> storable_vec::Result<Vec<(I, T)>> + 'static,
I: StorableVecType,
T: StorableVecIndex,
{
let vec = bindex::StorableVec::import(path, version)?;
Ok(Self { vec, f: Box::new(f) })
other.iter_from(self.last()?.map(|v| *v).unwrap_or_default(), |(v, i)| {
self.push_if_needed(*i, v)
})
}
pub fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> {
(self.f)(indexer, computer)?
.into_iter()
.try_for_each(|(i, v)| self.push_if_needed(i, v))
pub fn compute_inverse_less_to_more(
&mut self,
first_indexes: &storable_vec::StorableVec<T, I>,
last_indexes: &storable_vec::StorableVec<T, I>,
) -> color_eyre::Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(T::from(self.len()), |(value, first_index)| {
let first_index: usize = (*first_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let last_index: usize = (*last_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
(first_index..last_index).try_for_each(|index| self.push_if_needed(I::from(index), value))?;
Ok(())
})?;
self.flush()?;
Ok(())
}
// pub fn fill(&mut self) {
// self
// .vecs()
// .height_to_timestamp
// .read_iter(move |(_height, timestamp)| {
// let height = Height::from(_height);
// let date = Date::from(timestamp);
// self.vecs.date_to_first_height.push_if_needed(date, height)?;
// Ok(())
// })?;
// }
}
pub fn compute_transform<A, F>(&mut self, other: &storable_vec::StorableVec<I, A>, t: F) -> storable_vec::Result<()>
where
A: StorableVecType,
F: Fn(&A) -> T,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = bindex::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
pub trait AnyComputedStorableVec {
fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()>;
}
impl<I, T> AnyComputedStorableVec for StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
{
fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> {
self.compute(indexer, computer)
pub fn compute_is_first_ordered<A>(
&mut self,
self_to_other: &storable_vec::StorableVec<I, A>,
other_to_self: &storable_vec::StorableVec<A, I>,
) -> storable_vec::Result<()>
where
A: StorableVecIndex + StorableVecType,
{
// let mut prev_a_opt = None;
// self_to_other.iter_from(I::from(self.len()), |(i, a)| {
// if prev_a_opt.is_none() {
// prev_a_opt.replace(a);
// self.push_if_needed(i, other_to_self.read_at(a) == i);
// } else {
// let prev_a = prev_a_opt.unwrap();
// if a != prev_a
// }
// other_to_self.seek_read(a);
// self.push_if_needed(i, t(a));
// Ok(())
// })
Ok(())
}
pub fn compute_last_index_from_first(
&mut self,
first_index_vec: &storable_vec::StorableVec<I, T>,
final_len: usize,
) -> color_eyre::Result<()>
where
T: Copy + From<usize> + Sub<T, Output = T> + StorableVecIndex,
{
let mut prev_index: Option<I> = None;
first_index_vec.iter_from(I::from(self.len()), |(i, v)| {
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, *v - T::from(1))?;
}
prev_index.replace(i);
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
})?;
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, T::from(final_len) - T::from(1))?;
}
self.flush()?;
Ok(())
}
pub fn compute_count_from_indexes<T2>(
&mut self,
first_indexes: &storable_vec::StorableVec<I, T2>,
last_indexes: &storable_vec::StorableVec<I, T2>,
) -> color_eyre::Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())?;
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
})?;
self.flush()?;
Ok(())
}
}
computer/src/storage/storable_vecs/mod.rs
+10 -5
View File
@@ -1,12 +1,13 @@
use std::{fs, path::Path};
use bindex::{Addressindex, Amount, Height, StorableVec, Timestamp, Txindex, Txinindex, Txoutindex, Version};
use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::Version;
use crate::structs::{Date, Feerate};
// mod base;
mod base;
// use base::*;
use base::*;
pub struct StorableVecs {
pub date_to_first_height: StorableVec<Date, Height>,
@@ -15,7 +16,7 @@ pub struct StorableVecs {
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Txindex, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
// pub height_to_last_txindex: StorableVec<Height, Txindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate>,
@@ -25,6 +26,8 @@ pub struct StorableVecs {
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_txcount: StorableVec<Txindex, u32>,
pub txindex_to_fee: StorableVec<Txindex, Amount>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_inputcount: StorableVec<Txindex, u32>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
@@ -46,7 +49,7 @@ impl StorableVecs {
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
// height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
// height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?,
@@ -56,6 +59,8 @@ impl StorableVecs {
// height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_is_coinbase: StorableVec::import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
// txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?,
txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?,
computer/src/structs/date.rs
+17 -2
View File
@@ -1,9 +1,11 @@
use std::ops::Add;
use bindex::Timestamp;
use color_eyre::eyre::eyre;
use derive_deref::Deref;
use jiff::{civil::Date as _Date, tz::TimeZone};
use jiff::{civil::Date as _Date, tz::TimeZone, Span};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
pub struct Date(_Date);
impl Date {
@@ -39,3 +41,16 @@ impl TryFrom<Date> for usize {
}
}
}
impl From<usize> for Date {
fn from(value: usize) -> Self {
Self(Self::INDEX_ZERO.checked_add(Span::new().days(value as i64)).unwrap())
}
}
impl Add<usize> for Date {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0.checked_add(Span::new().days(rhs as i64)).unwrap())
}
}
indexer/Cargo.toml
-1
View File
@@ -5,7 +5,6 @@ edition = "2021"
[dependencies]
bitcoin = { workspace = true }
bitcoin_hashes = "0.16.0"
biter = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
indexer/src/lib.rs
+6 -6
View File
@@ -104,7 +104,7 @@ impl Indexer {
height = Height::from(_height);
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if let Some(saved_blockhash) = vecs.height_to_blockhash.cached_get(height)? {
if &blockhash != saved_blockhash.as_ref() {
todo!("Rollback not implemented");
// trees.rollback_from(&mut rtx, height, &exit)?;
@@ -260,7 +260,7 @@ impl Indexer {
let txoutindex = *vecs
.txindex_to_first_txoutindex
.get(prev_txindex)?
.cached_get(prev_txindex)?
.context("Expect txoutindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
@@ -336,12 +336,12 @@ impl Indexer {
let prev_addresstype = *vecs
.addressindex_to_addresstype
.get(addressindex)?
.cached_get(addressindex)?
.context("Expect to have address type")?;
let addresstypeindex = *vecs
.addressindex_to_addresstypeindex
.get(addressindex)?
.cached_get(addressindex)?
.context("Expect to have address type index")?;
// Good first time
// Wrong after rerun
@@ -605,7 +605,7 @@ impl Indexer {
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.get(prev_txindex)?
.cached_get(prev_txindex)?
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, txid, len);
@@ -630,7 +630,7 @@ impl Indexer {
if !is_dup {
let prev_height =
vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
vecs.txindex_to_height.cached_get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}
indexer/src/storage/storable_vecs/base.rs
+17 -47
View File
@@ -1,79 +1,44 @@
use std::{
fmt::Debug,
fs, io,
io,
ops::{Deref, DerefMut},
path::{Path, PathBuf},
};
use super::{Height, Version};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use super::Height;
#[derive(Debug)]
pub struct StorableVec<I, T> {
height: Option<Height>,
pathbuf: PathBuf,
version: Version,
vec: storable_vec::StorableVec<I, T>,
}
impl<I, T> StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
fs::create_dir_all(path)?;
let pathbuf = path.to_owned();
let path_vec = Self::path_vec_(path);
let path_version = Self::path_version_(path);
let is_same_version =
Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
let _ = fs::remove_file(&path_vec);
let _ = fs::remove_file(&path_version);
let _ = fs::remove_file(Self::path_height_(path));
}
let this = Self {
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
pathbuf,
version,
vec: storable_vec::StorableVec::import(&path_vec)?,
};
this.version.write(&this.path_version())?;
Ok(this)
vec: storable_vec::StorableVec::import(path, version)?,
})
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
if self.needs(height) {
height.write(&self.path_height())?;
}
self.vec.flush()
}
// fn path_vec(&self) -> PathBuf {
// Self::_path_vec(&self.path)
// }
fn path_vec_(path: &Path) -> PathBuf {
path.join("vec")
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
pub fn height(&self) -> color_eyre::Result<Height> {
Height::try_from(self.path_height().as_path())
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
Self::path_height_(self.vec.path())
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
@@ -103,17 +68,22 @@ impl<I, T> DerefMut for StorableVec<I, T> {
pub trait AnyStorableVec {
fn height(&self) -> color_eyre::Result<Height>;
fn flush(&mut self, height: Height) -> io::Result<()>;
fn reset_cache(&mut self);
}
impl<I, T> AnyStorableVec for StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug + Clone,
I: StorableVecIndex,
T: StorableVecType,
{
fn height(&self) -> color_eyre::Result<Height> {
self.height()
}
fn reset_cache(&mut self) {
self.vec.reset_cache()
}
fn flush(&mut self, height: Height) -> io::Result<()> {
self.flush(height)
}
indexer/src/storage/storable_vecs/mod.rs
+13 -8
View File
@@ -3,11 +3,12 @@ use std::{fs, io, path::Path};
use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::Version;
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes,
P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes, Timestamp, Txindex,
Txinindex, Txoutindex, Version,
Txinindex, Txoutindex,
};
mod base;
@@ -203,37 +204,37 @@ impl StorableVecs {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.get(addresstypeindex)?
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
_ => unreachable!(),
@@ -354,6 +355,10 @@ impl StorableVecs {
// Ok(())
}
pub fn reset_cache(&mut self) {
self.as_mut_slice().into_par_iter().for_each(|vec| vec.reset_cache())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_slice()
.into_par_iter()
indexer/src/structs/addressindex.rs
+16
View File
@@ -1,3 +1,5 @@
use std::ops::Add;
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
use unsafe_slice_serde::UnsafeSliceSerde;
@@ -60,3 +62,17 @@ impl From<Addressindex> for Slice {
Self::new(value.unsafe_as_slice())
}
}
impl Add<usize> for Addressindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl Add<Addressindex> for Addressindex {
type Output = Self;
fn add(self, rhs: Addressindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
indexer/src/structs/addresstypeindex.rs
+16
View File
@@ -1,3 +1,5 @@
use std::ops::Add;
use derive_deref::{Deref, DerefMut};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
@@ -50,3 +52,17 @@ impl From<Addresstypeindex> for usize {
value.0 as usize
}
}
impl Add<usize> for Addresstypeindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl Add<Addresstypeindex> for Addresstypeindex {
type Output = Self;
fn add(self, rhs: Addresstypeindex) -> Self::Output {
Self(self.0 + rhs.0)
}
}
indexer/src/structs/height.rs
+9 -2
View File
@@ -24,11 +24,19 @@ impl PartialEq<u64> for Height {
}
}
impl Add<Height> for Height {
type Output = Height;
fn add(self, rhs: Height) -> Self::Output {
Self::from(self.0 + rhs.0)
}
}
impl Add<u32> for Height {
type Output = Height;
fn add(self, rhs: u32) -> Self::Output {
Self::from(*self + rhs)
Self::from(self.0 + rhs)
}
}
@@ -42,7 +50,6 @@ impl Add<usize> for Height {
impl Sub<Height> for Height {
type Output = Height;
fn sub(self, rhs: Height) -> Self::Output {
Self::from(*self - *rhs)
}
indexer/src/structs/txindex.rs
+15 -1
View File
@@ -1,4 +1,4 @@
use std::ops::{Add, AddAssign};
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use fjall::Slice;
@@ -24,12 +24,26 @@ impl Add<Txindex> for Txindex {
}
}
impl Add<usize> for Txindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u32)
}
}
impl AddAssign<Txindex> for Txindex {
fn add_assign(&mut self, rhs: Txindex) {
self.0 += rhs.0
}
}
impl Sub<Txindex> for Txindex {
type Output = Txindex;
fn sub(self, rhs: Txindex) -> Self::Output {
Self::from(*self - *rhs)
}
}
impl From<u32> for Txindex {
fn from(value: u32) -> Self {
Self(value)
indexer/src/structs/txinindex.rs
+27 -4
View File
@@ -1,8 +1,8 @@
use std::ops::{Add, AddAssign};
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
use super::Vout;
use super::Vin;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default)]
pub struct Txinindex(u64);
@@ -24,19 +24,42 @@ impl Add<Txinindex> for Txinindex {
}
}
impl Add<Vout> for Txinindex {
impl Add<Vin> for Txinindex {
type Output = Self;
fn add(self, rhs: Vout) -> Self::Output {
fn add(self, rhs: Vin) -> Self::Output {
Self(self.0 + u64::from(rhs))
}
}
impl Add<usize> for Txinindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u64)
}
}
impl AddAssign<Txinindex> for Txinindex {
fn add_assign(&mut self, rhs: Txinindex) {
self.0 += rhs.0
}
}
impl Sub<Txinindex> for Txinindex {
type Output = Self;
fn sub(self, rhs: Txinindex) -> Self::Output {
Self(self.0 - rhs.0)
}
}
impl From<Txinindex> for u32 {
fn from(value: Txinindex) -> Self {
if value.0 > u32::MAX as u64 {
panic!()
}
value.0 as u32
}
}
impl From<u64> for Txinindex {
fn from(value: u64) -> Self {
Self(value)
indexer/src/structs/txoutindex.rs
+24 -1
View File
@@ -1,4 +1,4 @@
use std::ops::{Add, AddAssign};
use std::ops::{Add, AddAssign, Sub};
use derive_deref::{Deref, DerefMut};
@@ -37,12 +37,35 @@ impl Add<Vout> for Txoutindex {
}
}
impl Add<usize> for Txoutindex {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0 + rhs as u64)
}
}
impl AddAssign<Txoutindex> for Txoutindex {
fn add_assign(&mut self, rhs: Txoutindex) {
self.0 += rhs.0
}
}
impl Sub<Txoutindex> for Txoutindex {
type Output = Self;
fn sub(self, rhs: Txoutindex) -> Self::Output {
Self(self.0 - rhs.0)
}
}
impl From<Txoutindex> for u32 {
fn from(value: Txoutindex) -> Self {
if value.0 > u32::MAX as u64 {
panic!()
}
value.0 as u32
}
}
impl From<u64> for Txoutindex {
fn from(value: u64) -> Self {
Self(value)
indexer/src/structs/vin.rs
+2 -1
View File
@@ -4,7 +4,8 @@ use derive_deref::Deref;
pub struct Vin(u32);
impl Vin {
const ZERO: Self = Vin(0_u32);
pub const ZERO: Self = Vin(0);
pub const ONE: Self = Vin(1);
pub fn is_zero(&self) -> bool {
*self == Self::ZERO
storable_vec/src/any.rs
+27
View File
@@ -0,0 +1,27 @@
use std::io;
use crate::{StorableVec, StorableVecIndex, StorableVecType};
pub trait AnyStorableVec {
fn len(&self) -> usize;
fn is_empty(&self) -> bool;
fn unsafe_flush(&mut self) -> io::Result<()>;
}
impl<I, T> AnyStorableVec for StorableVec<I, T>
where
I: StorableVecIndex,
T: StorableVecType,
{
fn len(&self) -> usize {
self.len()
}
fn is_empty(&self) -> bool {
self.is_empty()
}
fn unsafe_flush(&mut self) -> io::Result<()> {
self.flush()
}
}
storable_vec/src/bytes.rs
+11
View File
@@ -0,0 +1,11 @@
use std::sync::Arc;
use crate::Result;
pub trait Bytes: Sized {
const LEN: usize = size_of::<Self>();
fn to_bytes(&self) -> Arc<[u8]>;
fn try_from_bytes(bytes: &[u8]) -> Result<Self>;
}
pub trait UnsafeBytes {}
storable_vec/src/error.rs
+34
View File
@@ -0,0 +1,34 @@
use std::{
fmt::{self, Debug},
io,
};
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
MmapsVecIsTooSmall,
IO(io::Error),
UnsafeSliceSerde(unsafe_slice_serde::Error),
IndexTooHigh,
ExpectFileToHaveIndex,
ExpectVecToHaveIndex,
FailedKeyTryIntoUsize,
}
impl fmt::Display for Error {
// This trait requires `fmt` with this exact signature.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"),
}
}
}
impl std::error::Error for Error {}
storable_vec/src/index.rs
+11
View File
@@ -0,0 +1,11 @@
use std::{fmt::Debug, ops::Add};
pub trait StorableVecIndex
where
Self: Debug + Default + Copy + Clone + TryInto<usize> + From<usize> + Add<usize, Output = Self>,
{
}
impl<I> StorableVecIndex for I where
I: Debug + Default + Copy + Clone + TryInto<usize> + From<usize> + Add<usize, Output = Self>
{
}
storable_vec/src/lib.rs
+136 -146
View File
@@ -1,21 +1,34 @@
use std::{
cmp::Ordering,
fmt::{self, Debug},
fs::{File, OpenOptions},
fmt::Debug,
fs::{self, File, OpenOptions},
io::{self, Read, Seek, SeekFrom, Write},
marker::PhantomData,
mem,
ops::{Deref, Range},
ops::Range,
path::{Path, PathBuf},
sync::{
// atomic::{AtomicUsize, Ordering as AtomicOrdering},
OnceLock,
},
sync::OnceLock,
};
use memmap2::{Mmap, MmapOptions};
use unsafe_slice_serde::UnsafeSliceSerde;
mod any;
// mod bytes;
mod error;
mod index;
mod type_;
mod value;
mod version;
pub use any::*;
// pub use bytes::*;
pub use error::*;
pub use index::*;
pub use type_::*;
pub use value::*;
pub use version::*;
///
/// A very small, fast, efficient and simple storable Vec
///
@@ -30,7 +43,7 @@ use unsafe_slice_serde::UnsafeSliceSerde;
#[derive(Debug)]
pub struct StorableVec<I, T> {
pathbuf: PathBuf,
file: File,
unsafe_file: File,
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed Mmap to reduce the size of the Lock (from 24 to 16)
disk_len: usize,
pushed: Vec<T>,
@@ -50,8 +63,8 @@ const MAX_CACHE_SIZE: usize = 100 * ONE_MB;
impl<I, T> StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
I: StorableVecIndex,
T: StorableVecType,
{
pub const SIZE_OF_T: usize = size_of::<T>();
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
@@ -59,13 +72,23 @@ where
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
pub fn import(path: &Path) -> Result<Self, io::Error> {
let file = Self::open_file_(path)?;
pub fn import(path: &Path, version: Version) -> Result<Self, io::Error> {
fs::create_dir_all(path)?;
let path_version = Self::path_version_(path);
let is_same_version =
Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
if !is_same_version {
fs::remove_dir_all(path)?;
}
version.write(&path_version)?;
let unsafe_file = Self::open_file_(&Self::path_vec_(path))?;
let mut this = Self {
pathbuf: path.to_owned(),
disk_len: Self::byte_index_to_index(Self::file_len(&file)?),
file,
disk_len: Self::disk_len(&unsafe_file)?,
unsafe_file,
cache: vec![],
pushed: vec![],
// updated: BTreeMap::new(),
@@ -76,20 +99,17 @@ where
// opened_mmaps: AtomicUsize::new(0),
};
// TODO: Only if write mode
this.reset_cache();
Ok(this)
}
fn file_len(file: &File) -> io::Result<usize> {
Ok(file.metadata()?.len() as usize)
pub fn disk_len(file: &File) -> io::Result<usize> {
Ok(Self::byte_index_to_index(file.metadata()?.len() as usize))
}
fn reset_cache(&mut self) {
// let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
// self.cache.clear();
// self.cache.resize_with(len, Default::default);
pub fn reset_cache(&mut self) {
// par_iter_mut ?
self.cache.iter_mut().for_each(|lock| {
lock.take();
@@ -105,7 +125,7 @@ where
}
fn open_file(&self) -> Result<File, Error> {
Self::open_file_(&self.pathbuf).map_err(Error::IO)
Self::open_file_(&self.path_vec()).map_err(Error::IO)
}
fn open_file_(path: &Path) -> Result<File, io::Error> {
OpenOptions::new()
@@ -146,10 +166,10 @@ where
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.get_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)
pub fn cached_get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.cached_get_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)
}
pub fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
fn cached_get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
@@ -166,7 +186,6 @@ where
// }
// }
let byte_index = Self::index_to_byte_index(index);
let page_index = index / Self::PER_PAGE;
let last_index = self.disk_len - 1;
let max_page_index = last_index / Self::PER_PAGE;
@@ -188,7 +207,7 @@ where
MmapOptions::new()
.len(Self::PAGE_SIZE)
.offset((page_index * Self::PAGE_SIZE) as u64)
.map(&self.file)
.map(&self.unsafe_file)
.unwrap()
})
});
@@ -201,49 +220,78 @@ where
T::unsafe_try_from_slice(slice).map_err(Error::UnsafeSliceSerde)?,
)))
} else {
let mut file = self.open_file()?;
let mut buf = vec![0; Self::SIZE_OF_T];
file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?;
file.read_exact(&mut buf).map_err(Error::IO)?;
let value = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?;
let (mut file, mut buf) = self.prepare_to_read()?;
Self::seek_(&mut file, index)?;
let value = self.read_exact(&mut file, &mut buf)?;
Ok(Some(Value::Owned(value.to_owned())))
}
}
pub fn get_or_default(&self, index: I) -> Result<T>
where
T: Default + Clone,
{
Ok(self.get(index)?.map(|v| (*v).clone()).unwrap_or(Default::default()))
Ok(self
.cached_get(index)?
.map(|v| (*v).clone())
.unwrap_or(Default::default()))
}
pub fn read_iter<F>(&self, f: F) -> Result<()>
where
F: FnMut((usize, &T)) -> Result<()>,
{
self.read_from_(0, f)
pub fn seek(file: &mut File, index: I) -> Result<()> {
Self::seek_(file, index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)
}
pub fn read_from_<F>(&self, index: usize, mut f: F) -> Result<()>
where
F: FnMut((usize, &T)) -> Result<()>,
{
let mut file = self.open_file()?;
let mut buf = vec![0; Self::SIZE_OF_T];
pub fn seek_(file: &mut File, index: usize) -> Result<()> {
let byte_index = Self::index_to_byte_index(index);
file.seek(SeekFrom::Start(byte_index as u64)).map_err(Error::IO)?;
Ok(())
}
let mut i = 0;
pub fn iter<F>(&self, f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
{
self.iter_from(I::from(0_usize), f)
}
loop {
if file.read_exact(&mut buf).map_err(Error::IO).is_err() {
break;
}
let v = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?;
f((i, v))?;
pub fn prepare_to_read(&self) -> Result<(File, Vec<u8>)> {
let file = self.open_file()?;
let buf = vec![0; Self::SIZE_OF_T];
Ok((file, buf))
}
pub fn prepare_to_read_at(&self, index: I) -> Result<(File, Vec<u8>)> {
self.prepare_to_read_at_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?)
}
pub fn prepare_to_read_at_(&self, index: usize) -> Result<(File, Vec<u8>)> {
let (mut file, buf) = self.prepare_to_read()?;
Self::seek_(&mut file, index)?;
Ok((file, buf))
}
pub fn read_exact<'a>(&self, file: &'a mut File, buf: &'a mut [u8]) -> Result<&'a T> {
file.read_exact(buf).map_err(Error::IO)?;
let v = T::unsafe_try_from_slice(&buf[..]).map_err(Error::UnsafeSliceSerde)?;
Ok(v)
}
pub fn iter_from<F>(&self, index: I, mut f: F) -> Result<()>
where
F: FnMut((I, &T)) -> Result<()>,
{
let (mut file, mut buf) = self.prepare_to_read()?;
let disk_len = Self::disk_len(&file).map_err(Error::IO)?;
Self::seek(&mut file, index)?;
let mut i: usize = index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?;
while i < disk_len {
let v = self.read_exact(&mut file, &mut buf)?;
f((I::from(i), v))?;
i += 1;
}
i = 0;
while i < self.pushed_len() {
f((I::from(i + disk_len), self.pushed.get(i).as_ref().unwrap()))?;
i += 1;
}
@@ -252,7 +300,7 @@ where
#[allow(unused)]
pub fn first(&self) -> Result<Option<Value<'_, T>>> {
self.get_(0)
self.cached_get_(0)
}
#[allow(unused)]
@@ -261,7 +309,7 @@ where
if len == 0 {
return Ok(None);
}
self.get_(len - 1)
self.cached_get_(len - 1)
}
pub fn push(&mut self, value: T) {
@@ -271,7 +319,7 @@ where
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
self.push_if_needed_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?, value)
}
pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> {
fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> {
let len = self.len();
match len.cmp(&index) {
Ordering::Greater => {
@@ -331,7 +379,11 @@ where
// }
pub fn len(&self) -> usize {
self.disk_len + self.pushed.len()
self.disk_len + self.pushed_len()
}
pub fn pushed_len(&self) -> usize {
self.pushed.len()
}
pub fn is_empty(&self) -> bool {
@@ -341,18 +393,30 @@ where
pub fn has(&self, index: I) -> Result<bool> {
Ok(self.has_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?))
}
pub fn has_(&self, index: usize) -> bool {
fn has_(&self, index: usize) -> bool {
index < self.len()
}
pub fn hasnt(&self, index: I) -> Result<bool> {
Ok(self.hasnt_(index.try_into().map_err(|_| Error::FailedKeyTryIntoUsize)?))
}
pub fn hasnt_(&self, index: usize) -> bool {
fn hasnt_(&self, index: usize) -> bool {
!self.has_(index)
}
// pub fn flush(&mut self) -> io::Result<()>
// where
// T: Bytes,
// {
// self.flush_(|bytes, v| bytes.extend_from_slice(&v.to_bytes()))
// }
pub fn flush(&mut self) -> io::Result<()> {
// self.flush_(|bytes, v| bytes.extend_from_slice(v.unsafe_as_slice()))
// }
// fn flush_<F>(&mut self, mut extend: F) -> io::Result<()>
// where
// F: FnMut(&mut Vec<u8>, T),
// {
self.reset_cache();
if self.pushed.is_empty() {
@@ -366,99 +430,25 @@ where
mem::take(&mut self.pushed)
.into_iter()
.for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
// .for_each(|v| extend(&mut bytes, v));
self.file.write_all(&bytes)?;
self.unsafe_file.write_all(&bytes)?;
Ok(())
}
}
pub trait AnyStorableVec {
fn len(&self) -> usize;
fn is_empty(&self) -> bool;
fn flush(&mut self) -> io::Result<()>;
}
impl<I, T> AnyStorableVec for StorableVec<I, T>
where
I: Into<usize>,
T: Sized + Debug + Clone,
{
fn len(&self) -> usize {
self.len()
pub fn path(&self) -> &Path {
&self.pathbuf
}
fn is_empty(&self) -> bool {
self.is_empty()
fn path_vec(&self) -> PathBuf {
Self::path_vec_(&self.pathbuf)
}
fn path_vec_(path: &Path) -> PathBuf {
path.join("vec")
}
fn flush(&mut self) -> io::Result<()> {
self.flush()
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
}
#[derive(Debug, Clone)]
pub enum Value<'a, T> {
Ref(&'a T),
Owned(T),
}
impl<T> Value<'_, T>
where
T: Sized + Debug + Clone,
{
pub fn into_inner(self) -> T {
match self {
Self::Ref(t) => t.to_owned(),
Self::Owned(t) => t,
}
}
}
impl<T> Deref for Value<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Ref(t) => t,
Self::Owned(t) => t,
}
}
}
impl<T> AsRef<T> for Value<'_, T>
where
T: Sized + Debug + Clone,
{
fn as_ref(&self) -> &T {
match self {
Self::Ref(t) => t,
Self::Owned(t) => t,
}
}
}
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
MmapsVecIsTooSmall,
IO(io::Error),
UnsafeSliceSerde(unsafe_slice_serde::Error),
IndexTooHigh,
ExpectFileToHaveIndex,
ExpectVecToHaveIndex,
FailedKeyTryIntoUsize,
}
impl fmt::Display for Error {
// This trait requires `fmt` with this exact signature.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::UnsafeSliceSerde(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
Error::FailedKeyTryIntoUsize => write!(f, "Failed to convert key to usize"),
}
}
}
impl std::error::Error for Error {}
storable_vec/src/main.rs
+6 -6
View File
@@ -1,24 +1,24 @@
use std::path::Path;
use storable_vec::StorableVec;
use storable_vec::{StorableVec, Version};
fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"), Version::from(1))?;
vec.push(0);
vec.push(1);
vec.push(2);
dbg!(vec.get(0)?); // Some(0)
dbg!(vec.get(21)?); // None
dbg!(vec.cached_get(0)?); // Some(0)
dbg!(vec.cached_get(21)?); // None
vec.flush()?;
}
{
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"), Version::from(1))?;
dbg!(vec.get(0)?); // 0
dbg!(vec.cached_get(0)?); // 0
}
Ok(())
storable_vec/src/type_.rs
+8
View File
@@ -0,0 +1,8 @@
use std::fmt::Debug;
pub trait StorableVecType
where
Self: Sized + Debug + Clone,
{
}
impl<T> StorableVecType for T where T: Sized + Debug + Clone {}
storable_vec/src/value.rs
+39
View File
@@ -0,0 +1,39 @@
use std::{fmt::Debug, ops::Deref};
#[derive(Debug, Clone)]
pub enum Value<'a, T> {
Ref(&'a T),
Owned(T),
}
impl<T> Value<'_, T>
where
T: Sized + Debug + Clone,
{
pub fn into_inner(self) -> T {
match self {
Self::Ref(t) => t.to_owned(),
Self::Owned(t) => t,
}
}
}
impl<T> Deref for Value<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Ref(t) => t,
Self::Owned(t) => t,
}
}
}
impl<T> AsRef<T> for Value<'_, T>
where
T: Sized + Debug + Clone,
{
fn as_ref(&self) -> &T {
match self {
Self::Ref(t) => t,
Self::Owned(t) => t,
}
}
}
storable_vec/src/version.rs
+29
View File
@@ -0,0 +1,29 @@
use std::{
fs,
io::{self, Read},
path::Path,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Version(u32);
impl Version {
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
fs::write(path, self.0.to_le_bytes())
}
}
impl From<u32> for Version {
fn from(value: u32) -> Self {
Self(value)
}
}
impl TryFrom<&Path> for Version {
type Error = io::Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 4];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(Self(u32::from_le_bytes(buf)))
}
}
struct_iterable/struct_iterable_derive/Cargo.toml
-2
View File
@@ -13,5 +13,3 @@ proc-macro = true
[dependencies]
syn = "2.0.96"
quote = "1.0.38"
proc-macro2 = "1.0.93"
struct_iterable_internal = { path = "../struct_iterable_internal" }