Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-14 08:43:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: caching only in iter

Browse Source
This commit is contained in:
nym21
2025-04-30 18:29:18 +02:00
parent 664b125ce2
commit 700352ec45
22 changed files with 411 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_cli/Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ color-eyre = { workspace = true }
log = { workspace = true }
serde = { workspace = true }
tabled = { workspace = true }
toml = "0.8.20"
toml = "0.8.22"
[[bin]]
name = "brk"
crates/brk_computer/src/storage/vecs/blocks.rs
+2 -2
View File
@@ -209,14 +209,14 @@ impl Vecs {
self.difficultyepoch_to_timestamp.compute_transform(
starting_indexes.difficultyepoch,
indexes.difficultyepoch_to_first_height.vec(),
|(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()),
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
exit,
)?;
self.halvingepoch_to_timestamp.compute_transform(
starting_indexes.halvingepoch,
indexes.halvingepoch_to_first_height.vec(),
|(i, h, ..)| (i, height_to_timestamp_iter.get(h).unwrap().1.into_inner()),
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
exit,
)?;
crates/brk_computer/src/storage/vecs/grouped/builder.rs
+11 -13
View File
@@ -186,12 +186,12 @@ where
.try_for_each(|(i, first_index)| -> Result<()> {
let first_index = first_index.into_inner();
let count_index = count_indexes_iter.get(i).unwrap().1.into_inner();
let count_index = count_indexes_iter.unwrap_get_inner(i);
if let Some(first) = self.first.as_mut() {
let f = source_iter
.get(first_index)
.map_or(T::from(0_usize), |f| f.1.into_inner());
.get_inner(first_index)
.unwrap_or_else(|| T::from(0_usize));
first.forced_push_at(index, f, exit)?;
}
@@ -201,15 +201,13 @@ where
panic!("should compute last if count can be 0")
}
let last_index = first_index + (count_index - 1);
let v = source_iter
.get(last_index)
.context("to work")
.inspect_err(|_| {
dbg!(first_index, count_index, last_index);
})
.unwrap()
.1
.into_inner();
let v = source_iter.unwrap_get_inner(last_index);
// .context("to work")
// .inspect_err(|_| {
// dbg!(first_index, count_index, last_index);
// })
// .unwrap()
// .into_inner();
last.forced_push_at(index, v, exit)?;
}
@@ -357,7 +355,7 @@ where
.try_for_each(|(i, first_index, ..)| -> Result<()> {
let first_index = first_index.into_inner();
let count_index = count_indexes_iter.get(i).unwrap().1.into_inner();
let count_index = count_indexes_iter.unwrap_get_inner(i);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
crates/brk_computer/src/storage/vecs/marketprice.rs
+2 -3
View File
@@ -342,9 +342,8 @@ impl Vecs {
.get_height(
h,
t,
h.decremented().map(|prev_h| {
height_to_timestamp_iter.get(prev_h).unwrap().1.into_inner()
}),
h.decremented()
.map(|prev_h| height_to_timestamp_iter.unwrap_get_inner(prev_h)),
)
.unwrap();
(h, ohlc)
crates/brk_computer/src/storage/vecs/transactions.rs
+3 -11
View File
@@ -471,11 +471,7 @@ impl Vecs {
indexer.vecs().height_to_first_txindex.vec(),
indexer.vecs().txindex_to_txid.vec(),
|txindex| {
let v = txindex_to_txversion_iter
.get(txindex)
.unwrap()
.1
.into_inner();
let v = txindex_to_txversion_iter.unwrap_get_inner(txindex);
v == txversion
},
exit,
@@ -499,11 +495,7 @@ impl Vecs {
starting_indexes.txindex,
indexer.vecs().txindex_to_base_size.vec(),
|(txindex, base_size, ..)| {
let total_size = txindex_to_total_size_iter
.get(txindex)
.unwrap()
.1
.into_inner();
let total_size = txindex_to_total_size_iter.unwrap_get_inner(txindex);
// This is the exact definition of a weight unit, as defined by BIP-141 (quote above).
let wu = usize::from(base_size) * 3 + usize::from(total_size);
@@ -533,7 +525,7 @@ impl Vecs {
|(inputindex, outputindex, ..)| {
let value = if outputindex == OutputIndex::COINBASE {
Sats::ZERO
} else if let Some((_, value)) = outputindex_to_value_iter.get(outputindex) {
} else if let Some(value) = outputindex_to_value_iter.get(outputindex) {
value.into_inner()
} else {
dbg!(inputindex, outputindex, inputs_len);
crates/brk_computer/src/storage/vecs/vec/eager.rs
+9 -21
View File
@@ -12,7 +12,6 @@ use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
};
use color_eyre::eyre::ContextCompat;
use log::info;
const ONE_KIB: usize = 1024;
@@ -223,11 +222,7 @@ where
if prev_i.is_some_and(|prev_i| prev_i == i) {
return Ok(());
}
if self
.inner
.get(i)?
.is_none_or(|old_v| old_v.into_inner() > v)
{
if self.iter().get_inner(i).is_none_or(|old_v| old_v > v) {
self.forced_push_at(i, v, exit)?;
}
prev_i.replace(i);
@@ -350,8 +345,8 @@ where
.iter_at(index)
.try_for_each(|(i, first_index)| {
let end = other_iter
.get(i + 1)
.map(|(_, v)| v.into_inner().unwrap_to_usize())
.get_inner(i + 1)
.map(|v| v.unwrap_to_usize())
.unwrap_or_else(|| other_to_else.len());
let range = first_index.unwrap_to_usize()..end;
@@ -387,14 +382,7 @@ where
self_to_other.iter_at(index).try_for_each(|(i, other)| {
self.forced_push_at(
i,
T::from(
other_to_self_iter
.get(other.into_inner())
.unwrap()
.1
.into_inner()
== i,
),
T::from(other_to_self_iter.unwrap_get_inner(other.into_inner()) == i),
exit,
)
})?;
@@ -424,12 +412,12 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| {
let count = *indexes_count_iter.get(i).unwrap().1.into_inner();
let count = *indexes_count_iter.unwrap_get_inner(i);
let first_index = first_index.unwrap_to_usize();
let range = first_index..first_index + count;
let mut sum = T::from(0_usize);
range.into_iter().for_each(|i| {
sum = sum.clone() + source_iter.get(T2::from(i)).unwrap().1.into_inner();
sum = sum.clone() + source_iter.unwrap_get_inner(T2::from(i));
});
self.forced_push_at(i, sum, exit)
})?;
@@ -477,7 +465,7 @@ impl EagerVec<Height, Dollars> {
let mut price_iter = price.iter();
let index = max_from.min(Height::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.get(i).unwrap().1.into_inner();
let dollars = price_iter.unwrap_get_inner(i);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
@@ -503,8 +491,8 @@ impl EagerVec<TxIndex, Dollars> {
let mut price_iter = price.iter();
let index = max_from.min(TxIndex::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin, ..)| {
let height = i_to_height_iter.get(i).unwrap().1.into_inner();
let dollars = price_iter.get(height).unwrap().1.into_inner();
let height = i_to_height_iter.unwrap_get_inner(i);
let dollars = price_iter.unwrap_get_inner(height);
let (i, v) = (i, *dollars * bitcoin.into_inner());
self.forced_push_at(i, v, exit)
})?;
crates/brk_indexer/examples/main.rs
+1 -1
View File
@@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, false, true)?;
let mut indexer = Indexer::new(outputs, false, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;
crates/brk_indexer/src/indexes.rs
+40 -39
View File
@@ -5,7 +5,7 @@ use brk_core::{
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{Result, StoredIndex, StoredType, Value};
use brk_vec::{StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{IndexedVec, Stores, Vecs};
@@ -107,115 +107,116 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
})
.unwrap();
vecs.height_to_blockhash.get(*height).map_or(true, |opt| {
opt.is_none_or(|saved_blockhash| {
vecs.height_to_blockhash
.iter()
.get(*height)
.is_none_or(|saved_blockhash| {
let b = &rpc_blockhash != saved_blockhash.as_ref();
if b {
dbg!(rpc_blockhash, saved_blockhash.as_ref());
}
b
})
})
})
.unwrap_or(starting_height);
Ok(Self {
emptyoutputindex: *starting_index(
emptyoutputindex: starting_index(
&vecs.height_to_first_emptyoutputindex,
&vecs.emptyoutputindex_to_txindex,
height,
)?
)
.context("")?,
height,
p2msindex: *starting_index(
p2msindex: starting_index(
&vecs.height_to_first_p2msindex,
&vecs.p2msindex_to_txindex,
height,
)?
)
.context("")?,
opreturnindex: *starting_index(
opreturnindex: starting_index(
&vecs.height_to_first_opreturnindex,
&vecs.opreturnindex_to_txindex,
height,
)?
)
.context("")?,
p2pk33index: *starting_index(
p2pk33index: starting_index(
&vecs.height_to_first_p2pk33index,
&vecs.p2pk33index_to_p2pk33bytes,
height,
)?
)
.context("")?,
p2pk65index: *starting_index(
p2pk65index: starting_index(
&vecs.height_to_first_p2pk65index,
&vecs.p2pk65index_to_p2pk65bytes,
height,
)?
)
.context("")?,
p2pkhindex: *starting_index(
p2pkhindex: starting_index(
&vecs.height_to_first_p2pkhindex,
&vecs.p2pkhindex_to_p2pkhbytes,
height,
)?
)
.context("")?,
p2shindex: *starting_index(
p2shindex: starting_index(
&vecs.height_to_first_p2shindex,
&vecs.p2shindex_to_p2shbytes,
height,
)?
)
.context("")?,
p2trindex: *starting_index(
p2trindex: starting_index(
&vecs.height_to_first_p2trindex,
&vecs.p2trindex_to_p2trbytes,
height,
)?
)
.context("")?,
p2wpkhindex: *starting_index(
p2wpkhindex: starting_index(
&vecs.height_to_first_p2wpkhindex,
&vecs.p2wpkhindex_to_p2wpkhbytes,
height,
)?
)
.context("")?,
p2wshindex: *starting_index(
p2wshindex: starting_index(
&vecs.height_to_first_p2wshindex,
&vecs.p2wshindex_to_p2wshbytes,
height,
)?
)
.context("")?,
p2aindex: *starting_index(
p2aindex: starting_index(
&vecs.height_to_first_p2aindex,
&vecs.p2aindex_to_p2abytes,
height,
)?
)
.context("")?,
txindex: *starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)?
txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)
.context("")?,
inputindex: *starting_index(
inputindex: starting_index(
&vecs.height_to_first_inputindex,
&vecs.inputindex_to_outputindex,
height,
)?
)
.context("")?,
outputindex: *starting_index(
outputindex: starting_index(
&vecs.height_to_first_outputindex,
&vecs.outputindex_to_value,
height,
)?
)
.context("")?,
unknownoutputindex: *starting_index(
unknownoutputindex: starting_index(
&vecs.height_to_first_unknownoutputindex,
&vecs.unknownoutputindex_to_txindex,
height,
)?
)
.context("")?,
})
}
}
pub fn starting_index<'a, I, T>(
height_to_index: &'a IndexedVec<Height, I>,
index_to_else: &'a IndexedVec<I, T>,
pub fn starting_index<I, T>(
height_to_index: &IndexedVec<Height, I>,
index_to_else: &IndexedVec<I, T>,
starting_height: Height,
) -> Result<Option<Value<'a, I>>>
) -> Option<I>
where
I: StoredType + StoredIndex + From<usize>,
T: StoredType,
@@ -224,8 +225,8 @@ where
.height()
.is_ok_and(|h| h + 1_u32 == starting_height)
{
Ok(Some(Value::Owned(I::from(index_to_else.len()))))
Some(I::from(index_to_else.len()))
} else {
height_to_index.get(starting_height)
height_to_index.iter().get_inner(starting_height)
}
}
crates/brk_indexer/src/lib.rs
+59 -10
View File
@@ -18,7 +18,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::Compressed;
use brk_vec::{Compressed, DynamicVec};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -242,7 +242,11 @@ impl Indexer {
})
});
let input_source_vec_handle = scope.spawn(|| {
let txindex_to_first_outputindex_mmap = vecs
.txindex_to_first_outputindex.vec().mmap().load();
inputs
.into_par_iter()
.enumerate()
@@ -273,13 +277,11 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let outputindex = *vecs
.txindex_to_first_outputindex
.get(prev_txindex)?
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?
})?.into_inner()
+ vout;
Ok((inputindex, InputSource::PreviousBlock((
@@ -305,6 +307,16 @@ impl Indexer {
});
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
let p2pk65index_to_p2pk65bytes_mmap = vecs
.p2pk65index_to_p2pk65bytes.vec().mmap().load();
let p2pk33index_to_p2pk33bytes_mmap = vecs.p2pk33index_to_p2pk33bytes.vec().mmap().load();
let p2pkhindex_to_p2pkhbytes_mmap = vecs.p2pkhindex_to_p2pkhbytes.vec().mmap().load();
let p2shindex_to_p2shbytes_mmap = vecs.p2shindex_to_p2shbytes.vec().mmap().load();
let p2wpkhindex_to_p2wpkhbytes_mmap = vecs.p2wpkhindex_to_p2wpkhbytes.vec().mmap().load();
let p2wshindex_to_p2wshbytes_mmap = vecs.p2wshindex_to_p2wshbytes.vec().mmap().load();
let p2trindex_to_p2trbytes_mmap = vecs.p2trindex_to_p2trbytes.vec().mmap().load();
let p2aindex_to_p2abytes_mmap = vecs.p2aindex_to_p2abytes.vec().mmap().load();
outputs
.into_par_iter()
.enumerate()
@@ -349,8 +361,43 @@ impl Indexer {
if let Some(Some(outputtypeindex)) = check_collisions.then_some(outputtypeindex_opt) {
let addressbytes = address_bytes_res.as_ref().unwrap();
let prev_addressbytes_opt =
vecs.get_addressbytes(outputtype, outputtypeindex)?;
let prev_addressbytes_opt = match outputtype {
OutputType::P2PK65 => vecs
.p2pk65index_to_p2pk65bytes
.get_or_read(outputtypeindex.into(), &p2pk65index_to_p2pk65bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => vecs
.p2pk33index_to_p2pk33bytes
.get_or_read(outputtypeindex.into(), &p2pk33index_to_p2pk33bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => vecs
.p2pkhindex_to_p2pkhbytes
.get_or_read(outputtypeindex.into(), &p2pkhindex_to_p2pkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => vecs
.p2shindex_to_p2shbytes
.get_or_read(outputtypeindex.into(), &p2shindex_to_p2shbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => vecs
.p2wpkhindex_to_p2wpkhbytes
.get_or_read(outputtypeindex.into(), &p2wpkhindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => vecs
.p2wshindex_to_p2wshbytes
.get_or_read(outputtypeindex.into(), &p2wshindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => vecs
.p2trindex_to_p2trbytes
.get_or_read(outputtypeindex.into(), &p2trindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => vecs
.p2aindex_to_p2abytes
.get_or_read(outputtypeindex.into(), &p2aindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
@@ -600,6 +647,9 @@ impl Indexer {
let mut txindex_to_tx_and_txid: BTreeMap<TxIndex, (&Transaction, Txid)> = BTreeMap::default();
let mut txindex_to_txid_iter = vecs
.txindex_to_txid.iter();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
@@ -626,9 +676,8 @@ impl Indexer {
let len = vecs.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.get(prev_txindex)?
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, len);
crates/brk_indexer/src/stores/mod.rs
+47 -30
View File
@@ -96,12 +96,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk65index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65bytes
.get(index)?
let mut p2pk65index_to_p2pk65bytes_iter = vecs.p2pk65index_to_p2pk65bytes.iter();
while let Some(typedbytes) = p2pk65index_to_p2pk65bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -113,12 +115,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk33index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33bytes
.get(index)?
let mut p2pk33index_to_p2pk33bytes_iter = vecs.p2pk33index_to_p2pk33bytes.iter();
while let Some(typedbytes) = p2pk33index_to_p2pk33bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -130,12 +134,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhbytes
.get(index)?
let mut p2pkhindex_to_p2pkhbytes_iter = vecs.p2pkhindex_to_p2pkhbytes.iter();
while let Some(typedbytes) = p2pkhindex_to_p2pkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -147,12 +153,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2shindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2shindex_to_p2shbytes
.get(index)?
let mut p2shindex_to_p2shbytes_iter = vecs.p2shindex_to_p2shbytes.iter();
while let Some(typedbytes) = p2shindex_to_p2shbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -164,12 +172,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2trindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2trindex_to_p2trbytes
.get(index)?
let mut p2trindex_to_p2trbytes_iter = vecs.p2trindex_to_p2trbytes.iter();
while let Some(typedbytes) = p2trindex_to_p2trbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -181,12 +191,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wpkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhbytes
.get(index)?
let mut p2wpkhindex_to_p2wpkhbytes_iter = vecs.p2wpkhindex_to_p2wpkhbytes.iter();
while let Some(typedbytes) = p2wpkhindex_to_p2wpkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -198,12 +210,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wshindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wshindex_to_p2wshbytes
.get(index)?
let mut p2wshindex_to_p2wshbytes_iter = vecs.p2wshindex_to_p2wshbytes.iter();
while let Some(typedbytes) = p2wshindex_to_p2wshbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -215,11 +229,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2aindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
let mut p2aindex_to_p2abytes_iter = vecs.p2aindex_to_p2abytes.iter();
while let Some(typedbytes) =
vecs.p2aindex_to_p2abytes.get(index)?.map(Value::into_inner)
p2aindex_to_p2abytes_iter.get(index).map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from((&bytes, OutputType::P2A));
crates/brk_indexer/src/vecs/base.rs
+4 -8
View File
@@ -5,7 +5,7 @@ use std::{
};
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
};
@@ -31,19 +31,15 @@ where
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let mut inner = StoredVec::forced_import(path, version, compressed)?;
inner.enable_large_cache_if_needed();
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
inner,
inner: StoredVec::forced_import(path, version, compressed)?,
})
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.inner.get(index)
pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
self.inner.get_or_read(index, mmap)
}
#[inline]
crates/brk_indexer/src/vecs/mod.rs
+11 -55
View File
@@ -79,7 +79,7 @@ impl Vecs {
height_to_blockhash: IndexedVec::forced_import(
&path.join("height_to_blockhash"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
height_to_difficulty: IndexedVec::forced_import(
&path.join("height_to_difficulty"),
@@ -204,7 +204,7 @@ impl Vecs {
p2aindex_to_p2abytes: IndexedVec::forced_import(
&path.join("p2aindex_to_p2abytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2msindex_to_txindex: IndexedVec::forced_import(
&path.join("p2msindex_to_txindex"),
@@ -214,37 +214,37 @@ impl Vecs {
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
&path.join("p2pk33index_to_p2pk33bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pk65index_to_p2pk65bytes: IndexedVec::forced_import(
&path.join("p2pk65index_to_p2pk65bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import(
&path.join("p2pkhindex_to_p2pkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2shindex_to_p2shbytes: IndexedVec::forced_import(
&path.join("p2shindex_to_p2shbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2trindex_to_p2trbytes: IndexedVec::forced_import(
&path.join("p2trindex_to_p2trbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import(
&path.join("p2wpkhindex_to_p2wpkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wshindex_to_p2wshbytes: IndexedVec::forced_import(
&path.join("p2wshindex_to_p2wshbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_base_size: IndexedVec::forced_import(
&path.join("txindex_to_base_size"),
@@ -259,7 +259,7 @@ impl Vecs {
txindex_to_first_outputindex: IndexedVec::forced_import(
&path.join("txindex_to_first_outputindex"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
&path.join("txindex_to_is_explicitly_rbf"),
@@ -279,7 +279,7 @@ impl Vecs {
txindex_to_txid: IndexedVec::forced_import(
&path.join("txindex_to_txid"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_txversion: IndexedVec::forced_import(
&path.join("txindex_to_txversion"),
@@ -408,50 +408,6 @@ impl Vecs {
Ok(())
}
pub fn get_addressbytes(
&self,
outputtype: OutputType,
outputtypeindex: OutputTypeIndex,
) -> brk_vec::Result<Option<AddressBytes>> {
Ok(match outputtype {
OutputType::P2PK65 => self
.p2pk65index_to_p2pk65bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => self
.p2pk33index_to_p2pk33bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => self
.p2pkhindex_to_p2pkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => self
.p2shindex_to_p2shbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => self
.p2wpkhindex_to_p2wpkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => self
.p2wshindex_to_p2wshbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => self
.p2trindex_to_p2trbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => self
.p2aindex_to_p2abytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
})
}
pub fn push_bytes_if_needed(
&mut self,
index: OutputTypeIndex,
crates/brk_server/Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.66.0", features = ["codegen", "minifier"] }
oxc = { version = "0.67.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.44.2", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full", "trace"] }
crates/brk_vec/examples/main.rs
+30 -23
View File
@@ -15,9 +15,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
(0..21_u32).for_each(|v| {
vec.push(v);
});
dbg!(vec.get(0)?);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
let mut iter = vec.iter();
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
vec.flush()?;
}
@@ -25,21 +27,25 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
dbg!(vec.get(0)?);
dbg!(vec.get(0)?);
dbg!(vec.get(1)?);
dbg!(vec.get(2)?);
dbg!(vec.get(20)?);
dbg!(vec.get(20)?);
dbg!(vec.get(0)?);
dbg!(iter.get(0));
dbg!(iter.get(0));
dbg!(iter.get(1));
dbg!(iter.get(2));
dbg!(iter.get(20));
dbg!(iter.get(20));
dbg!(iter.get(0));
vec.push(21);
vec.push(22);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
dbg!(vec.get(22)?);
dbg!(vec.get(23)?);
let mut iter = vec.iter();
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
dbg!(iter.get(23));
vec.flush()?;
}
@@ -47,19 +53,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
{
let mut vec: StoredVec<usize, u32> =
StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
let mut iter = vec.iter();
vec.enable_large_cache_if_needed();
dbg!(vec.get(0)?);
dbg!(vec.get(20)?);
dbg!(vec.get(21)?);
dbg!(vec.get(22)?);
dbg!(iter.get(0));
dbg!(iter.get(20));
dbg!(iter.get(21));
dbg!(iter.get(22));
vec.truncate_if_needed(14)?;
dbg!(vec.get(0)?);
dbg!(vec.get(5)?);
dbg!(vec.get(20)?);
let mut iter = vec.iter();
dbg!(iter.get(0));
dbg!(iter.get(5));
dbg!(iter.get(20));
dbg!(vec.collect_signed_range(Some(-5), None)?);
crates/brk_vec/src/lib.rs
+10 -35
View File
@@ -8,15 +8,12 @@ mod structs;
mod traits;
mod variants;
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use std::path::{Path, PathBuf};
use arc_swap::{ArcSwap, Guard};
use arc_swap::ArcSwap;
use axum::response::Response;
pub use enums::*;
use memmap2::Mmap;
pub use memmap2::Mmap;
pub use structs::*;
pub use traits::*;
use variants::*;
@@ -46,13 +43,6 @@ where
}
}
pub fn enable_large_cache_if_needed(&mut self) {
match self {
StoredVec::Compressed(v) => v.enable_large_cache(),
Self::Raw(_) => {}
}
}
pub fn iter(&self) -> StoredVecIterator<'_, I, T> {
self.into_iter()
}
@@ -73,10 +63,10 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
fn read_(&self, index: usize, guard: &Mmap) -> Result<Option<T>> {
match self {
StoredVec::Raw(v) => v.get_stored_(index, guard),
StoredVec::Compressed(v) => v.get_stored_(index, guard),
StoredVec::Raw(v) => v.read_(index, guard),
StoredVec::Compressed(v) => v.read_(index, guard),
}
}
@@ -88,21 +78,6 @@ where
}
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
match self {
StoredVec::Raw(v) => v.guard(),
StoredVec::Compressed(v) => v.guard(),
}
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
match self {
StoredVec::Raw(v) => v.mut_guard(),
StoredVec::Compressed(v) => v.mut_guard(),
}
}
#[inline]
fn stored_len(&self) -> usize {
match self {
@@ -250,21 +225,21 @@ where
{
#[inline]
pub fn unwrap_get_inner(&mut self, i: I) -> T {
self.get_(i.unwrap_to_usize()).unwrap().1.into_inner()
self.get_(i.unwrap_to_usize()).unwrap().into_inner()
}
#[inline]
pub fn get_inner(&mut self, i: I) -> Option<T> {
self.get_(i.unwrap_to_usize()).map(|(_, v)| v.into_inner())
self.get_(i.unwrap_to_usize()).map(|v| v.into_inner())
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.get_(i.unwrap_to_usize())
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
match self {
Self::Compressed(iter) => iter.get_(i),
Self::Raw(iter) => iter.get_(i),
crates/brk_vec/src/traits/dynamic.rs
+23 -44
View File
@@ -1,45 +1,42 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use arc_swap::{ArcSwap, Guard};
use arc_swap::ArcSwap;
use memmap2::Mmap;
use crate::{Error, Result, Value};
use crate::{Result, Value};
use super::{StoredIndex, StoredType};
pub trait DynamicVec: Send + Sync {
type I: StoredIndex;
type T: StoredType;
const SIZE_OF_T: usize = size_of::<Self::T>();
#[inline]
fn get(&self, index: Self::I) -> Result<Option<Value<Self::T>>> {
self.get_(index.to_usize()?)
fn read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Self::T>> {
self.read_(index.to_usize()?, mmap)
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
#[inline]
fn get_or_read(&self, index: Self::I, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
self.get_or_read_(index.to_usize()?, mmap)
}
#[inline]
fn get_(&self, index: usize) -> Result<Option<Value<Self::T>>> {
match self.index_to_pushed_index(index) {
Ok(index) => {
if let Some(index) = index {
return Ok(self.pushed().get(index).map(Value::Ref));
}
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Value<Self::T>>> {
let stored_len = mmap.len() / Self::SIZE_OF_T;
if index >= stored_len {
let pushed = self.pushed();
let j = index - stored_len;
if j >= pushed.len() {
return Ok(None);
}
Err(Error::IndexTooHigh) => return Ok(None),
Err(Error::IndexTooLow) => {}
Err(error) => return Err(error),
Ok(pushed.get(j).map(Value::Ref))
} else {
Ok(self.read_(index, mmap)?.map(Value::Owned))
}
Ok(self
.get_stored_(index.to_usize()?, self.guard().as_ref().unwrap())?
.map(Value::Owned))
}
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
// fn last(&self) -> Result<Option<Value<Self::T>>> {
// let len = self.len();
// if len == 0 {
// return Ok(None);
// }
// self.get_(len - 1)
// }
#[inline]
fn len(&self) -> usize {
@@ -52,9 +49,6 @@ pub trait DynamicVec: Send + Sync {
fn mmap(&self) -> &ArcSwap<Mmap>;
fn guard(&self) -> &Option<Guard<Arc<Mmap>>>;
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>>;
fn stored_len(&self) -> usize;
fn pushed(&self) -> &[Self::T];
@@ -67,21 +61,6 @@ pub trait DynamicVec: Send + Sync {
fn push(&mut self, value: Self::T) {
self.mut_pushed().push(value)
}
#[inline]
fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
let stored_len = self.stored_len();
if index >= stored_len {
let index = index - stored_len;
if index >= self.pushed_len() {
Err(Error::IndexTooHigh)
} else {
Ok(Some(index))
}
} else {
Err(Error::IndexTooLow)
}
}
fn path(&self) -> &Path;
}
crates/brk_vec/src/traits/generic.rs
-8
View File
@@ -20,8 +20,6 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<Self::T>();
fn open_file(&self) -> io::Result<File> {
Self::open_file_(&self.path_vec())
}
@@ -70,12 +68,6 @@ where
fn update_mmap(&mut self, file: File) -> Result<()> {
let mmap = Self::new_mmap(file)?;
self.mmap().store(mmap);
if self.guard().is_some() {
let guard = self.mmap().load();
self.mut_guard().replace(guard);
} else {
unreachable!("This function shouldn't be called in a cloned instance")
}
Ok(())
}
crates/brk_vec/src/variants/compressed.rs
+43 -134
View File
@@ -2,7 +2,7 @@ use std::{
fs::{self, File},
mem,
path::Path,
sync::{Arc, OnceLock},
sync::Arc,
};
use arc_swap::{ArcSwap, Guard};
@@ -23,7 +23,6 @@ pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
#[derive(Debug)]
pub struct CompressedVec<I, T> {
inner: RawVec<I, T>,
decoded_pages: Option<Vec<OnceLock<Vec<T>>>>,
pages_meta: Arc<ArcSwap<CompressedPagesMetadata>>,
}
@@ -67,33 +66,10 @@ where
Ok(Self {
inner: RawVec::import(path, version)?,
decoded_pages: None,
pages_meta: Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(path)?))),
})
}
fn cached_get_stored__(
index: usize,
mmap: &Mmap,
stored_len: usize,
decoded_page: &mut Option<(usize, Vec<T>)>,
compressed_pages_meta: &CompressedPagesMetadata,
) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
if decoded_page.as_ref().is_none_or(|b| b.0 != page_index) {
let values = Self::decode_page_(stored_len, page_index, mmap, compressed_pages_meta)?;
decoded_page.replace((page_index, values));
}
Ok(decoded_page
.as_ref()
.unwrap()
.1
.get(index % Self::PER_PAGE)
.cloned())
}
fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result<Vec<T>> {
Self::decode_page_(self.stored_len(), page_index, mmap, &self.pages_meta.load())
}
@@ -140,40 +116,6 @@ where
zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap()
}
pub fn enable_large_cache(&mut self) {
self.decoded_pages.replace(vec![]);
self.reset_large_cache();
}
pub fn disable_large_cache(&mut self) {
self.decoded_pages.take();
}
fn reset_large_cache(&mut self) {
let stored_len = self.stored_len();
if let Some(pages) = self.decoded_pages.as_mut() {
pages.par_iter_mut().for_each(|lock| {
lock.take();
});
let len = (stored_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
let len = Self::CACHE_LENGTH.min(len);
if pages.len() != len {
pages.resize_with(len, Default::default);
}
}
}
pub fn large_cache_len(&self) -> usize {
self.decoded_pages.as_ref().map_or(0, |v| v.len())
}
fn reset_caches(&mut self) {
self.reset_large_cache();
}
#[inline(always)]
fn index_to_page_index(index: usize) -> usize {
index / Self::PER_PAGE
@@ -219,32 +161,9 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let cached_start = self
.stored_len()
.checked_sub(Self::CACHE_LENGTH)
.unwrap_or_default();
let decoded_index = index % Self::PER_PAGE;
if index >= cached_start {
let trimmed_index = index - cached_start;
if let Some(decoded_pages) = self.decoded_pages.as_ref() {
let decoded_page = decoded_pages
.get(Self::index_to_page_index(trimmed_index))
.unwrap();
return Ok(decoded_page
.get_or_init(|| {
self.decode_page(Self::index_to_page_index(index), mmap)
.unwrap()
})
.get(decoded_index)
.cloned());
}
}
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
let decoded_index = index % Self::PER_PAGE;
Ok(self
.decode_page(page_index, mmap)?
@@ -258,14 +177,6 @@ where
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
self.inner.guard()
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
self.inner.mut_guard()
}
fn stored_len(&self) -> usize {
Self::stored_len_(&self.pages_meta.load())
}
@@ -328,24 +239,16 @@ where
let last_page_index = pages_meta.len() - 1;
values = if let Some(values) = self
.decoded_pages
.as_mut()
.and_then(|v| v.last_mut().and_then(|lock| lock.take()))
{
values
} else {
Self::decode_page_(
stored_len,
last_page_index,
self.guard().as_ref().unwrap(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap()
};
values = Self::decode_page_(
stored_len,
last_page_index,
&self.mmap().load(),
&pages_meta,
)
.inspect_err(|_| {
dbg!(last_page_index, &pages_meta);
})
.unwrap();
truncate_at.replace(pages_meta.pop().unwrap().start);
starting_page_index = last_page_index;
@@ -394,8 +297,6 @@ where
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
Ok(())
}
@@ -404,7 +305,6 @@ where
pages_meta.truncate(0);
pages_meta.write()?;
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
self.file_truncate_and_write_all(0, &[])
}
@@ -424,9 +324,7 @@ where
let page_index = Self::index_to_page_index(index);
let guard = self.guard().as_ref().unwrap();
let values = self.decode_page(page_index, guard)?;
let values = self.decode_page(page_index, &self.mmap().load())?;
let mut buf = vec![];
let mut page = pages_meta.truncate(page_index).unwrap();
@@ -452,8 +350,6 @@ where
self.file_truncate_and_write_all(len, &buf)?;
self.reset_caches();
Ok(())
}
@@ -471,7 +367,6 @@ where
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
decoded_pages: None,
pages_meta: self.pages_meta.clone(),
}
}
@@ -493,6 +388,9 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<T>();
const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
#[inline]
pub fn set(&mut self, i: I) -> &mut Self {
self.index = i.unwrap_to_usize();
@@ -505,14 +403,14 @@ where
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
self.set(i).next()
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.set(i).next().map(|(_, v)| v)
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
self.set_(i);
self.next()
self.next().map(|(_, v)| v)
}
}
@@ -537,15 +435,25 @@ where
.get(j)
.map(|v| (I::from(i), Value::Ref(v)))
} else {
CompressedVec::<I, T>::cached_get_stored__(
i,
mmap,
stored_len,
&mut self.decoded_page,
&self.pages_meta,
)
.unwrap()
.map(|v| (I::from(i), Value::Owned(v)))
let page_index = i / Self::PER_PAGE;
if self.decoded_page.as_ref().is_none_or(|b| b.0 != page_index) {
let values = CompressedVec::<I, T>::decode_page_(
stored_len,
page_index,
mmap,
&self.pages_meta,
)
.unwrap();
self.decoded_page.replace((page_index, values));
}
self.decoded_page
.as_ref()
.unwrap()
.1
.get(i % Self::PER_PAGE)
.map(|v| (I::from(i), Value::Owned(v.clone())))
};
self.index += 1;
@@ -562,8 +470,9 @@ where
if len == 0 {
return None;
}
self.get_(len - 1)
.map(|(i, v)| (i, Value::Owned(v.into_inner())))
let i = len - 1;
self.get_(i)
.map(|v| (I::from(i), Value::Owned(v.into_inner())))
}
}
crates/brk_vec/src/variants/raw.rs
+19 -45
View File
@@ -20,7 +20,6 @@ pub struct RawVec<I, T> {
pathbuf: PathBuf,
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: Arc<ArcSwap<Mmap>>,
guard: Option<Guard<Arc<Mmap>>>,
pushed: Vec<T>,
phantom: PhantomData<I>,
}
@@ -51,11 +50,9 @@ where
let file = Self::open_file_(Self::path_vec_(path).as_path())?;
let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
let guard = Some(mmap.load());
Ok(Self {
mmap,
guard,
version,
pathbuf: path.to_owned(),
pushed: vec![],
@@ -90,7 +87,7 @@ where
type T = T;
#[inline]
fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
let index = index * Self::SIZE_OF_T;
let slice = &mmap[index..(index + Self::SIZE_OF_T)];
Self::T::try_read_from_bytes(slice)
@@ -103,22 +100,9 @@ where
&self.mmap
}
#[inline]
fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
&self.guard
}
#[inline]
fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
&mut self.guard
}
#[inline]
fn stored_len(&self) -> usize {
if let Some(guard) = self.guard() {
guard.len() / Self::SIZE_OF_T
} else {
self.mmap.load().len() / Self::SIZE_OF_T
}
self.mmap.load().len() / Self::SIZE_OF_T
}
#[inline]
@@ -218,9 +202,7 @@ where
Self {
version: self.version,
pathbuf: self.pathbuf.clone(),
// Consider Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
mmap: self.mmap.clone(),
guard: None,
pushed: vec![],
phantom: PhantomData,
}
@@ -239,8 +221,6 @@ where
I: StoredIndex,
T: StoredType,
{
const SIZE_OF_T: usize = size_of::<T>();
#[inline]
pub fn set(&mut self, i: I) -> &mut Self {
self.index = i.unwrap_to_usize();
@@ -253,14 +233,14 @@ where
}
#[inline]
pub fn get(&mut self, i: I) -> Option<(I, Value<'_, T>)> {
self.set(i).next()
pub fn get(&mut self, i: I) -> Option<Value<'_, T>> {
self.set(i).next().map(|(_, v)| v)
}
#[inline]
pub fn get_(&mut self, i: usize) -> Option<(I, Value<'_, T>)> {
pub fn get_(&mut self, i: usize) -> Option<Value<'_, T>> {
self.set_(i);
self.next()
self.next().map(|(_, v)| v)
}
}
@@ -272,28 +252,21 @@ where
type Item = (I, Value<'a, T>);
fn next(&mut self) -> Option<Self::Item> {
let mmap = &self.guard;
let vec = self.vec;
let i = self.index;
let index = self.index;
let stored_len = mmap.len() / Self::SIZE_OF_T;
let opt = self
.vec
.get_or_read_(index, mmap)
.unwrap()
.map(|v| (I::from(index), v));
let result = if i >= stored_len {
let j = i - stored_len;
if j >= vec.pushed_len() {
return None;
}
vec.pushed().get(j).map(|v| (I::from(i), Value::Ref(v)))
} else {
vec.get_stored_(i, mmap)
.unwrap()
.map(|v| (I::from(i), Value::Owned(v)))
};
if opt.is_some() {
self.index += 1;
}
self.index += 1;
result
opt
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
@@ -302,8 +275,9 @@ where
if len == 0 {
return None;
}
self.get_(len - 1)
.map(|(i, v)| (i, Value::Owned(v.into_inner())))
let i = len - 1;
self.get_(i)
.map(|v| (I::from(i), Value::Owned(v.into_inner())))
}
}