Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-10 14:19:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

vec: caching only in iter

Browse Source
This commit is contained in:
nym21
2025-04-30 18:29:18 +02:00
parent 664b125ce2
commit 700352ec45
22 changed files with 411 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/brk_indexer/examples/main.rs
View File

@@ -24,7 +24,7 @@ fn main() -> color_eyre::Result<()> {
let outputs = Path::new("../../_outputs");
let mut indexer = Indexer::new(outputs, false, true)?;
let mut indexer = Indexer::new(outputs, false, false)?;
indexer.import_stores()?;
indexer.import_vecs()?;

79
crates/brk_indexer/src/indexes.rs
View File

@@ -5,7 +5,7 @@ use brk_core::{
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{Result, StoredIndex, StoredType, Value};
use brk_vec::{StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::{IndexedVec, Stores, Vecs};
@@ -107,115 +107,116 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
})
.unwrap();
vecs.height_to_blockhash.get(*height).map_or(true, |opt| {
opt.is_none_or(|saved_blockhash| {
vecs.height_to_blockhash
.iter()
.get(*height)
.is_none_or(|saved_blockhash| {
let b = &rpc_blockhash != saved_blockhash.as_ref();
if b {
dbg!(rpc_blockhash, saved_blockhash.as_ref());
}
b
})
})
})
.unwrap_or(starting_height);
Ok(Self {
emptyoutputindex: *starting_index(
emptyoutputindex: starting_index(
&vecs.height_to_first_emptyoutputindex,
&vecs.emptyoutputindex_to_txindex,
height,
)?
)
.context("")?,
height,
p2msindex: *starting_index(
p2msindex: starting_index(
&vecs.height_to_first_p2msindex,
&vecs.p2msindex_to_txindex,
height,
)?
)
.context("")?,
opreturnindex: *starting_index(
opreturnindex: starting_index(
&vecs.height_to_first_opreturnindex,
&vecs.opreturnindex_to_txindex,
height,
)?
)
.context("")?,
p2pk33index: *starting_index(
p2pk33index: starting_index(
&vecs.height_to_first_p2pk33index,
&vecs.p2pk33index_to_p2pk33bytes,
height,
)?
)
.context("")?,
p2pk65index: *starting_index(
p2pk65index: starting_index(
&vecs.height_to_first_p2pk65index,
&vecs.p2pk65index_to_p2pk65bytes,
height,
)?
)
.context("")?,
p2pkhindex: *starting_index(
p2pkhindex: starting_index(
&vecs.height_to_first_p2pkhindex,
&vecs.p2pkhindex_to_p2pkhbytes,
height,
)?
)
.context("")?,
p2shindex: *starting_index(
p2shindex: starting_index(
&vecs.height_to_first_p2shindex,
&vecs.p2shindex_to_p2shbytes,
height,
)?
)
.context("")?,
p2trindex: *starting_index(
p2trindex: starting_index(
&vecs.height_to_first_p2trindex,
&vecs.p2trindex_to_p2trbytes,
height,
)?
)
.context("")?,
p2wpkhindex: *starting_index(
p2wpkhindex: starting_index(
&vecs.height_to_first_p2wpkhindex,
&vecs.p2wpkhindex_to_p2wpkhbytes,
height,
)?
)
.context("")?,
p2wshindex: *starting_index(
p2wshindex: starting_index(
&vecs.height_to_first_p2wshindex,
&vecs.p2wshindex_to_p2wshbytes,
height,
)?
)
.context("")?,
p2aindex: *starting_index(
p2aindex: starting_index(
&vecs.height_to_first_p2aindex,
&vecs.p2aindex_to_p2abytes,
height,
)?
)
.context("")?,
txindex: *starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)?
txindex: starting_index(&vecs.height_to_first_txindex, &vecs.txindex_to_txid, height)
.context("")?,
inputindex: *starting_index(
inputindex: starting_index(
&vecs.height_to_first_inputindex,
&vecs.inputindex_to_outputindex,
height,
)?
)
.context("")?,
outputindex: *starting_index(
outputindex: starting_index(
&vecs.height_to_first_outputindex,
&vecs.outputindex_to_value,
height,
)?
)
.context("")?,
unknownoutputindex: *starting_index(
unknownoutputindex: starting_index(
&vecs.height_to_first_unknownoutputindex,
&vecs.unknownoutputindex_to_txindex,
height,
)?
)
.context("")?,
})
}
}
pub fn starting_index<'a, I, T>(
height_to_index: &'a IndexedVec<Height, I>,
index_to_else: &'a IndexedVec<I, T>,
pub fn starting_index<I, T>(
height_to_index: &IndexedVec<Height, I>,
index_to_else: &IndexedVec<I, T>,
starting_height: Height,
) -> Result<Option<Value<'a, I>>>
) -> Option<I>
where
I: StoredType + StoredIndex + From<usize>,
T: StoredType,
@@ -224,8 +225,8 @@ where
.height()
.is_ok_and(|h| h + 1_u32 == starting_height)
{
Ok(Some(Value::Owned(I::from(index_to_else.len()))))
Some(I::from(index_to_else.len()))
} else {
height_to_index.get(starting_height)
height_to_index.iter().get_inner(starting_height)
}
}

69
crates/brk_indexer/src/lib.rs
View File

@@ -18,7 +18,7 @@ pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::Compressed;
use brk_vec::{Compressed, DynamicVec};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -242,7 +242,11 @@ impl Indexer {
})
});
let input_source_vec_handle = scope.spawn(|| {
let txindex_to_first_outputindex_mmap = vecs
.txindex_to_first_outputindex.vec().mmap().load();
inputs
.into_par_iter()
.enumerate()
@@ -273,13 +277,11 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let outputindex = *vecs
.txindex_to_first_outputindex
.get(prev_txindex)?
let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, &txindex_to_first_outputindex_mmap)?
.context("Expect outputindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
})?
})?.into_inner()
+ vout;
Ok((inputindex, InputSource::PreviousBlock((
@@ -305,6 +307,16 @@ impl Indexer {
});
let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
let p2pk65index_to_p2pk65bytes_mmap = vecs
.p2pk65index_to_p2pk65bytes.vec().mmap().load();
let p2pk33index_to_p2pk33bytes_mmap = vecs.p2pk33index_to_p2pk33bytes.vec().mmap().load();
let p2pkhindex_to_p2pkhbytes_mmap = vecs.p2pkhindex_to_p2pkhbytes.vec().mmap().load();
let p2shindex_to_p2shbytes_mmap = vecs.p2shindex_to_p2shbytes.vec().mmap().load();
let p2wpkhindex_to_p2wpkhbytes_mmap = vecs.p2wpkhindex_to_p2wpkhbytes.vec().mmap().load();
let p2wshindex_to_p2wshbytes_mmap = vecs.p2wshindex_to_p2wshbytes.vec().mmap().load();
let p2trindex_to_p2trbytes_mmap = vecs.p2trindex_to_p2trbytes.vec().mmap().load();
let p2aindex_to_p2abytes_mmap = vecs.p2aindex_to_p2abytes.vec().mmap().load();
outputs
.into_par_iter()
.enumerate()
@@ -349,8 +361,43 @@ impl Indexer {
if let Some(Some(outputtypeindex)) = check_collisions.then_some(outputtypeindex_opt) {
let addressbytes = address_bytes_res.as_ref().unwrap();
let prev_addressbytes_opt =
vecs.get_addressbytes(outputtype, outputtypeindex)?;
let prev_addressbytes_opt = match outputtype {
OutputType::P2PK65 => vecs
.p2pk65index_to_p2pk65bytes
.get_or_read(outputtypeindex.into(), &p2pk65index_to_p2pk65bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => vecs
.p2pk33index_to_p2pk33bytes
.get_or_read(outputtypeindex.into(), &p2pk33index_to_p2pk33bytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => vecs
.p2pkhindex_to_p2pkhbytes
.get_or_read(outputtypeindex.into(), &p2pkhindex_to_p2pkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => vecs
.p2shindex_to_p2shbytes
.get_or_read(outputtypeindex.into(), &p2shindex_to_p2shbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => vecs
.p2wpkhindex_to_p2wpkhbytes
.get_or_read(outputtypeindex.into(), &p2wpkhindex_to_p2wpkhbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => vecs
.p2wshindex_to_p2wshbytes
.get_or_read(outputtypeindex.into(), &p2wshindex_to_p2wshbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => vecs
.p2trindex_to_p2trbytes
.get_or_read(outputtypeindex.into(), &p2trindex_to_p2trbytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => vecs
.p2aindex_to_p2abytes
.get_or_read(outputtypeindex.into(), &p2aindex_to_p2abytes_mmap)?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
};
let prev_addressbytes =
prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
@@ -600,6 +647,9 @@ impl Indexer {
let mut txindex_to_tx_and_txid: BTreeMap<TxIndex, (&Transaction, Txid)> = BTreeMap::default();
let mut txindex_to_txid_iter = vecs
.txindex_to_txid.iter();
txid_prefix_to_txid_and_block_txindex_and_prev_txindex
.into_iter()
.try_for_each(
@@ -626,9 +676,8 @@ impl Indexer {
let len = vecs.txindex_to_txid.len();
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.get(prev_txindex)?
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, len);

77
crates/brk_indexer/src/stores/mod.rs
View File

@@ -96,12 +96,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk65index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk65index_to_p2pk65bytes
.get(index)?
let mut p2pk65index_to_p2pk65bytes_iter = vecs.p2pk65index_to_p2pk65bytes.iter();
while let Some(typedbytes) = p2pk65index_to_p2pk65bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -113,12 +115,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pk33index
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pk33index_to_p2pk33bytes
.get(index)?
let mut p2pk33index_to_p2pk33bytes_iter = vecs.p2pk33index_to_p2pk33bytes.iter();
while let Some(typedbytes) = p2pk33index_to_p2pk33bytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -130,12 +134,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2pkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2pkhindex_to_p2pkhbytes
.get(index)?
let mut p2pkhindex_to_p2pkhbytes_iter = vecs.p2pkhindex_to_p2pkhbytes.iter();
while let Some(typedbytes) = p2pkhindex_to_p2pkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -147,12 +153,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2shindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2shindex_to_p2shbytes
.get(index)?
let mut p2shindex_to_p2shbytes_iter = vecs.p2shindex_to_p2shbytes.iter();
while let Some(typedbytes) = p2shindex_to_p2shbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -164,12 +172,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2trindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2trindex_to_p2trbytes
.get(index)?
let mut p2trindex_to_p2trbytes_iter = vecs.p2trindex_to_p2trbytes.iter();
while let Some(typedbytes) = p2trindex_to_p2trbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -181,12 +191,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wpkhindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wpkhindex_to_p2wpkhbytes
.get(index)?
let mut p2wpkhindex_to_p2wpkhbytes_iter = vecs.p2wpkhindex_to_p2wpkhbytes.iter();
while let Some(typedbytes) = p2wpkhindex_to_p2wpkhbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -198,12 +210,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2wshindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
while let Some(typedbytes) = vecs
.p2wshindex_to_p2wshbytes
.get(index)?
let mut p2wshindex_to_p2wshbytes_iter = vecs.p2wshindex_to_p2wshbytes.iter();
while let Some(typedbytes) = p2wshindex_to_p2wshbytes_iter
.get(index)
.map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
@@ -215,11 +229,14 @@ impl Stores {
if let Some(mut index) = vecs
.height_to_first_p2aindex
.get(starting_indexes.height)?
.iter()
.get(starting_indexes.height)
.map(Value::into_inner)
{
let mut p2aindex_to_p2abytes_iter = vecs.p2aindex_to_p2abytes.iter();
while let Some(typedbytes) =
vecs.p2aindex_to_p2abytes.get(index)?.map(Value::into_inner)
p2aindex_to_p2abytes_iter.get(index).map(Value::into_inner)
{
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from((&bytes, OutputType::P2A));

12
crates/brk_indexer/src/vecs/base.rs
View File

@@ -5,7 +5,7 @@ use std::{
};
use brk_vec::{
Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec,
Compressed, DynamicVec, Error, GenericVec, Mmap, Result, StoredIndex, StoredType, StoredVec,
StoredVecIterator, Value, Version,
};
@@ -31,19 +31,15 @@ where
version: Version,
compressed: Compressed,
) -> brk_vec::Result<Self> {
let mut inner = StoredVec::forced_import(path, version, compressed)?;
inner.enable_large_cache_if_needed();
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
inner,
inner: StoredVec::forced_import(path, version, compressed)?,
})
}
#[inline]
pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
self.inner.get(index)
pub fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Value<T>>> {
self.inner.get_or_read(index, mmap)
}
#[inline]

66
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -79,7 +79,7 @@ impl Vecs {
height_to_blockhash: IndexedVec::forced_import(
&path.join("height_to_blockhash"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
height_to_difficulty: IndexedVec::forced_import(
&path.join("height_to_difficulty"),
@@ -204,7 +204,7 @@ impl Vecs {
p2aindex_to_p2abytes: IndexedVec::forced_import(
&path.join("p2aindex_to_p2abytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2msindex_to_txindex: IndexedVec::forced_import(
&path.join("p2msindex_to_txindex"),
@@ -214,37 +214,37 @@ impl Vecs {
p2pk33index_to_p2pk33bytes: IndexedVec::forced_import(
&path.join("p2pk33index_to_p2pk33bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pk65index_to_p2pk65bytes: IndexedVec::forced_import(
&path.join("p2pk65index_to_p2pk65bytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import(
&path.join("p2pkhindex_to_p2pkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2shindex_to_p2shbytes: IndexedVec::forced_import(
&path.join("p2shindex_to_p2shbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2trindex_to_p2trbytes: IndexedVec::forced_import(
&path.join("p2trindex_to_p2trbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import(
&path.join("p2wpkhindex_to_p2wpkhbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
p2wshindex_to_p2wshbytes: IndexedVec::forced_import(
&path.join("p2wshindex_to_p2wshbytes"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_base_size: IndexedVec::forced_import(
&path.join("txindex_to_base_size"),
@@ -259,7 +259,7 @@ impl Vecs {
txindex_to_first_outputindex: IndexedVec::forced_import(
&path.join("txindex_to_first_outputindex"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
&path.join("txindex_to_is_explicitly_rbf"),
@@ -279,7 +279,7 @@ impl Vecs {
txindex_to_txid: IndexedVec::forced_import(
&path.join("txindex_to_txid"),
Version::ZERO,
compressed,
Compressed::NO,
)?,
txindex_to_txversion: IndexedVec::forced_import(
&path.join("txindex_to_txversion"),
@@ -408,50 +408,6 @@ impl Vecs {
Ok(())
}
pub fn get_addressbytes(
&self,
outputtype: OutputType,
outputtypeindex: OutputTypeIndex,
) -> brk_vec::Result<Option<AddressBytes>> {
Ok(match outputtype {
OutputType::P2PK65 => self
.p2pk65index_to_p2pk65bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PK33 => self
.p2pk33index_to_p2pk33bytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2PKH => self
.p2pkhindex_to_p2pkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2SH => self
.p2shindex_to_p2shbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WPKH => self
.p2wpkhindex_to_p2wpkhbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2WSH => self
.p2wshindex_to_p2wshbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2TR => self
.p2trindex_to_p2trbytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::P2A => self
.p2aindex_to_p2abytes
.get(outputtypeindex.into())?
.map(|v| AddressBytes::from(v.into_inner())),
OutputType::Empty | OutputType::OpReturn | OutputType::P2MS | OutputType::Unknown => {
unreachable!()
}
})
}
pub fn push_bytes_if_needed(
&mut self,
index: OutputTypeIndex,