Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-12 13:18:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-28 11:52:25 +01:00
parent 5b1ca3711a
commit 1b93ccf608
35 changed files with 460 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/brk_indexer/src/structs/indexes.rs → crates/brk_indexer/src/indexes.rs
View File

@@ -7,9 +7,9 @@ use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::CACHED_GETS;
use color_eyre::eyre::ContextCompat;
use crate::storage::{Stores, Vecs};
use crate::{Stores, Vecs};
#[derive(Debug, Default)]
#[derive(Debug, Default, Clone)]
pub struct Indexes {
pub addressindex: Addressindex,
pub emptyindex: Emptyindex,

25
crates/brk_indexer/src/lib.rs
View File

@@ -1,3 +1,5 @@
#![doc = include_str!("../README.md")]
use std::{
collections::BTreeMap,
path::Path,
@@ -17,13 +19,13 @@ use brk_vec::CACHED_GETS;
use color_eyre::eyre::{ContextCompat, eyre};
use log::info;
use rayon::prelude::*;
mod indexes;
mod stores;
mod vecs;
mod storage;
mod structs;
pub use storage::{AnyStorableVec, StorableVec, Store, StoreMeta};
use storage::{Stores, Vecs};
pub use structs::*;
pub use indexes::*;
pub use stores::*;
pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
@@ -47,7 +49,7 @@ impl<const MODE: u8> Indexer<MODE> {
}
impl Indexer<CACHED_GETS> {
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
pub fn index(&mut self, parser: &Parser, rpc: &'static rpc::Client, exit: &Exit) -> color_eyre::Result<Indexes> {
let check_collisions = true;
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc)).unwrap_or_else(|_| {
@@ -82,14 +84,13 @@ impl Indexer<CACHED_GETS> {
let vecs = &mut self.vecs;
let stores = &mut self.stores;
let mut idxs = starting_indexes;
if idxs.height > Height::try_from(rpc)? {
return Ok(());
if starting_indexes.height > Height::try_from(rpc)? {
return Ok(starting_indexes);
}
info!("Started indexing...");
let mut idxs = starting_indexes.clone();
parser.parse(Some(idxs.height), None).iter().try_for_each(
|(height, block, blockhash)| -> color_eyre::Result<()> {
info!("Indexing block {height}...");
@@ -633,7 +634,7 @@ impl Indexer<CACHED_GETS> {
export_if_needed(stores, vecs, idxs.height, true, exit)?;
Ok(())
Ok(starting_indexes)
}
}

3
crates/brk_indexer/src/main.rs
View File

@@ -40,6 +40,5 @@ fn main() -> color_eyre::Result<()> {
}
#[allow(unreachable_code)]
// To make sure that Fjall had the time to flush everything properly
sleep(Duration::from_millis(100));
Ok(())
}

5
crates/brk_indexer/src/storage/mod.rs
View File

@@ -1,5 +0,0 @@
mod stores;
mod vecs;
pub use stores::*;
pub use vecs::*;

25
crates/brk_indexer/src/storage/stores/base.rs → crates/brk_indexer/src/stores/base.rs
View File

@@ -29,8 +29,8 @@ const CHECK_COLLISISONS: bool = true;
impl<K, V> Store<K, V>
where
K: Debug + Into<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Into<ByteView> + TryFrom<ByteView>,
K: Debug + Clone + Into<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
@@ -161,7 +161,26 @@ where
fn open_partition_handle(keyspace: &TransactionalKeyspace) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
"partition",
PartitionCreateOptions::default().manual_journal_persist(true),
PartitionCreateOptions::default()
.bloom_filter_bits(Some(5))
.manual_journal_persist(true),
)
}
}
impl<Key, Value> Clone for Store<Key, Value>
where
Key: Clone,
Value: Clone,
{
fn clone(&self) -> Self {
Self {
meta: self.meta.clone(),
keyspace: self.keyspace.clone(),
part: self.part.clone(),
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
}
}
}

2
crates/brk_indexer/src/storage/stores/meta.rs → crates/brk_indexer/src/stores/meta.rs
View File

@@ -8,7 +8,7 @@ use zerocopy::{FromBytes, IntoBytes};
use super::Height;
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct StoreMeta {
pathbuf: PathBuf,
version: Version,

1
crates/brk_indexer/src/storage/stores/mod.rs → crates/brk_indexer/src/stores/mod.rs
View File

@@ -13,6 +13,7 @@ pub use meta::*;
use super::Vecs;
#[derive(Clone)]
pub struct Stores {
pub addresshash_to_addressindex: Store<AddressHash, Addressindex>,
pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,

3
crates/brk_indexer/src/structs/mod.rs
View File

@@ -1,3 +0,0 @@
mod indexes;
pub use indexes::*;

20
crates/brk_indexer/src/storage/vecs/base.rs → crates/brk_indexer/src/vecs/base.rs
View File

@@ -5,7 +5,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_vec::{StoredIndex, StoredType, Version};
use brk_vec::{CACHED_GETS, StoredIndex, StoredType, Version};
use super::Height;
@@ -27,11 +27,6 @@ where
})
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.vec.flush()
}
pub fn truncate_if_needed(&mut self, index: I, height: Height) -> brk_vec::Result<Option<T>> {
if self.height.is_none_or(|self_height| self_height != height) {
height.write(&self.path_height())?;
@@ -50,6 +45,17 @@ where
}
}
impl<I, T> StorableVec<I, T, CACHED_GETS>
where
I: StoredIndex,
T: StoredType,
{
pub fn flush(&mut self, height: Height) -> io::Result<()> {
height.write(&self.path_height())?;
self.vec.flush()
}
}
impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> {
type Target = brk_vec::StorableVec<I, T, MODE>;
fn deref(&self) -> &Self::Target {
@@ -67,7 +73,7 @@ pub trait AnyStorableVec: Send + Sync {
fn flush(&mut self, height: Height) -> io::Result<()>;
}
impl<I, T, const MODE: u8> AnyStorableVec for StorableVec<I, T, MODE>
impl<I, T> AnyStorableVec for StorableVec<I, T, CACHED_GETS>
where
I: StoredIndex,
T: StoredType,

128
crates/brk_indexer/src/storage/vecs/mod.rs → crates/brk_indexer/src/vecs/mod.rs
View File

@@ -293,22 +293,8 @@ impl<const MODE: u8> Vecs<MODE> {
Ok(())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_any_vec_slice()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn starting_height(&mut self) -> Height {
self.as_mut_any_vec_slice()
.into_iter()
.map(|vec| vec.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 43] {
[
pub fn as_any_json_vecs(&self) -> Vec<&dyn AnyJsonStorableVec> {
vec![
&*self.addressindex_to_addresstype as &dyn AnyJsonStorableVec,
&*self.addressindex_to_addresstypeindex,
&*self.addressindex_to_height,
@@ -354,54 +340,6 @@ impl<const MODE: u8> Vecs<MODE> {
&*self.txoutindex_to_value,
]
}
pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 43] {
[
&mut self.addressindex_to_addresstype as &mut dyn AnyStorableVec,
&mut self.addressindex_to_addresstypeindex,
&mut self.addressindex_to_height,
&mut self.height_to_blockhash,
&mut self.height_to_difficulty,
&mut self.height_to_first_addressindex,
&mut self.height_to_first_emptyindex,
&mut self.height_to_first_multisigindex,
&mut self.height_to_first_opreturnindex,
&mut self.height_to_first_pushonlyindex,
&mut self.height_to_first_txindex,
&mut self.height_to_first_txinindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_first_unknownindex,
&mut self.height_to_first_p2pk33index,
&mut self.height_to_first_p2pk65index,
&mut self.height_to_first_p2pkhindex,
&mut self.height_to_first_p2shindex,
&mut self.height_to_first_p2trindex,
&mut self.height_to_first_p2wpkhindex,
&mut self.height_to_first_p2wshindex,
&mut self.height_to_size,
&mut self.height_to_timestamp,
&mut self.height_to_weight,
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
&mut self.p2shindex_to_p2shaddressbytes,
&mut self.p2trindex_to_p2traddressbytes,
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
&mut self.p2wshindex_to_p2wshaddressbytes,
&mut self.txindex_to_first_txinindex,
&mut self.txindex_to_first_txoutindex,
&mut self.txindex_to_height,
&mut self.txindex_to_locktime,
&mut self.txindex_to_txid,
&mut self.txindex_to_base_size,
&mut self.txindex_to_total_size,
&mut self.txindex_to_is_explicitly_rbf,
&mut self.txindex_to_txversion,
&mut self.txinindex_to_txoutindex,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_value,
]
}
}
impl Vecs<CACHED_GETS> {
@@ -471,4 +409,66 @@ impl Vecs<CACHED_GETS> {
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index.into(), bytes),
}
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_any_vecs()
.into_par_iter()
.try_for_each(|vec| vec.flush(height))
}
pub fn starting_height(&mut self) -> Height {
self.as_mut_any_vecs()
.into_iter()
.map(|vec| vec.height().map(Height::incremented).unwrap_or_default())
.min()
.unwrap()
}
fn as_mut_any_vecs(&mut self) -> Vec<&mut dyn AnyStorableVec> {
vec![
&mut self.addressindex_to_addresstype as &mut dyn AnyStorableVec,
&mut self.addressindex_to_addresstypeindex,
&mut self.addressindex_to_height,
&mut self.height_to_blockhash,
&mut self.height_to_difficulty,
&mut self.height_to_first_addressindex,
&mut self.height_to_first_emptyindex,
&mut self.height_to_first_multisigindex,
&mut self.height_to_first_opreturnindex,
&mut self.height_to_first_pushonlyindex,
&mut self.height_to_first_txindex,
&mut self.height_to_first_txinindex,
&mut self.height_to_first_txoutindex,
&mut self.height_to_first_unknownindex,
&mut self.height_to_first_p2pk33index,
&mut self.height_to_first_p2pk65index,
&mut self.height_to_first_p2pkhindex,
&mut self.height_to_first_p2shindex,
&mut self.height_to_first_p2trindex,
&mut self.height_to_first_p2wpkhindex,
&mut self.height_to_first_p2wshindex,
&mut self.height_to_size,
&mut self.height_to_timestamp,
&mut self.height_to_weight,
&mut self.p2pk33index_to_p2pk33addressbytes,
&mut self.p2pk65index_to_p2pk65addressbytes,
&mut self.p2pkhindex_to_p2pkhaddressbytes,
&mut self.p2shindex_to_p2shaddressbytes,
&mut self.p2trindex_to_p2traddressbytes,
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
&mut self.p2wshindex_to_p2wshaddressbytes,
&mut self.txindex_to_first_txinindex,
&mut self.txindex_to_first_txoutindex,
&mut self.txindex_to_height,
&mut self.txindex_to_locktime,
&mut self.txindex_to_txid,
&mut self.txindex_to_base_size,
&mut self.txindex_to_total_size,
&mut self.txindex_to_is_explicitly_rbf,
&mut self.txindex_to_txversion,
&mut self.txinindex_to_txoutindex,
&mut self.txoutindex_to_addressindex,
&mut self.txoutindex_to_value,
]
}
}