Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-04 19:29:09 -07:00
Code Issues Packages Projects Releases Wiki Activity

storable_vec: add modes

Browse Source
This commit is contained in:
nym21
2025-02-04 20:56:48 +01:00
parent 42c996e16e
commit d11a1622f8
16 changed files with 696 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
indexer/src/lib.rs
View File

@@ -13,6 +13,7 @@ use bitcoin::{Transaction, TxIn, TxOut, Txid};
use color_eyre::eyre::{eyre, ContextCompat};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::CACHED_GETS;
mod storage;
mod structs;
@@ -29,26 +30,20 @@ pub use structs::{
const UNSAFE_BLOCKS: u32 = 100;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
pub struct Indexer {
vecs: StorableVecs,
trees: Fjalls,
pub struct Indexer<const MODE: u8> {
pub vecs: StorableVecs<MODE>,
pub trees: Fjalls,
}
impl Indexer {
impl<const MODE: u8> Indexer<MODE> {
pub fn import(indexes_dir: &Path) -> color_eyre::Result<Self> {
let vecs = StorableVecs::import(&indexes_dir.join("vecs"))?;
let trees = Fjalls::import(&indexes_dir.join("fjall"))?;
Ok(Self { vecs, trees })
}
}
pub fn vecs(&self) -> &StorableVecs {
&self.vecs
}
pub fn trees(&self) -> &Fjalls {
&self.trees
}
impl Indexer<CACHED_GETS> {
pub fn index(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
let check_collisions = true;
@@ -80,22 +75,23 @@ impl Indexer {
let mut p2wpkhindex_global = vecs.height_to_first_p2wpkhindex.get_or_default(height)?;
let mut p2wshindex_global = vecs.height_to_first_p2wshindex.get_or_default(height)?;
let export = |trees: &mut Fjalls, vecs: &mut StorableVecs, height: Height| -> color_eyre::Result<()> {
println!("Exporting...");
let export =
|trees: &mut Fjalls, vecs: &mut StorableVecs<CACHED_GETS>, height: Height| -> color_eyre::Result<()> {
println!("Exporting...");
exit.block();
exit.block();
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
trees.commit(height)?;
vecs_handle.join().unwrap()?;
Ok(())
})?;
exit.unblock();
thread::scope(|scope| -> color_eyre::Result<()> {
let vecs_handle = scope.spawn(|| vecs.flush(height));
trees.commit(height)?;
vecs_handle.join().unwrap()?;
Ok(())
})?;
exit.unblock();
Ok(())
};
};
biter::new(bitcoin_dir, Some(height.into()), None, rpc)
.iter()
@@ -104,7 +100,7 @@ impl Indexer {
height = Height::from(_height);
if let Some(saved_blockhash) = vecs.height_to_blockhash.cached_get(height)? {
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
if &blockhash != saved_blockhash.as_ref() {
todo!("Rollback not implemented");
// trees.rollback_from(&mut rtx, height, &exit)?;
@@ -260,7 +256,7 @@ impl Indexer {
let txoutindex = *vecs
.txindex_to_first_txoutindex
.cached_get(prev_txindex)?
.get(prev_txindex)?
.context("Expect txoutindex to not be none")
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
@@ -336,12 +332,12 @@ impl Indexer {
let prev_addresstype = *vecs
.addressindex_to_addresstype
.cached_get(addressindex)?
.get(addressindex)?
.context("Expect to have address type")?;
let addresstypeindex = *vecs
.addressindex_to_addresstypeindex
.cached_get(addressindex)?
.get(addressindex)?
.context("Expect to have address type index")?;
// Good first time
// Wrong after rerun
@@ -605,7 +601,7 @@ impl Indexer {
// Ok if `get` is not par as should happen only twice
let prev_txid = vecs
.txindex_to_txid
.cached_get(prev_txindex)?
.get(prev_txindex)?
.context("To have txid for txindex")
.inspect_err(|_| {
dbg!(txindex, txid, len);
@@ -630,7 +626,7 @@ impl Indexer {
if !is_dup {
let prev_height =
vecs.txindex_to_height.cached_get(prev_txindex)?.expect("To have height");
vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
dbg!(height, txid, txindex, prev_height, prev_txid, prev_txindex);
return Err(eyre!("Expect none"));
}

3
indexer/src/main.rs
View File

@@ -3,6 +3,7 @@ use std::path::Path;
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
use storable_vec::CACHED_GETS;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -16,7 +17,7 @@ fn main() -> color_eyre::Result<()> {
let i = std::time::Instant::now();
let mut indexer = Indexer::import(Path::new("../_outputs/indexes"))?;
let mut indexer: Indexer<CACHED_GETS> = Indexer::import(Path::new("../_outputs/indexes"))?;
indexer.index(data_dir, rpc, &exit)?;

25
indexer/src/storage/storable_vecs/base.rs
View File

@@ -5,25 +5,25 @@ use std::{
path::{Path, PathBuf},
};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use storable_vec::{StorableVecIndex, StorableVecType, Version, CACHED_GETS};
use super::Height;
#[derive(Debug)]
pub struct StorableVec<I, T> {
pub struct StorableVec<I, T, const MODE: u8> {
height: Option<Height>,
vec: storable_vec::StorableVec<I, T>,
vec: storable_vec::StorableVec<I, T, MODE>,
}
impl<I, T> StorableVec<I, T>
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
pub fn import(path: &Path, version: Version) -> storable_vec::Result<Self> {
Ok(Self {
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
vec: storable_vec::StorableVec::import(path, version)?,
vec: storable_vec::StorableVec::forced_import(path, version)?,
})
}
@@ -53,13 +53,13 @@ where
}
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = storable_vec::StorableVec<I, T>;
impl<I, T, const MODE: u8> Deref for StorableVec<I, T, MODE> {
type Target = storable_vec::StorableVec<I, T, MODE>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
impl<I, T, const MODE: u8> DerefMut for StorableVec<I, T, MODE> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
@@ -68,10 +68,9 @@ impl<I, T> DerefMut for StorableVec<I, T> {
pub trait AnyStorableVec {
fn height(&self) -> color_eyre::Result<Height>;
fn flush(&mut self, height: Height) -> io::Result<()>;
fn reset_cache(&mut self);
}
impl<I, T> AnyStorableVec for StorableVec<I, T>
impl<I, T, const MODE: u8> AnyStorableVec for StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
@@ -80,10 +79,6 @@ where
self.height()
}
fn reset_cache(&mut self) {
self.vec.reset_cache()
}
fn flush(&mut self, height: Height) -> io::Result<()> {
self.flush(height)
}

214
indexer/src/storage/storable_vecs/mod.rs
View File

@@ -3,7 +3,7 @@ use std::{fs, io, path::Path};
use biter::bitcoin::{self, transaction, BlockHash, Txid, Weight};
use exit::Exit;
use rayon::prelude::*;
use storable_vec::Version;
use storable_vec::{Version, CACHED_GETS};
use crate::structs::{
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Height, P2PK33AddressBytes, P2PK65AddressBytes,
@@ -15,52 +15,52 @@ mod base;
pub use base::*;
pub struct StorableVecs {
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
pub addressindex_to_height: StorableVec<Addressindex, Height>,
pub height_to_blockhash: StorableVec<Height, BlockHash>,
pub height_to_difficulty: StorableVec<Height, f64>,
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
pub height_to_first_emptyindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_multisigindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_opreturnindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_pushonlyindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_txindex: StorableVec<Height, Txindex>,
pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex>,
pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex>,
pub height_to_size: StorableVec<Height, usize>,
pub height_to_timestamp: StorableVec<Height, Timestamp>,
pub height_to_weight: StorableVec<Height, Weight>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_locktime: StorableVec<Txindex, bitcoin::absolute::LockTime>,
pub txindex_to_txid: StorableVec<Txindex, Txid>,
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version>,
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
pub struct StorableVecs<const MODE: u8> {
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype, MODE>,
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex, MODE>,
pub addressindex_to_height: StorableVec<Addressindex, Height, MODE>,
pub height_to_blockhash: StorableVec<Height, BlockHash, MODE>,
pub height_to_difficulty: StorableVec<Height, f64, MODE>,
pub height_to_first_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_first_emptyindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_multisigindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_opreturnindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_pushonlyindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_txindex: StorableVec<Height, Txindex, MODE>,
pub height_to_first_txinindex: StorableVec<Height, Txinindex, MODE>,
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex, MODE>,
pub height_to_first_unknownindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pk33index: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pk65index: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2pkhindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2shindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2trindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2wpkhindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_first_p2wshindex: StorableVec<Height, Addresstypeindex, MODE>,
pub height_to_size: StorableVec<Height, usize, MODE>,
pub height_to_timestamp: StorableVec<Height, Timestamp, MODE>,
pub height_to_weight: StorableVec<Height, Weight, MODE>,
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes, MODE>,
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes, MODE>,
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes, MODE>,
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes, MODE>,
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes, MODE>,
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes, MODE>,
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes, MODE>,
pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_locktime: StorableVec<Txindex, bitcoin::absolute::LockTime, MODE>,
pub txindex_to_txid: StorableVec<Txindex, Txid, MODE>,
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version, MODE>,
pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex, MODE>,
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex, MODE>,
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount, MODE>,
}
// const UNSAFE_BLOCKS: usize = 100;
impl StorableVecs {
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -180,67 +180,6 @@ impl StorableVecs {
})
}
pub fn push_addressbytes_if_needed(
&mut self,
index: Addresstypeindex,
addressbytes: Addressbytes,
) -> storable_vec::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
}
}
pub fn get_addressbytes(
&self,
addresstype: Addresstype,
addresstypeindex: Addresstypeindex,
) -> storable_vec::Result<Option<Addressbytes>> {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.cached_get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
_ => unreachable!(),
})
}
#[allow(unused)]
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
panic!();
@@ -355,10 +294,6 @@ impl StorableVecs {
// Ok(())
}
pub fn reset_cache(&mut self) {
self.as_mut_slice().into_par_iter().for_each(|vec| vec.reset_cache())
}
pub fn flush(&mut self, height: Height) -> io::Result<()> {
self.as_mut_slice()
.into_par_iter()
@@ -463,3 +398,66 @@ impl StorableVecs {
]
}
}
impl StorableVecs<CACHED_GETS> {
pub fn get_addressbytes(
&self,
addresstype: Addresstype,
addresstypeindex: Addresstypeindex,
) -> storable_vec::Result<Option<Addressbytes>> {
Ok(match addresstype {
Addresstype::P2PK65 => self
.p2pk65index_to_p2pk65addressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PK33 => self
.p2pk33index_to_p2pk33addressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2PKH => self
.p2pkhindex_to_p2pkhaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2SH => self
.p2shindex_to_p2shaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WPKH => self
.p2wpkhindex_to_p2wpkhaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2WSH => self
.p2wshindex_to_p2wshaddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
Addresstype::P2TR => self
.p2trindex_to_p2traddressbytes
.get(addresstypeindex)?
// .map(|v| Addressbytes::from(v.clone())),
.map(|v| Addressbytes::from(v.into_inner())),
_ => unreachable!(),
})
}
pub fn push_addressbytes_if_needed(
&mut self,
index: Addresstypeindex,
addressbytes: Addressbytes,
) -> storable_vec::Result<()> {
match addressbytes {
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
}
}
}